diff --git a/CHANGELOG.md b/CHANGELOG.md index 9083bcc..76621f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added opt-in extended sessions to gRPC workers with `WithExtendedSessions`. A session keeps an orchestration or entity loaded between turns, so DTS sends only the new messages and the worker skips the full replay. Sessions are off by default. A waiting session keeps its orchestration slot, the same as in Durable Functions, so raise `WithMaxConcurrentOrchestrationWorkItems` for slow backends or many short orchestrations. When the service returns `Unimplemented`, the worker completes work items without sessions until it reconnects. - Added required completion-transport isolation to gRPC workers: one intake and three dedicated completion/abandon connections by default, with a configurable 1–8 completion budget and bounded generation-safe retirement. `WithWorkerCompletionConnections(0)` is invalid. Borrowed workers and client listeners require separate caller-owned channels via `WithWorkerCompletionTransports`; owning factories must return distinct connections and non-nil closers for every invocation. - Added `ResourceId` connection-string support for every DTS authentication mode and optional `AuthorityHost` configuration for SDK-created DefaultAzure, WorkloadIdentity, Environment, and InteractiveBrowser credentials. Caller-supplied credentials own their authority; managed identity and developer-tool cloud configuration remain separate. - Added a flat catalogue of directly runnable DTS samples with outcome assertions, owned-resource cleanup, and process-level E2E validation selected by sample name using Go's `-run` flag. diff --git a/client/grpc_worker.go b/client/grpc_worker.go index ea36710..8d4ab5c 100644 --- a/client/grpc_worker.go +++ b/client/grpc_worker.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/microsoft/durabletask-go/api" @@ -118,6 +119,10 @@ type taskHubGrpcWorkerOptions struct { unversionedOrchestrators map[string]struct{} unversionedActivities map[string]struct{} reconnectRandom randomInt64N + extendedSessions *ExtendedSessionOptions + sessionCleanupTimeout time.Duration + // sessionTimerFn replaces the session timers in tests. + sessionTimerFn func(sessionTimerKind, time.Duration) <-chan time.Time // waitFn overrides every delay the worker imposes on itself: reconnect // backoff, transient RPC retry backoff, and work-item abandon delays. It is // only set by tests so the deterministic delay schedule can be observed @@ -144,6 +149,7 @@ func defaultTaskHubGrpcWorkerOptions() taskHubGrpcWorkerOptions { maxStreamedHistoryBytes: DefaultMaxStreamedHistoryBytes, maxOrchestratorCompletionBytes: DefaultMaxOrchestratorCompletionBytes, reconnectRandom: rand.Int64N, + sessionCleanupTimeout: defaultSessionCleanupTimeout, capabilities: []WorkerCapability{WorkerCapabilityHistoryStreaming}, } } @@ -537,6 +543,7 @@ type grpcWorkerRun struct { orchestrationSlots chan struct{} activitySlots chan struct{} entitySlots chan struct{} + sessions *sessionManager pending sync.WaitGroup retired sync.WaitGroup err error @@ -547,6 +554,11 @@ type grpcWorkerConnection struct { stream workItemsStream cancelStream context.CancelFunc closer io.Closer + // streamCtx ends with this intake generation. Sessions dispatched by the + // generation end with it. + streamCtx context.Context + sessionsUnsupported atomic.Bool + sessionCleanupFailed atomic.Bool pendingMu sync.Mutex pendingChanged *sync.Cond @@ -590,6 +602,14 @@ func (c *grpcWorkerConnection) waitForWorkItems() { c.waitForWorkItemsLocked() } +// abortForSessionCleanup ends the intake stream after a session work item +// could not be released or abandoned. The service then abandons every work +// item of this generation, and the worker reconnects. +func (c *grpcWorkerConnection) abortForSessionCleanup() { + c.sessionCleanupFailed.Store(true) + c.cancelStream() +} + func (c *grpcWorkerConnection) stopIntake() { c.pendingMu.Lock() defer c.pendingMu.Unlock() @@ -707,6 +727,9 @@ func newTaskHubGrpcWorker( return nil, err } } + if err := options.resolveExtendedSessionLimits(); err != nil { + return nil, err + } hasLargePayloadCapability := slices.Contains(options.capabilities, WorkerCapabilityLargePayloads) if hasLargePayloadCapability && options.largePayloads == nil { return nil, fmt.Errorf("large-payload capability requires worker large-payload options") @@ -1067,6 +1090,9 @@ func (w *TaskHubGrpcWorker) begin(ctx context.Context) (*grpcWorkerRun, *grpcWor activitySlots: make(chan struct{}, w.options.maxConcurrentActivities), entitySlots: make(chan struct{}, w.options.maxConcurrentEntities), } + if w.options.extendedSessions != nil { + run.sessions = newSessionManager(w.options.extendedSessions) + } w.run = run w.lastErr = nil w.mu.Unlock() @@ -1236,6 +1262,7 @@ func (w *TaskHubGrpcWorker) connect(ctx context.Context) (*grpcWorkerConnection, stream: stream, cancelStream: cancelStream, closer: closer, + streamCtx: streamCtx, }, nil } @@ -1415,7 +1442,7 @@ func isTransientWorkerError(err error) bool { if errors.Is(err, ErrStreamedHistoryLimitExceeded) { return false } - if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) { + if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) || errors.Is(err, errSessionCleanupFailed) { return true } grpcStatus, ok := status.FromError(err) diff --git a/client/grpc_worker_connection.go b/client/grpc_worker_connection.go index acde82a..b8d275c 100644 --- a/client/grpc_worker_connection.go +++ b/client/grpc_worker_connection.go @@ -18,6 +18,16 @@ import ( const maxWorkerCompletionConnections = 8 +// workerSessionStreamsPerConnection is the extended-session stream budget of +// one completion connection: half of the 100 HTTP/2 streams per connection +// that DTS documents, so other completions always have room. +const workerSessionStreamsPerConnection = 50 + +// errSessionStreamBudgetFull means that no completion connection can open +// another extended-session stream. Nothing was sent, so the caller completes +// the work item with the unary RPC instead. +var errSessionStreamBudgetFull = errors.New("worker extended-session stream budget is full") + func validateWorkerCompletionBudget(n int) error { if n < 1 || n > maxWorkerCompletionConnections { return fmt.Errorf("worker completion connections must be between 1 and %d", maxWorkerCompletionConnections) @@ -67,7 +77,10 @@ type workerCompletionTransport struct { closers []io.Closer next atomic.Uint64 closeOnce sync.Once - closeErr error + + sessionMu sync.Mutex + sessionStreams []int + closeErr error } func newWorkerCompletionTransport( @@ -158,7 +171,9 @@ func isWorkerCompletionMethod(method string) bool { protos.TaskHubSidecarService_CompleteEntityTask_FullMethodName, protos.TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName, protos.TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName, - protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName: + protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName: return true default: return false @@ -171,7 +186,71 @@ func (c *workerCompletionTransport) NewStream( method string, opts ...grpc.CallOption, ) (grpc.ClientStream, error) { - return c.intake.NewStream(ctx, desc, method, opts...) + if !isWorkerSessionStreamMethod(method) { + return c.intake.NewStream(ctx, desc, method, opts...) + } + index, ok := c.reserveSessionStream() + if !ok { + return nil, errSessionStreamBudgetFull + } + stream, err := c.completions[index].NewStream(ctx, desc, method, opts...) + if err != nil { + c.releaseSessionStream(index) + return nil, err + } + budgeted := &budgetedClientStream{ClientStream: stream} + var once sync.Once + budgeted.release = func() { once.Do(func() { c.releaseSessionStream(index) }) } + // Callers cancel the stream context when they finish with the stream. + context.AfterFunc(ctx, budgeted.release) + return budgeted, nil +} + +func isWorkerSessionStreamMethod(method string) bool { + return method == protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName || + method == protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName +} + +// reserveSessionStream picks the least-loaded completion connection that +// still has session stream budget. +func (c *workerCompletionTransport) reserveSessionStream() (int, bool) { + c.sessionMu.Lock() + defer c.sessionMu.Unlock() + if c.sessionStreams == nil { + c.sessionStreams = make([]int, len(c.completions)) + } + best := -1 + for i, streams := range c.sessionStreams { + if streams < workerSessionStreamsPerConnection && (best < 0 || streams < c.sessionStreams[best]) { + best = i + } + } + if best < 0 { + return 0, false + } + c.sessionStreams[best]++ + return best, true +} + +func (c *workerCompletionTransport) releaseSessionStream(index int) { + c.sessionMu.Lock() + c.sessionStreams[index]-- + c.sessionMu.Unlock() +} + +// budgetedClientStream returns its session stream budget when the stream +// ends with any status or when its context ends. +type budgetedClientStream struct { + grpc.ClientStream + release func() +} + +func (s *budgetedClientStream) RecvMsg(m any) error { + err := s.ClientStream.RecvMsg(m) + if err != nil { + s.release() + } + return err } func (c *workerCompletionTransport) Close() error { diff --git a/client/grpc_worker_drain_test.go b/client/grpc_worker_drain_test.go index c59fa49..0114527 100644 --- a/client/grpc_worker_drain_test.go +++ b/client/grpc_worker_drain_test.go @@ -333,7 +333,9 @@ func TestWorkerGracefulStopKeepsRejectedItemLeaseUntilAbandoned(t *testing.T) { require.True(t, worker.Running()) if stop == "shutdown deadline" { - shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), time.Second) + // Stay below the 1 s RPC timeout of the blocked abandon. With equal + // timeouts, synctest fires both at once, and the run can end first. + shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), 500*time.Millisecond) defer stopShutdown() require.ErrorIs(t, worker.Shutdown(shutdownCtx), context.DeadlineExceeded) } else { diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index 9267557..5c3f26e 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -17,6 +17,7 @@ import ( "github.com/microsoft/durabletask-go/internal/largepayload" "github.com/microsoft/durabletask-go/internal/protos" "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -73,7 +74,8 @@ func (w *TaskHubGrpcWorker) consumeConnection(run *grpcWorkerRun, connection *gr } case *protos.WorkItem_EntityRequestV2: if err := w.dispatchEntity(run, connection, workItem.GetCompletionToken(), func(ctx context.Context) { - w.processEntityV2(ctx, connection.client, workItem.GetCompletionToken(), request.EntityRequestV2) + scope := &workerSessionScope{run: run, connection: connection} + w.processEntityWorkItem(ctx, connection.client, scope, workItem.GetCompletionToken(), request.EntityRequestV2) }); err != nil { return observedMessage, err } @@ -94,6 +96,9 @@ func (w *TaskHubGrpcWorker) receiveWorkItem(run *grpcWorkerRun, connection *grpc } workItem, err := recvBeforeSilenceTimeout(connection.stream.Recv, cancelOnSilence, w.options.silentDisconnectTimeout) if err != nil { + if connection.sessionCleanupFailed.Load() { + return nil, errSessionCleanupFailed + } return nil, err } if workItem == nil { @@ -156,7 +161,9 @@ func (w *TaskHubGrpcWorker) dispatchOrchestration( ) error { return w.dispatch(run, connection, run.orchestrationSlots, func(ctx context.Context) { w.abandonOrchestration(ctx, connection.client, completionToken) }, - func(ctx context.Context) { w.processOrchestration(ctx, connection.client, completionToken, request) }, + func(ctx context.Context) { + w.processOrchestrationWorkItem(ctx, connection.client, &workerSessionScope{run: run, connection: connection}, completionToken, request) + }, ) } @@ -227,6 +234,18 @@ func (w *TaskHubGrpcWorker) processOrchestration( client protos.TaskHubSidecarServiceClient, completionToken string, request *protos.OrchestratorRequest, +) { + w.processOrchestrationWorkItem(ctx, client, nil, completionToken, request) +} + +// processOrchestrationWorkItem runs an orchestration work item. When scope +// allows extended sessions, the orchestration can stay loaded for later turns. +func (w *TaskHubGrpcWorker) processOrchestrationWorkItem( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + completionToken string, + request *protos.OrchestratorRequest, ) { pastEvents := request.PastEvents if request.RequiresHistoryStreaming { @@ -260,13 +279,29 @@ func (w *TaskHubGrpcWorker) processOrchestration( } } - results, err := w.executor.ExecuteOrchestrator( - ctx, - api.InstanceID(request.InstanceId), - pastEvents, - request.NewEvents, - request.EntityParameters, + var ( + results *task.ExecutionResults + session task.OrchestratorSession + err error ) + sessionExecutor, canSession := w.executor.(task.SessionExecutor) + if canSession && scope.available() { + results, session, err = sessionExecutor.StartOrchestratorSession( + ctx, + api.InstanceID(request.InstanceId), + pastEvents, + request.NewEvents, + request.EntityParameters, + ) + } else { + results, err = w.executor.ExecuteOrchestrator( + ctx, + api.InstanceID(request.InstanceId), + pastEvents, + request.NewEvents, + request.EntityParameters, + ) + } var delayed workItemAbandonDelayError if errors.As(err, &delayed) { w.logger.Warnf("%s: orchestration work item rejected; abandoning it: %v", request.InstanceId, err) @@ -280,14 +315,54 @@ func (w *TaskHubGrpcWorker) processOrchestration( return } + turn := orchestrationTurn{ + instanceID: request.InstanceId, + token: completionToken, + traceContext: request.OrchestrationTraceContext, + parentTrace: executionStartedTraceContext(pastEvents, request.NewEvents), + } + response, ok := w.orchestratorResponse(ctx, turn, results, err, func() { + w.abandonOrchestration(ctx, client, completionToken) + }) + if !ok { + if session != nil { + session.Close() + } + return + } + if session != nil { + w.runOrchestrationSession(ctx, client, scope, turn, session, response) + return + } + w.completeOrchestration(ctx, client, nil, turn, response, 0) +} + +// orchestrationTurn holds what every turn of one orchestration work item +// needs to build and send its response. +type orchestrationTurn struct { + instanceID string + token string + traceContext *protos.OrchestrationTraceContext + parentTrace *protos.TraceContext +} + +// orchestratorResponse builds the response for one turn. It returns false +// after it called abandon. +func (w *TaskHubGrpcWorker) orchestratorResponse( + ctx context.Context, + turn orchestrationTurn, + results *task.ExecutionResults, + err error, + abandon func(), +) (*protos.OrchestratorResponse, bool) { response := &protos.OrchestratorResponse{ - InstanceId: request.InstanceId, - CompletionToken: completionToken, - OrchestrationTraceContext: request.OrchestrationTraceContext, + InstanceId: turn.instanceID, + CompletionToken: turn.token, + OrchestrationTraceContext: turn.traceContext, } switch { case err != nil: - w.logger.Errorf("%s: orchestration execution failed: %v", request.InstanceId, err) + w.logger.Errorf("%s: orchestration execution failed: %v", turn.instanceID, err) response.Actions = []*protos.OrchestratorAction{helpers.NewCompleteOrchestrationAction( -1, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, @@ -308,38 +383,152 @@ func (w *TaskHubGrpcWorker) processOrchestration( )} default: response = results.Response - response.InstanceId = request.InstanceId - response.CompletionToken = completionToken + response.InstanceId = turn.instanceID + response.CompletionToken = turn.token if response.OrchestrationTraceContext == nil { - response.OrchestrationTraceContext = request.OrchestrationTraceContext + response.OrchestrationTraceContext = turn.traceContext } - if err := populateOrchestratorActionTraceContexts( - response.Actions, - pastEvents, - request.NewEvents, - ); err != nil { - w.logger.Warnf("%s: failed to populate action trace contexts: %v", request.InstanceId, err) + if err := populateActionTraceContexts(response.Actions, turn.parentTrace); err != nil { + w.logger.Warnf("%s: failed to populate action trace contexts: %v", turn.instanceID, err) } } if err := largepayload.TransformOrchestratorResponse(ctx, w.options.largePayloads, response); err != nil { - w.logger.Errorf("%s: failed to externalize orchestration response payloads: %v", request.InstanceId, err) - w.abandonOrchestration(ctx, client, completionToken) - return + w.logger.Errorf("%s: failed to externalize orchestration response payloads: %v", turn.instanceID, err) + abandon() + return nil, false } + return failOversizedOrchestratorResponse(response, w.orchestratorCompletionLimit()), true +} - response = failOversizedOrchestratorResponse(response, w.options.maxOrchestratorCompletionBytes) - err = w.executeRPCWithRetry(ctx, "complete orchestration task", func(callCtx context.Context) error { +// completeOrchestration sends a turn with the unary RPC. The count and token +// are set right before sending, because earlier steps may rebuild the +// response. scope is set for a work item that is in a session. +func (w *TaskHubGrpcWorker) completeOrchestration( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + turn orchestrationTurn, + response *protos.OrchestratorResponse, + completionCount int32, +) { + response.CompletionToken = turn.token + response.CompletionCount = completionCount + err := w.executeRPCWithRetry(ctx, "complete orchestration task", func(callCtx context.Context) error { _, callErr := client.CompleteOrchestratorTask(callCtx, response) return callErr }) if err != nil { if isWorkItemGone(err) { - w.logger.Warnf("%s: orchestration work item was no longer available before completion", request.InstanceId) + w.logger.Warnf("%s: orchestration work item was no longer available before completion", turn.instanceID) return } - w.logger.Errorf("%s: failed to complete orchestration work item: %v", request.InstanceId, err) - w.abandonOrchestration(ctx, client, completionToken) + w.logger.Errorf("%s: failed to complete orchestration work item: %v", turn.instanceID, err) + if scope != nil { + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "abandon", abandonOrchestrationOnce(client, turn.token)) + return + } + w.abandonOrchestration(ctx, client, turn.token) + } +} + +// runOrchestrationSession keeps an orchestration loaded while the service +// sends new batches. This goroutine owns the session: it sends each turn, +// waits for the next batch, and is the only caller of release and abandon. +func (w *TaskHubGrpcWorker) runOrchestrationSession( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + turn orchestrationTurn, + session task.OrchestratorSession, + response *protos.OrchestratorResponse, +) { + lease := scope.run.sessions.acquire(orchestrationSessionKind) + defer lease.release() + var turnCount int32 + for { + if session != nil && (lease == nil || !scope.available() || !canContinueOrchestrationSession(response)) { + session.Close() + session = nil + } + if session == nil { + w.completeOrchestration(ctx, client, sessionScopeAfterAck(scope, turnCount), turn, response, turnCount) + return + } + var historyBase int32 + outcome, batch := w.awaitSessionBatch(ctx, scope, sessionCall{ + instanceID: turn.instanceID, + open: func(callCtx context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + response.CompletionToken = turn.token + response.CompletionCount = turnCount + return client.CompleteOrchestratorTaskWithExtendedSession(callCtx, response) + }, + onAck: func(ack *protos.ExtendedSessionAck) { + session.Commit(response.Actions) + historyBase = ack.GetHistoryEventCount() + turnCount++ + }, + release: releaseOrchestrationOnce(client, turn.token), + abandon: abandonOrchestrationOnce(client, turn.token), + }) + switch outcome { + case sessionUseUnary: + session.Close() + session = nil + continue + case sessionEnded: + session.Close() + return + } + + payloadRequest := &protos.OrchestratorRequest{NewEvents: batch} + if err := largepayload.TransformOrchestratorRequest(ctx, w.options.largePayloads, payloadRequest); err != nil { + w.logger.Errorf("%s: failed to hydrate session batch payloads: %v", turn.instanceID, err) + session.Close() + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "release", releaseOrchestrationOnce(client, turn.token)) + return + } + results, next, err := session.Resume(ctx, batch, int(historyBase)) + if err != nil { + w.logger.Errorf("%s: failed to run session batch: %v", turn.instanceID, err) + session.Close() + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "release", releaseOrchestrationOnce(client, turn.token)) + return + } + session = next + var ok bool + if response, ok = w.orchestratorResponse(ctx, turn, results, nil, func() { + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "abandon", abandonOrchestrationOnce(client, turn.token)) + }); !ok { + if session != nil { + session.Close() + } + return + } + } +} + +// orchestratorCompletionLimit is the size bound for a response before its +// completion count is set. Sessions send a nonzero count, so they keep room. +func (w *TaskHubGrpcWorker) orchestratorCompletionLimit() int { + if w.options.extendedSessions != nil { + return w.options.maxOrchestratorCompletionBytes - sessionCompletionCountReserve + } + return w.options.maxOrchestratorCompletionBytes +} + +// canContinueOrchestrationSession reports whether a response may keep the +// session. A worker-built failure, for example for an oversized response, +// ends it. +func canContinueOrchestrationSession(response *protos.OrchestratorResponse) bool { + if response.NumEventsProcessed != nil { + return false + } + for _, action := range response.Actions { + if action.GetCompleteOrchestration() != nil { + return false + } } + return true } // failOversizedOrchestratorResponse converts a response that modern DTS cannot @@ -394,7 +583,10 @@ func populateOrchestratorActionTraceContexts( pastEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent, ) error { - parent := executionStartedTraceContext(pastEvents, newEvents) + return populateActionTraceContexts(actions, executionStartedTraceContext(pastEvents, newEvents)) +} + +func populateActionTraceContexts(actions []*protos.OrchestratorAction, parent *protos.TraceContext) error { if parent == nil { return nil } @@ -605,44 +797,77 @@ func (w *TaskHubGrpcWorker) processEntityV2( completionToken string, request *protos.EntityRequest, ) { + w.processEntityWorkItem(ctx, client, nil, completionToken, request) +} + +// processEntityWorkItem runs a V2 entity work item. When scope allows +// extended sessions, the entity state stays cached for later batches. +func (w *TaskHubGrpcWorker) processEntityWorkItem( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + completionToken string, + request *protos.EntityRequest, +) { + result, state, ok := w.entityBatchResult(ctx, completionToken, request, func() { + w.abandonEntity(ctx, client, completionToken) + }) + if !ok { + return + } + if result.FailureDetails == nil && scope.available() { + w.runEntitySession(ctx, client, scope, request.GetInstanceId(), completionToken, result, state) + return + } + w.completeEntityBatchResult(ctx, client, nil, request.GetInstanceId(), result, 0) +} + +// entityBatchResult runs one entity batch and returns the result to send and +// the entity state before externalization, which a session caches. It returns +// false after it called abandon. +func (w *TaskHubGrpcWorker) entityBatchResult( + ctx context.Context, + completionToken string, + request *protos.EntityRequest, + abandon func(), +) (*protos.EntityBatchResult, *wrapperspb.StringValue, bool) { batch, operationInfos, err := entityBatchFromRequestV2(request) if err != nil { w.logger.Errorf("invalid V2 entity work item: %v", err) - w.abandonEntity(ctx, client, completionToken) - return + abandon() + return nil, nil, false } entityID, parseErr := api.EntityIDFromString(batch.GetInstanceId()) if parseErr != nil { w.logger.Errorf("%s: invalid entity instance ID: %v", batch.GetInstanceId(), parseErr) - w.abandonEntity(ctx, client, completionToken) - return + abandon() + return nil, nil, false } if !matchesEntityWorkItemFilters(w.options.workItemFilters, entityID.Name) { w.logger.Warnf("%s: entity work item does not match configured filters; abandoning it", request.GetInstanceId()) - w.abandonEntity(ctx, client, completionToken) - return + abandon() + return nil, nil, false } if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, batch); err != nil { w.logger.Errorf("%s: failed to hydrate entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { - w.abandonEntity(ctx, client, completionToken) - return + abandon() + return nil, nil, false } - w.completeEntityBatchFailure(ctx, client, request.GetInstanceId(), completionToken, err) - return + return newEntityBatchFailure(completionToken, err), nil, true } executor, ok := w.executor.(task.EntityExecutor) if !ok { w.logger.Error("task executor does not support entity work items") - w.abandonEntity(ctx, client, completionToken) - return + abandon() + return nil, nil, false } result, err := executeEntitySafely(ctx, executor, batch) if err != nil { if entityProcessingCanceled(ctx, err) { w.logger.Warnf("%s: entity execution canceled; abandoning work item", request.GetInstanceId()) - w.abandonEntity(ctx, client, completionToken) - return + abandon() + return nil, nil, false } w.logger.Errorf("%s: entity execution failed: %v", request.GetInstanceId(), err) result = newEntityBatchFailure(completionToken, err) @@ -656,17 +881,89 @@ func (w *TaskHubGrpcWorker) processEntityV2( w.logger.Errorf("%s: invalid entity executor result: %v", request.GetInstanceId(), err) result = newEntityBatchFailure(completionToken, err) } + // Externalizing replaces the state field, so this keeps the full value. + state := result.EntityState if err := largepayload.TransformEntityBatchResult(ctx, w.options.largePayloads, result); err != nil { w.logger.Errorf("%s: failed to externalize entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { - w.abandonEntity(ctx, client, completionToken) + abandon() + return nil, nil, false + } + return newEntityBatchFailure(completionToken, err), nil, true + } + return result, state, true +} + +// runEntitySession keeps an entity's state cached while the service sends new +// operation batches. Each batch runs with the state of the last accepted +// result. A failed batch ends the session with the unary RPC, and the service +// then abandons the work item. +func (w *TaskHubGrpcWorker) runEntitySession( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + instanceID string, + completionToken string, + result *protos.EntityBatchResult, + resultState *wrapperspb.StringValue, +) { + lease := scope.run.sessions.acquire(entitySessionKind) + defer lease.release() + streaming := lease != nil + var ( + turnCount int32 + state *wrapperspb.StringValue + ) + for { + if !streaming || !scope.available() || result.FailureDetails != nil { + w.completeEntityBatchResult(ctx, client, sessionScopeAfterAck(scope, turnCount), instanceID, result, turnCount) + return + } + outcome, batch := w.awaitSessionBatch(ctx, scope, sessionCall{ + instanceID: instanceID, + open: func(callCtx context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + result.CompletionToken = completionToken + result.CompletionCount = turnCount + return client.CompleteEntityTaskWithExtendedSession(callCtx, result) + }, + onAck: func(*protos.ExtendedSessionAck) { + state = resultState + turnCount++ + }, + release: releaseEntityOnce(client, completionToken), + abandon: abandonEntityOnce(client, completionToken), + }) + switch outcome { + case sessionUseUnary: + streaming = false + continue + case sessionEnded: + return + } + var ok bool + result, resultState, ok = w.entityBatchResult(ctx, completionToken, &protos.EntityRequest{ + InstanceId: instanceID, + EntityState: state, + OperationRequests: entityOperationEvents(batch), + }, func() { + w.cleanupSessionWorkItem(ctx, scope, instanceID, "abandon", abandonEntityOnce(client, completionToken)) + }) + if !ok { return } - w.completeEntityBatchFailure(ctx, client, request.GetInstanceId(), completionToken, err) - return } +} - w.completeEntityBatchResult(ctx, client, request.GetInstanceId(), result) +// entityOperationEvents drops the control events that may start a session +// batch. Entity batches contain only operation requests. +func entityOperationEvents(batch []*protos.HistoryEvent) []*protos.HistoryEvent { + operations := make([]*protos.HistoryEvent, 0, len(batch)) + for _, event := range batch { + if event.GetOrchestratorStarted() == nil { + operations = append(operations, event) + } + } + return operations } func executeEntitySafely( @@ -705,16 +1002,6 @@ func newEntityBatchFailure(completionToken string, err error) *protos.EntityBatc } } -func (w *TaskHubGrpcWorker) completeEntityBatchFailure( - ctx context.Context, - client protos.TaskHubSidecarServiceClient, - instanceID string, - completionToken string, - err error, -) { - w.completeEntityBatchResult(ctx, client, instanceID, newEntityBatchFailure(completionToken, err)) -} - func validateEntityBatchResult( result *protos.EntityBatchResult, operationInfos []*protos.OperationInfo, @@ -750,12 +1037,18 @@ func entityProcessingCanceled(ctx context.Context, err error) bool { errors.Is(err, context.DeadlineExceeded) } +// completeEntityBatchResult sends an entity result with the unary RPC. The +// count is set right before sending, because a failure result is a new +// message. scope is set for a work item that is in a session. func (w *TaskHubGrpcWorker) completeEntityBatchResult( ctx context.Context, client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, instanceID string, result *protos.EntityBatchResult, + completionCount int32, ) { + result.CompletionCount = completionCount err := w.executeRPCWithRetry(ctx, "complete entity task", func(callCtx context.Context) error { _, callErr := client.CompleteEntityTask(callCtx, result) return callErr @@ -766,6 +1059,10 @@ func (w *TaskHubGrpcWorker) completeEntityBatchResult( return } w.logger.Errorf("%s: failed to complete entity work item: %v", instanceID, err) + if scope != nil { + w.cleanupSessionWorkItem(ctx, scope, instanceID, "abandon", abandonEntityOnce(client, result.CompletionToken)) + return + } w.abandonEntity(ctx, client, result.CompletionToken) } } diff --git a/client/grpc_worker_session.go b/client/grpc_worker_session.go new file mode 100644 index 0000000..1328ea6 --- /dev/null +++ b/client/grpc_worker_session.go @@ -0,0 +1,498 @@ +package client + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// sessionCompletionCountReserve is the largest encoding of a non-negative +// completionCount field: a one-byte tag and a varint of up to five bytes. +const sessionCompletionCountReserve = 6 + +// DefaultExtendedSessionTimeout is the default idle and transfer timeout of an +// extended session. +const DefaultExtendedSessionTimeout = 30 * time.Second + +// defaultSessionCleanupTimeout bounds all release or abandon attempts for one +// session work item. +const defaultSessionCleanupTimeout = 30 * time.Second + +// errSessionCleanupFailed ends an intake generation after a session work item +// could not be released or abandoned. Closing intake makes the service abandon +// every work item of that generation. +var errSessionCleanupFailed = errors.New("extended-session cleanup failed; reconnecting the work item stream") + +// ExtendedSessionOptions configures extended sessions. In an extended session +// the worker keeps an orchestration or entity loaded in memory after a turn, +// and the service sends only the new messages for the next turn. This saves +// the replay of the full history. +// +// A waiting session keeps its concurrency slot, a service credit, and an +// open stream, the same as in Durable Functions. With a slow backend or many +// short orchestrations, waiting sessions can leave fewer slots for new work, +// so raise [WithMaxConcurrentOrchestrationWorkItems]. Leave sessions off for +// hosts that scale to zero and behind proxies that close idle streams quickly. +type ExtendedSessionOptions struct { + // IdleTimeout is the longest time a session waits for new messages after + // the service accepts a turn. Zero means [DefaultExtendedSessionTimeout]. + IdleTimeout time.Duration + // TransferTimeout is the longest time from the first part of a batch to + // the end of the batch. Zero means [DefaultExtendedSessionTimeout]. + TransferTimeout time.Duration + // MaxOrchestrationSessions limits waiting orchestration sessions. Zero + // means half of the orchestration concurrency limit, rounded down. The + // value must be lower than that limit, so new work always has a slot. + MaxOrchestrationSessions int + // MaxEntitySessions limits waiting entity sessions in the same way. + MaxEntitySessions int +} + +// WithExtendedSessions turns on extended sessions. Sessions are off unless +// this option is set. When the service does not support sessions, the worker +// completes work items without them. +func WithExtendedSessions(options ExtendedSessionOptions) TaskHubGrpcWorkerOption { + return func(target *taskHubGrpcWorkerOptions) error { + if options.IdleTimeout < 0 || options.TransferTimeout < 0 { + return fmt.Errorf("extended session timeouts cannot be negative") + } + if options.MaxOrchestrationSessions < 0 || options.MaxEntitySessions < 0 { + return fmt.Errorf("extended session limits cannot be negative") + } + // Each worker resolves its own defaults, so it needs its own copy. + copied := options + if copied.IdleTimeout == 0 { + copied.IdleTimeout = DefaultExtendedSessionTimeout + } + if copied.TransferTimeout == 0 { + copied.TransferTimeout = DefaultExtendedSessionTimeout + } + target.extendedSessions = &copied + return nil + } +} + +// resolveExtendedSessionLimits applies the default session limits after all +// concurrency options are known. +func (options *taskHubGrpcWorkerOptions) resolveExtendedSessionLimits() error { + sessions := options.extendedSessions + if sessions == nil { + return nil + } + resolve := func(kind string, limit *int, concurrency int) error { + if *limit == 0 { + *limit = concurrency / 2 + return nil + } + if *limit >= concurrency { + return fmt.Errorf("maximum %s sessions must be lower than the %s concurrency limit %d", kind, kind, concurrency) + } + return nil + } + if err := resolve("orchestration", &sessions.MaxOrchestrationSessions, options.maxConcurrentOrchestrations); err != nil { + return err + } + return resolve("entity", &sessions.MaxEntitySessions, options.maxConcurrentEntities) +} + +type sessionKind int + +const ( + orchestrationSessionKind sessionKind = iota + entitySessionKind +) + +// sessionManager enforces the session limits of one worker run. +type sessionManager struct { + options *ExtendedSessionOptions + + mu sync.Mutex + orchestrations int + entities int +} + +// sessionLease is one session's place within the limits. +type sessionLease struct { + manager *sessionManager + kind sessionKind + once sync.Once +} + +func newSessionManager(options *ExtendedSessionOptions) *sessionManager { + return &sessionManager{options: options} +} + +// acquire returns nil when the limit for kind is reached. +func (m *sessionManager) acquire(kind sessionKind) *sessionLease { + m.mu.Lock() + defer m.mu.Unlock() + count, limit := m.counter(kind) + if *count >= limit { + return nil + } + *count++ + return &sessionLease{manager: m, kind: kind} +} + +func (m *sessionManager) counter(kind sessionKind) (*int, int) { + if kind == entitySessionKind { + return &m.entities, m.options.MaxEntitySessions + } + return &m.orchestrations, m.options.MaxOrchestrationSessions +} + +// release returns the lease. It is safe to call on a nil lease. +func (l *sessionLease) release() { + if l == nil { + return + } + l.once.Do(func() { + l.manager.mu.Lock() + defer l.manager.mu.Unlock() + count, _ := l.manager.counter(l.kind) + *count-- + }) +} + +// workerSessionScope is the intake generation that dispatched a work item. +// Sessions belong to it: when its stream ends, the service abandons the work +// items, so the sessions close without a release. +type workerSessionScope struct { + run *grpcWorkerRun + connection *grpcWorkerConnection +} + +// sessionScopeAfterAck returns scope once the service has accepted at least +// one session turn, so the work item is in a session. Otherwise it returns +// nil, and the work item is handled like any other. +func sessionScopeAfterAck(scope *workerSessionScope, turnCount int32) *workerSessionScope { + if turnCount == 0 { + return nil + } + return scope +} + +// generationContext returns a context that also ends when the intake stream +// of this generation ends. +func (s *workerSessionScope) generationContext(ctx context.Context) (context.Context, context.CancelFunc) { + generationCtx, cancel := context.WithCancel(ctx) + stop := context.AfterFunc(s.connection.streamCtx, cancel) + return generationCtx, func() { + stop() + cancel() + } +} + +// available reports whether a new session turn may use the streaming RPC. +func (s *workerSessionScope) available() bool { + return s != nil && s.run.sessions != nil && + !s.connection.sessionsUnsupported.Load() && + s.run.intakeCtx.Err() == nil && + s.connection.streamCtx.Err() == nil +} + +type sessionOutcome int + +const ( + // sessionBatchReady means the turn was accepted and a full batch arrived. + sessionBatchReady sessionOutcome = iota + // sessionUseUnary means nothing was applied; complete the same turn with + // the unary RPC and the same count. + sessionUseUnary + // sessionEnded means the session is over and the work item needs no more + // calls from this worker. + sessionEnded +) + +type sessionTimerKind int + +const ( + sessionAckTimer sessionTimerKind = iota + sessionIdleTimer + sessionTransferTimer +) + +// sessionCall describes one turn of a session completion. +type sessionCall struct { + instanceID string + open func(context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) + // onAck runs when the service accepts the turn. + onAck func(*protos.ExtendedSessionAck) + release func(context.Context) error + abandon func(context.Context) error +} + +type sessionMessage struct { + response *protos.CompleteTaskWithExtendedSessionResponse + err error +} + +// awaitSessionBatch completes one turn with the streaming RPC and waits for +// the next batch. Transient errors before the acknowledgement retry the same +// request. It releases or abandons the work item itself when the session +// cannot continue. +func (w *TaskHubGrpcWorker) awaitSessionBatch( + ctx context.Context, + scope *workerSessionScope, + call sessionCall, +) (sessionOutcome, []*protos.HistoryEvent) { + for attempt := 1; ; attempt++ { + outcome, batch, preAckErr := w.runSessionStream(ctx, scope, call) + if preAckErr == nil { + return outcome, batch + } + unimplemented := status.Code(preAckErr) == codes.Unimplemented + if unimplemented { + w.logger.Infof("%s: the service does not support extended sessions; completing without them until the next reconnect", call.instanceID) + scope.connection.sessionsUnsupported.Store(true) + } + switch { + case (unimplemented || errors.Is(preAckErr, errSessionStreamBudgetFull)) && attempt == 1: + return sessionUseUnary, nil + case unimplemented || errors.Is(preAckErr, errSessionStreamBudgetFull): + // An earlier attempt may have been applied without its + // acknowledgement. A unary completion with the same count could + // then fail and leave the session open, so abandon instead. + w.logger.Warnf("%s: cannot retry the completion on a session stream; abandoning the work item: %v", call.instanceID, preAckErr) + w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "abandon", call.abandon) + return sessionEnded, nil + case ctx.Err() != nil || scope.connection.streamCtx.Err() != nil: + return sessionEnded, nil + case isTransientWorkerRPCError(preAckErr) && attempt < w.options.transientRetryMaxAttempts: + waitCtx, cancelWait := scope.generationContext(ctx) + err := w.wait(waitCtx, w.retryDelay(attempt)) + cancelWait() + if err != nil { + return sessionEnded, nil + } + default: + w.logger.Warnf("%s: extended-session completion failed before the service accepted it; abandoning the work item: %v", call.instanceID, preAckErr) + w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "abandon", call.abandon) + return sessionEnded, nil + } + } +} + +// runSessionStream runs one streaming completion attempt. This goroutine owns +// the stream and all of its timers, so a timeout and a message cannot both +// win. A non-nil error means that the attempt failed before the service +// accepted the turn. +func (w *TaskHubGrpcWorker) runSessionStream( + ctx context.Context, + scope *workerSessionScope, + call sessionCall, +) (sessionOutcome, []*protos.HistoryEvent, error) { + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + // Opening the stream also sends the request, which can block. The + // acknowledgement timeout covers it, and so does the end of the intake + // generation. + opened := time.Now() + openTimer := time.AfterFunc(w.options.rpcTimeout, cancel) + stopGenerationLink := context.AfterFunc(scope.connection.streamCtx, cancel) + stream, err := call.open(streamCtx) + openTimedOut := !openTimer.Stop() + stopGenerationLink() + if openTimedOut { + return 0, nil, status.Error(codes.DeadlineExceeded, "opening the session stream timed out") + } + if err != nil { + return 0, nil, err + } + messages := make(chan sessionMessage) + received := make(chan struct{}) + go func() { + defer close(received) + for { + response, recvErr := stream.Recv() + select { + case messages <- sessionMessage{response: response, err: recvErr}: + case <-streamCtx.Done(): + return + } + if recvErr != nil { + return + } + } + }() + defer func() { + cancel() + <-received + }() + + timer, stopTimer := w.sessionTimer(sessionAckTimer, max(0, w.options.rpcTimeout-time.Since(opened))) + defer func() { stopTimer() }() + restartTimer := func(kind sessionTimerKind, timeout time.Duration) { + stopTimer() + timer, stopTimer = w.sessionTimer(kind, timeout) + } + end := func(reason string, release bool) (sessionOutcome, []*protos.HistoryEvent, error) { + cancel() + w.logger.Debugf("%s: extended session ended: %s", call.instanceID, reason) + if release { + w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "release", call.release) + } + return sessionEnded, nil, nil + } + + acked, transferring := false, false + var ( + batch []*protos.HistoryEvent + batchBytes int64 + ) + for { + // Shutdown ends a session only while it waits. + var stopping <-chan struct{} + if acked && !transferring { + stopping = scope.run.intakeCtx.Done() + } + select { + case <-ctx.Done(): + return end("the worker stopped", false) + case <-scope.connection.streamCtx.Done(): + return end("the work item stream ended", false) + case <-stopping: + return end("the worker is shutting down", true) + case <-timer: + switch { + case !acked: + return 0, nil, status.Error(codes.DeadlineExceeded, "the service did not acknowledge the completion in time") + case transferring: + return end("the batch transfer timed out", true) + default: + return end("no new messages arrived before the idle timeout", true) + } + case message := <-messages: + if message.err != nil { + switch { + case !acked && errors.Is(message.err, io.EOF): + return 0, nil, status.Error(codes.Unknown, "the session stream ended before the acknowledgement") + case !acked: + return 0, nil, message.err + case transferring && errors.Is(message.err, io.EOF): + return sessionBatchReady, batch, nil + case errors.Is(message.err, io.EOF): + return end("the service ended the session", false) + default: + return end(fmt.Sprintf("the session stream failed: %v", message.err), true) + } + } + switch response := message.response.GetResponse().(type) { + case *protos.CompleteTaskWithExtendedSessionResponse_Ack: + if acked { + return end("the service sent a second acknowledgement", true) + } + acked = true + call.onAck(response.Ack) + restartTimer(sessionIdleTimer, w.options.extendedSessions.IdleTimeout) + case *protos.CompleteTaskWithExtendedSessionResponse_HealthPing: + // A ping shows only that the connection is alive. + case *protos.CompleteTaskWithExtendedSessionResponse_NewMessages: + if !acked { + return 0, nil, status.Error(codes.Internal, "the service sent messages before the acknowledgement") + } + if !transferring { + transferring = true + restartTimer(sessionTransferTimer, w.options.extendedSessions.TransferTimeout) + } + // Apply the streamed-history limits to the whole batch. + events := response.NewMessages.GetEvents() + if limit := w.options.maxStreamedHistoryEvents; limit > 0 && len(events) > limit-len(batch) { + return end(fmt.Sprintf("the batch exceeds %d events", limit), true) + } + chunkBytes := int64(proto.Size(response.NewMessages)) + if limit := w.options.maxStreamedHistoryBytes; limit > 0 && chunkBytes > limit-batchBytes { + return end(fmt.Sprintf("the batch exceeds %d bytes", limit), true) + } + batch = append(batch, events...) + batchBytes += chunkBytes + default: + if !acked { + return 0, nil, status.Error(codes.Internal, "the service sent an unknown session message") + } + return end("the service sent an unknown session message", true) + } + } + } +} + +func (w *TaskHubGrpcWorker) sessionTimer(kind sessionTimerKind, timeout time.Duration) (<-chan time.Time, func()) { + if w.options.sessionTimerFn != nil { + return w.options.sessionTimerFn(kind, timeout), func() {} + } + timer := time.NewTimer(timeout) + return timer.C, func() { timer.Stop() } +} + +// cleanupSessionWorkItem releases or abandons a session work item. Attempts +// stop at one absolute deadline. If that passes, the worker ends the intake +// stream, so the service abandons all work items of this generation. +func (w *TaskHubGrpcWorker) cleanupSessionWorkItem( + ctx context.Context, + scope *workerSessionScope, + instanceID string, + operation string, + action func(context.Context) error, +) { + // When the intake stream ends, the service takes the work item back, so + // cleanup stops too. + generationCtx, cancelGeneration := scope.generationContext(ctx) + defer cancelGeneration() + cleanupCtx, cancel := context.WithTimeout(generationCtx, w.options.sessionCleanupTimeout) + defer cancel() + for attempt := 1; ; attempt++ { + callCtx, cancelCall := context.WithTimeout(cleanupCtx, w.options.rpcTimeout) + err := action(callCtx) + cancelCall() + if err == nil || isWorkItemGone(err) || ctx.Err() != nil || scope.connection.streamCtx.Err() != nil { + return + } + if isTransientWorkerRPCError(err) && cleanupCtx.Err() == nil && + w.wait(cleanupCtx, w.retryDelay(attempt)) == nil && cleanupCtx.Err() == nil { + continue + } + if ctx.Err() != nil || scope.connection.streamCtx.Err() != nil { + return + } + w.logger.Errorf("%s: failed to %s the session work item; reconnecting the work item stream: %v", instanceID, operation, err) + scope.connection.abortForSessionCleanup() + return + } +} + +func releaseOrchestrationOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.ReleaseTaskOrchestratorWorkItem(ctx, &protos.ReleaseTaskWorkItemRequest{CompletionToken: token}) + return err + } +} + +func releaseEntityOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.ReleaseTaskEntityWorkItem(ctx, &protos.ReleaseTaskWorkItemRequest{CompletionToken: token}) + return err + } +} + +func abandonOrchestrationOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.AbandonTaskOrchestratorWorkItem(ctx, &protos.AbandonOrchestrationTaskRequest{CompletionToken: token}) + return err + } +} + +func abandonEntityOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.AbandonTaskEntityWorkItem(ctx, &protos.AbandonEntityTaskRequest{CompletionToken: token}) + return err + } +} diff --git a/client/grpc_worker_session_test.go b/client/grpc_worker_session_test.go new file mode 100644 index 0000000..0ed2904 --- /dev/null +++ b/client/grpc_worker_session_test.go @@ -0,0 +1,1258 @@ +package client + +import ( + "context" + "errors" + "io" + "math" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// fakeSessionStream is a session completion stream that the test drives. +type fakeSessionStream struct { + grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse] + ctx context.Context + messages chan sessionMessage +} + +func (s *fakeSessionStream) Recv() (*protos.CompleteTaskWithExtendedSessionResponse, error) { + select { + case <-s.ctx.Done(): + return nil, status.FromContextError(s.ctx.Err()).Err() + case message := <-s.messages: + return message.response, message.err + } +} + +func (s *fakeSessionStream) ack(count int32) { + s.messages <- sessionMessage{response: &protos.CompleteTaskWithExtendedSessionResponse{ + Response: &protos.CompleteTaskWithExtendedSessionResponse_Ack{Ack: &protos.ExtendedSessionAck{HistoryEventCount: count}}, + }} +} + +func (s *fakeSessionStream) ping() { + s.messages <- sessionMessage{response: &protos.CompleteTaskWithExtendedSessionResponse{ + Response: &protos.CompleteTaskWithExtendedSessionResponse_HealthPing{HealthPing: &protos.HealthPing{}}, + }} +} + +func (s *fakeSessionStream) chunk(events ...*protos.HistoryEvent) { + s.messages <- sessionMessage{response: &protos.CompleteTaskWithExtendedSessionResponse{ + Response: &protos.CompleteTaskWithExtendedSessionResponse_NewMessages{NewMessages: &protos.HistoryChunk{Events: events}}, + }} +} + +func (s *fakeSessionStream) end(err error) { + if err == nil { + err = io.EOF + } + s.messages <- sessionMessage{err: err} +} + +// batch sends a complete batch and ends the stream with OK. +func (s *fakeSessionStream) batch(events ...*protos.HistoryEvent) { + s.chunk(events...) + s.end(nil) +} + +type fakeSessionCall struct { + orchestration *protos.OrchestratorResponse + entity *protos.EntityBatchResult + stream *fakeSessionStream +} + +func (c *fakeSessionCall) count() int32 { + if c.entity != nil { + return c.entity.CompletionCount + } + return c.orchestration.CompletionCount +} + +// sessionSchedulerClient adds the extended-session RPCs to the fake scheduler. +type sessionSchedulerClient struct { + *fakeSchedulerClient + calls chan *fakeSessionCall + openErr func() error + // openWait runs before the stream opens and can block like a send. + openWait func(context.Context) error + unaryErr func() error + releaseErr func() error + abandonErr func() error + + releases atomic.Int32 + orchestrationAbort atomic.Int32 + connects atomic.Int32 +} + +func newSessionSchedulerClient() *sessionSchedulerClient { + return &sessionSchedulerClient{ + fakeSchedulerClient: &fakeSchedulerClient{stream: newFakeWorkItemStream(16)}, + calls: make(chan *fakeSessionCall, 16), + } +} + +func (c *sessionSchedulerClient) GetWorkItems( + ctx context.Context, + request *protos.GetWorkItemsRequest, + opts ...grpc.CallOption, +) (protos.TaskHubSidecarService_GetWorkItemsClient, error) { + c.connects.Add(1) + return c.fakeSchedulerClient.GetWorkItems(ctx, request, opts...) +} + +func (c *sessionSchedulerClient) CompleteOrchestratorTask( + ctx context.Context, + response *protos.OrchestratorResponse, + opts ...grpc.CallOption, +) (*protos.CompleteTaskResponse, error) { + if c.unaryErr != nil { + if err := c.unaryErr(); err != nil { + return nil, err + } + } + return c.fakeSchedulerClient.CompleteOrchestratorTask(ctx, response, opts...) +} + +func (c *sessionSchedulerClient) open(ctx context.Context, call *fakeSessionCall) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + if c.openWait != nil { + if err := c.openWait(ctx); err != nil { + return nil, err + } + } + if c.openErr != nil { + if err := c.openErr(); err != nil { + return nil, err + } + } + call.stream = &fakeSessionStream{ctx: ctx, messages: make(chan sessionMessage, 16)} + c.calls <- call + return call.stream, nil +} + +func (c *sessionSchedulerClient) CompleteOrchestratorTaskWithExtendedSession( + ctx context.Context, + response *protos.OrchestratorResponse, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + return c.open(ctx, &fakeSessionCall{orchestration: proto.Clone(response).(*protos.OrchestratorResponse)}) +} + +func (c *sessionSchedulerClient) CompleteEntityTaskWithExtendedSession( + ctx context.Context, + result *protos.EntityBatchResult, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + return c.open(ctx, &fakeSessionCall{entity: proto.Clone(result).(*protos.EntityBatchResult)}) +} + +func (c *sessionSchedulerClient) release() (*protos.ReleaseTaskWorkItemResponse, error) { + c.releases.Add(1) + if c.releaseErr != nil { + if err := c.releaseErr(); err != nil { + return nil, err + } + } + return &protos.ReleaseTaskWorkItemResponse{}, nil +} + +func (c *sessionSchedulerClient) ReleaseTaskOrchestratorWorkItem( + context.Context, + *protos.ReleaseTaskWorkItemRequest, + ...grpc.CallOption, +) (*protos.ReleaseTaskWorkItemResponse, error) { + return c.release() +} + +func (c *sessionSchedulerClient) ReleaseTaskEntityWorkItem( + context.Context, + *protos.ReleaseTaskWorkItemRequest, + ...grpc.CallOption, +) (*protos.ReleaseTaskWorkItemResponse, error) { + return c.release() +} + +func (c *sessionSchedulerClient) AbandonTaskOrchestratorWorkItem( + ctx context.Context, + request *protos.AbandonOrchestrationTaskRequest, + opts ...grpc.CallOption, +) (*protos.AbandonOrchestrationTaskResponse, error) { + c.orchestrationAbort.Add(1) + if c.abandonErr != nil { + if err := c.abandonErr(); err != nil { + return nil, err + } + } + return c.fakeSchedulerClient.AbandonTaskOrchestratorWorkItem(ctx, request, opts...) +} + +func (c *sessionSchedulerClient) nextCall(t *testing.T) *fakeSessionCall { + t.Helper() + select { + case call := <-c.calls: + return call + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a session completion") + return nil + } +} + +func (c *sessionSchedulerClient) noCall(t *testing.T) { + t.Helper() + select { + case call := <-c.calls: + t.Fatalf("unexpected session completion with count %d", call.count()) + default: + } +} + +func (c *sessionSchedulerClient) unaryOrchestrations() []*protos.OrchestratorResponse { + c.mu.Lock() + defer c.mu.Unlock() + return append([]*protos.OrchestratorResponse(nil), c.orchestrationCompletions...) +} + +func (c *sessionSchedulerClient) unaryEntities() []*protos.EntityBatchResult { + c.mu.Lock() + defer c.mu.Unlock() + return append([]*protos.EntityBatchResult(nil), c.entityCompletions...) +} + +func (c *sessionSchedulerClient) abandons() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.orchestrationAbandons +} + +// fakeSessionTimers lets a test fire session timers by hand. +type fakeSessionTimers struct { + mu sync.Mutex + created []sessionTimerKind + timers chan fakeSessionTimer +} + +type fakeSessionTimer struct { + kind sessionTimerKind + fire chan time.Time +} + +func newFakeSessionTimers() *fakeSessionTimers { + return &fakeSessionTimers{timers: make(chan fakeSessionTimer, 64)} +} + +func (f *fakeSessionTimers) new(kind sessionTimerKind, _ time.Duration) <-chan time.Time { + timer := fakeSessionTimer{kind: kind, fire: make(chan time.Time, 1)} + f.mu.Lock() + f.created = append(f.created, kind) + f.mu.Unlock() + f.timers <- timer + return timer.fire +} + +// next returns the next created timer of the given kind. +func (f *fakeSessionTimers) next(t *testing.T, kind sessionTimerKind) fakeSessionTimer { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case timer := <-f.timers: + if timer.kind == kind { + return timer + } + case <-deadline: + t.Fatalf("timed out waiting for session timer %d", kind) + } + } +} + +func (f *fakeSessionTimers) count(kind sessionTimerKind) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, created := range f.created { + if created == kind { + n++ + } + } + return n +} + +type sessionWorkerHarness struct { + worker *TaskHubGrpcWorker + client *sessionSchedulerClient + timers *fakeSessionTimers + starts *atomic.Int32 + cancel context.CancelFunc +} + +func sessionOrchestrator(starts *atomic.Int32, activities int) task.Orchestrator { + return func(ctx *task.OrchestrationContext) (any, error) { + starts.Add(1) + results := make([]string, 0, activities) + for range activities { + var result string + if err := ctx.CallActivity("work").Await(&result); err != nil { + return nil, err + } + results = append(results, result) + } + return strings.Join(results, ","), nil + } +} + +func startSessionWorker( + t *testing.T, + registry *task.TaskRegistry, + client *sessionSchedulerClient, + opts ...TaskHubGrpcWorkerOption, +) *sessionWorkerHarness { + t.Helper() + options := []TaskHubGrpcWorkerOption{ + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(10 * time.Second), + WithWorkerRPCTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + WithWorkerTransientRetryPolicy(3, time.Millisecond, 5*time.Millisecond), + WithMaxConcurrentOrchestrationWorkItems(4), + WithMaxConcurrentEntityWorkItems(4), + WithExtendedSessions(ExtendedSessionOptions{}), + } + worker, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return client, nil, nil + }, + registry, + api.DefaultLogger(), + append(options, opts...)..., + ) + require.NoError(t, err) + timers := newFakeSessionTimers() + worker.options.sessionTimerFn = timers.new + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + h := &sessionWorkerHarness{worker: worker, client: client, timers: timers, cancel: cancel} + t.Cleanup(func() { + cancel() + // A turn that a test never acknowledges waits for its acknowledgement + // timer, so end the worker quickly. + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancelShutdown() + _ = worker.Shutdown(shutdownCtx) + }) + return h +} + +func newSessionHarness(t *testing.T, activities int, opts ...TaskHubGrpcWorkerOption) *sessionWorkerHarness { + t.Helper() + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", sessionOrchestrator(starts, activities))) + h := startSessionWorker(t, registry, newSessionSchedulerClient(), opts...) + h.starts = starts + return h +} + +func (h *sessionWorkerHarness) dispatch(instanceID string, newEvents ...*protos.HistoryEvent) { + if len(newEvents) == 0 { + newEvents = []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("session", instanceID, nil, nil, nil, nil), + } + } + h.client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: instanceID, + NewEvents: newEvents, + }}, + CompletionToken: "token-" + instanceID, + }} +} + +func taskCompleted(id int32, result string) []*protos.HistoryEvent { + return []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskCompletedEvent(id, wrapperspb.String(`"`+result+`"`)), + } +} + +func (h *sessionWorkerHarness) waitUnary(t *testing.T, n int) []*protos.OrchestratorResponse { + t.Helper() + require.Eventually(t, func() bool { return len(h.client.unaryOrchestrations()) >= n }, 5*time.Second, time.Millisecond) + return h.client.unaryOrchestrations() +} + +func completedOutput(t *testing.T, response *protos.OrchestratorResponse) string { + t.Helper() + require.Len(t, response.Actions, 1) + completed := response.Actions[0].GetCompleteOrchestration() + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, completed.OrchestrationStatus, + "failure: %v", completed.GetFailureDetails()) + return completed.GetResult().GetValue() +} + +func TestWorkerSessionRunsTurnsWithoutReplay(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("multi") + + first := h.client.nextCall(t) + require.EqualValues(t, 0, first.count()) + require.Equal(t, "token-multi", first.orchestration.CompletionToken) + require.Equal(t, "work", first.orchestration.Actions[0].GetScheduleTask().GetName()) + first.stream.ack(4) + first.stream.ping() + first.stream.batch(taskCompleted(0, "a")...) + + second := h.client.nextCall(t) + require.EqualValues(t, 1, second.count()) + require.Len(t, second.orchestration.Actions, 1) + require.EqualValues(t, 1, second.orchestration.Actions[0].Id) + second.stream.ack(8) + second.stream.batch(taskCompleted(1, "b")...) + + final := h.waitUnary(t, 1)[0] + require.EqualValues(t, 2, final.CompletionCount, "the terminal unary completion must carry the session count") + require.Equal(t, `"a,b"`, completedOutput(t, final)) + require.EqualValues(t, 1, h.starts.Load(), "the orchestrator must not replay between session turns") + require.Zero(t, h.client.releases.Load()) + require.Zero(t, h.client.abandons()) +} + +func TestWorkerSessionRetriesTransientErrorWithSameCount(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("retry") + + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + + failed := h.client.nextCall(t) + require.EqualValues(t, 1, failed.count()) + failed.stream.end(status.Error(codes.Unavailable, "lost")) + retried := h.client.nextCall(t) + require.EqualValues(t, 1, retried.count(), "a retry before the acknowledgement must reuse the turn count") + require.True(t, proto.Equal(failed.orchestration, retried.orchestration)) + retried.stream.ack(8) + retried.stream.batch(taskCompleted(1, "b")...) + + require.EqualValues(t, 2, h.waitUnary(t, 1)[0].CompletionCount) +} + +func TestWorkerSessionAbandonsOnPreAckFailure(t *testing.T) { + for _, err := range []error{ + status.Error(codes.NotFound, "count mismatch"), + status.Error(codes.InvalidArgument, "bad request"), + } { + t.Run(status.Code(err).String(), func(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("abandon") + first := h.client.nextCall(t) + first.stream.end(err) + require.Eventually(t, func() bool { return h.client.abandons() == 1 }, 5*time.Second, time.Millisecond) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + require.Zero(t, h.client.releases.Load()) + }) + } +} + +func TestWorkerSessionAckTimeoutRetries(t *testing.T) { + h := newSessionHarness(t, 1) + h.dispatch("ack-timeout") + first := h.client.nextCall(t) + h.timers.next(t, sessionAckTimer).fire <- time.Now() + retried := h.client.nextCall(t) + require.EqualValues(t, 0, retried.count()) + require.True(t, proto.Equal(first.orchestration, retried.orchestration)) +} + +func TestWorkerSessionReleasesAfterAckFailures(t *testing.T) { + tests := []struct { + name string + fail func(t *testing.T, h *sessionWorkerHarness, stream *fakeSessionStream) + }{ + {name: "stream breaks while idle", fail: func(_ *testing.T, _ *sessionWorkerHarness, stream *fakeSessionStream) { + stream.end(status.Error(codes.Unavailable, "broken")) + }}, + {name: "batch cut off after first chunk", fail: func(_ *testing.T, _ *sessionWorkerHarness, stream *fakeSessionStream) { + stream.chunk(taskCompleted(0, "partial")...) + stream.end(status.Error(codes.Unavailable, "cut")) + }}, + {name: "idle timeout", fail: func(t *testing.T, h *sessionWorkerHarness, stream *fakeSessionStream) { + stream.ping() + stream.ping() + h.timers.next(t, sessionIdleTimer).fire <- time.Now() + }}, + {name: "transfer timeout", fail: func(t *testing.T, h *sessionWorkerHarness, stream *fakeSessionStream) { + stream.chunk(taskCompleted(0, "partial")...) + h.timers.next(t, sessionTransferTimer).fire <- time.Now() + }}, + {name: "second acknowledgement", fail: func(_ *testing.T, _ *sessionWorkerHarness, stream *fakeSessionStream) { + stream.ack(1) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("release") + first := h.client.nextCall(t) + first.stream.ack(4) + tt.fail(t, h, first.stream) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + // The partial batch is discarded: no further turn runs. + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + require.Zero(t, h.client.abandons()) + require.EqualValues(t, 1, h.starts.Load()) + }) + } +} + +func TestWorkerSessionPingsDoNotResetIdleTimer(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("pings") + first := h.client.nextCall(t) + first.stream.ack(4) + for range 5 { + first.stream.ping() + } + first.stream.chunk(taskCompleted(0, "a")...) + // The transfer timer starts only after every earlier message was handled. + h.timers.next(t, sessionTransferTimer) + require.Equal(t, 1, h.timers.count(sessionIdleTimer), "pings must not restart the idle timer") + first.stream.end(nil) + require.EqualValues(t, 1, h.client.nextCall(t).count()) +} + +func TestWorkerSessionTimeoutAndFirstChunkHaveOneWinner(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("race") + first := h.client.nextCall(t) + first.stream.ack(4) + idle := h.timers.next(t, sessionIdleTimer) + first.stream.chunk(taskCompleted(0, "a")...) + transfer := h.timers.next(t, sessionTransferTimer) + // The old idle timer fires after the first chunk switched to transfer. + idle.fire <- time.Now() + first.stream.end(nil) + second := h.client.nextCall(t) + require.EqualValues(t, 1, second.count()) + select { + case transfer.fire <- time.Now(): + default: + } + require.Zero(t, h.client.releases.Load()) +} + +func TestWorkerSessionEndsWhenServiceEndsStream(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("service-end") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.end(nil) + require.Never(t, func() bool { + return h.client.releases.Load() != 0 || h.client.abandons() != 0 || len(h.client.unaryOrchestrations()) != 0 + }, 50*time.Millisecond, 5*time.Millisecond) + h.client.noCall(t) +} + +func TestWorkerSessionUnimplementedFallsBackToUnary(t *testing.T) { + h := newSessionHarness(t, 1) + h.dispatch("first") + first := h.client.nextCall(t) + first.stream.end(status.Error(codes.Unimplemented, "no sessions")) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 0, unary.CompletionCount) + require.True(t, proto.Equal(first.orchestration, unary), "the fallback must send the same result") + + // Sessions stay off on this intake generation. + h.dispatch("second") + require.Len(t, h.waitUnary(t, 2), 2) + h.client.noCall(t) +} + +func TestWorkerSessionUnimplementedAfterFirstTurnUsesTurnCount(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("later") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + second := h.client.nextCall(t) + second.stream.end(status.Error(codes.Unimplemented, "switched off")) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.CompletionCount) + require.Equal(t, "work", unary.Actions[0].GetScheduleTask().GetName()) +} + +func TestWorkerSessionStreamBudgetFullUsesUnary(t *testing.T) { + h := newSessionHarness(t, 1) + h.client.openErr = func() error { return errSessionStreamBudgetFull } + h.dispatch("budget") + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 0, unary.CompletionCount) + require.Len(t, unary.Actions, 1) +} + +func TestWorkerSessionLimitUsesUnary(t *testing.T) { + h := newSessionHarness(t, 1, WithExtendedSessions(ExtendedSessionOptions{MaxOrchestrationSessions: 1})) + h.dispatch("one") + first := h.client.nextCall(t) + first.stream.ack(3) + h.dispatch("two") + unary := h.waitUnary(t, 1)[0] + require.Equal(t, "two", unary.InstanceId) +} + +func TestWorkerSessionPartialTurnUsesUnary(t *testing.T) { + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", sessionOrchestrator(starts, 1))) + h := startSessionWorker(t, registry, newSessionSchedulerClient()) + h.worker.executor = task.NewTaskExecutor(registry, + task.WithOrchestrationOptions(task.OrchestrationOptions{MaxEventsPerTurn: 1})) + h.dispatch("partial", + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("session", "partial", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("x", nil), + ) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.GetNumEventsProcessed().GetValue()) + h.client.noCall(t) +} + +func TestWorkerSessionRebuiltResponsesCarryCount(t *testing.T) { + t.Run("oversized response", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + return nil, ctx.CallActivity("big", task.WithRawActivityInput(strings.Repeat("x", 100*1024))).Await(nil) + })) + h := startSessionWorker(t, registry, newSessionSchedulerClient(), WithMaxOrchestratorCompletionBytes(64*1024)) + h.dispatch("oversized") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.CompletionCount) + require.Equal(t, "token-oversized", unary.CompletionToken) + require.Equal(t, string(api.ErrorTypeOrchestratorResponseTooLarge), + unary.Actions[0].GetCompleteOrchestration().GetFailureDetails().GetErrorType()) + }) + t.Run("orchestrator panic", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + panic("boom") + })) + h := startSessionWorker(t, registry, newSessionSchedulerClient()) + h.dispatch("panic") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.CompletionCount) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + unary.Actions[0].GetCompleteOrchestration().GetOrchestrationStatus()) + }) +} + +func TestWorkerSessionInvalidBatchReleases(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("invalid") + first := h.client.nextCall(t) + first.stream.ack(4) + // A batch must start with OrchestratorStarted. + first.stream.batch(helpers.NewTaskCompletedEvent(0, nil)) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + require.Empty(t, h.client.unaryOrchestrations()) +} + +func TestWorkerSessionCleanupFailureReconnectsIntake(t *testing.T) { + h := newSessionHarness(t, 2) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + h.client.releaseErr = func() error { return status.Error(codes.Unavailable, "down") } + h.dispatch("cleanup") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.end(status.Error(codes.Unavailable, "broken")) + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond, + "a failed cleanup must reconnect the work item stream") + require.Greater(t, h.client.releases.Load(), int32(1), "release must be retried before giving up") + require.True(t, h.worker.Running()) +} + +func TestWorkerSessionAbandonFailureReconnectsIntake(t *testing.T) { + h := newSessionHarness(t, 1) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + h.client.abandonErr = func() error { return status.Error(codes.Unavailable, "down") } + h.dispatch("abandon-fails") + first := h.client.nextCall(t) + first.stream.end(status.Error(codes.InvalidArgument, "rejected")) + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond) + require.Greater(t, h.client.orchestrationAbort.Load(), int32(1)) +} + +func TestWorkerSessionUnaryFailureAfterAckReconnectsIntake(t *testing.T) { + h := newSessionHarness(t, 1) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + h.client.unaryErr = func() error { return status.Error(codes.InvalidArgument, "rejected") } + h.client.abandonErr = func() error { return status.Error(codes.Unavailable, "down") } + h.dispatch("unary-fails") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond, + "a failed abandon of a session work item must reconnect intake") + require.Greater(t, h.client.orchestrationAbort.Load(), int32(1)) +} + +func TestWorkerEntitySessionUnaryFailureAfterAckReconnectsIntake(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(*task.EntityContext) (any, error) { return nil, nil })) + client := newSessionSchedulerClient() + client.entityCompletionErr = status.Error(codes.InvalidArgument, "rejected") + client.entityAbandonFailures = math.MaxInt32 + h := startSessionWorker(t, registry, client) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + // A batch that fails ends the session with the unary RPC. + h.worker.executor = &recordingExecutor{executeEntity: func(_ context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + if request.EntityState != nil { + return nil, errors.New("framework failure") + } + return &protos.EntityBatchResult{ + Results: []*protos.OperationResult{{ResultType: &protos.OperationResult_Success{Success: &protos.OperationResultSuccess{}}}}, + EntityState: wrapperspb.String("1"), + }, nil + }} + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@counter@c", entitySignal("add"))}, + CompletionToken: "entity-token", + }} + first := client.nextCall(t) + first.stream.ack(0) + first.stream.batch(entitySignal("add")) + require.Eventually(t, func() bool { return client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond, + "a failed abandon of a session work item must reconnect intake") + client.mu.Lock() + defer client.mu.Unlock() + require.Greater(t, client.entityAbandonAttempts, 1) +} + +func TestWorkerSessionClosesWithRetiredGeneration(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("retired") + first := h.client.nextCall(t) + first.stream.ack(4) + h.timers.next(t, sessionIdleTimer) + // The intake stream fails, so the service abandons the generation's work. + h.client.stream.results <- fakeWorkItemResult{err: status.Error(codes.Unavailable, "intake lost")} + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { + select { + case <-first.stream.ctx.Done(): + return true + default: + return false + } + }, 5*time.Second, time.Millisecond, "the session stream must close with its generation") + require.Zero(t, h.client.releases.Load(), "a retired generation's sessions close without release") + require.Zero(t, h.client.abandons()) +} + +func TestWorkerSessionShutdownReleasesIdleSessions(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("shutdown") + first := h.client.nextCall(t) + first.stream.ack(4) + h.timers.next(t, sessionIdleTimer) + h.cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, h.worker.Shutdown(ctx)) + require.EqualValues(t, 1, h.client.releases.Load()) +} + +func entitySessionRequest(instanceID string, operations ...*protos.HistoryEvent) *protos.EntityRequest { + return &protos.EntityRequest{InstanceId: instanceID, OperationRequests: operations} +} + +func entitySignal(operation string) *protos.HistoryEvent { + return newFakeEntitySignalRequest(operation, nil).OperationRequests[0] +} + +func TestWorkerEntitySessionUsesCachedState(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + return nil, ctx.SetState(value + 1) + })) + client := newSessionSchedulerClient() + h := startSessionWorker(t, registry, client) + var requests []*protos.EntityBatchRequest + var mu sync.Mutex + base := h.worker.executor.(task.EntityExecutor) + h.worker.executor = &recordingExecutor{executeEntity: func(ctx context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + mu.Lock() + requests = append(requests, proto.Clone(request).(*protos.EntityBatchRequest)) + mu.Unlock() + return base.ExecuteEntity(ctx, request) + }} + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@counter@a", entitySignal("add"))}, + CompletionToken: "entity-token", + }} + + first := client.nextCall(t) + require.EqualValues(t, 0, first.count()) + require.Equal(t, "1", first.entity.EntityState.GetValue()) + first.stream.ack(0) + first.stream.batch(helpers.NewOrchestratorStartedEvent(), entitySignal("add"), entitySignal("add")) + + second := client.nextCall(t) + require.EqualValues(t, 1, second.count()) + require.Equal(t, "3", second.entity.EntityState.GetValue()) + require.Len(t, second.entity.Results, 2) + second.stream.ack(0) + // A batch whose executor fails ends the session with the unary RPC. + h.worker.executor = &recordingExecutor{executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return nil, errors.New("framework failure") + }} + second.stream.batch(entitySignal("add")) + + require.Eventually(t, func() bool { return len(client.unaryEntities()) == 1 }, 5*time.Second, time.Millisecond) + failed := client.unaryEntities()[0] + require.EqualValues(t, 2, failed.CompletionCount) + require.Equal(t, "entity-token", failed.CompletionToken) + require.NotNil(t, failed.FailureDetails) + + mu.Lock() + defer mu.Unlock() + require.Len(t, requests, 2) + require.Equal(t, "1", requests[1].EntityState.GetValue(), "a resumed batch must use the cached state") + require.Len(t, requests[1].Operations, 2, "control events are not entity operations") +} + +func TestWorkerEntitySessionAbandonsInvalidBatch(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(*task.EntityContext) (any, error) { return nil, nil })) + client := newSessionSchedulerClient() + startSessionWorker(t, registry, client) + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@counter@b", entitySignal("noop"))}, + CompletionToken: "entity-token", + }} + first := client.nextCall(t) + first.stream.ack(0) + first.stream.batch(helpers.NewTaskCompletedEvent(1, nil)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.entityAbandonAttempts == 1 + }, 5*time.Second, time.Millisecond) + require.Empty(t, client.unaryEntities()) + require.Zero(t, client.releases.Load()) +} + +func TestWithExtendedSessionsValidation(t *testing.T) { + newWorker := func(opts ...TaskHubGrpcWorkerOption) error { + _, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { return nil, nil, nil }, + task.NewTaskRegistry(), + api.DefaultLogger(), + opts..., + ) + return err + } + require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{IdleTimeout: -1}))) + require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{MaxEntitySessions: -1}))) + require.ErrorContains(t, newWorker( + WithMaxConcurrentOrchestrationWorkItems(2), + WithExtendedSessions(ExtendedSessionOptions{MaxOrchestrationSessions: 2}), + ), "must be lower") + + shared := WithExtendedSessions(ExtendedSessionOptions{}) + small, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { return nil, nil, nil }, + task.NewTaskRegistry(), api.DefaultLogger(), WithMaxConcurrentOrchestrationWorkItems(4), shared) + require.NoError(t, err) + large, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { return nil, nil, nil }, + task.NewTaskRegistry(), api.DefaultLogger(), WithMaxConcurrentOrchestrationWorkItems(40), shared) + require.NoError(t, err) + require.Equal(t, 2, small.options.extendedSessions.MaxOrchestrationSessions, + "workers built from one option value must not share their session options") + require.Equal(t, 20, large.options.extendedSessions.MaxOrchestrationSessions) + + options := defaultTaskHubGrpcWorkerOptions() + require.NoError(t, WithExtendedSessions(ExtendedSessionOptions{})(&options)) + options.maxConcurrentOrchestrations = 1 + options.maxConcurrentEntities = 9 + require.NoError(t, options.resolveExtendedSessionLimits()) + require.Zero(t, options.extendedSessions.MaxOrchestrationSessions, "one slot leaves no room for sessions") + require.Equal(t, 4, options.extendedSessions.MaxEntitySessions) + require.Equal(t, DefaultExtendedSessionTimeout, options.extendedSessions.IdleTimeout) + require.Equal(t, DefaultExtendedSessionTimeout, options.extendedSessions.TransferTimeout) +} + +func TestWorkerSessionsOffByDefault(t *testing.T) { + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", sessionOrchestrator(starts, 1))) + client := newSessionSchedulerClient() + worker := newFakeWorker(t, client.fakeSchedulerClient) + worker.clientFactory = func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return client, nil, nil + } + worker.executor = task.NewTaskExecutor(registry) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, worker.Start(ctx)) + h := &sessionWorkerHarness{worker: worker, client: client} + h.dispatch("off") + h.waitUnary(t, 1) + client.noCall(t) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +// sessionStreamConn records session streams opened on a completion connection. +type sessionStreamConn struct { + fakeClientConn + opened atomic.Int32 +} + +func (c *sessionStreamConn) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) { + c.opened.Add(1) + return c.fakeClientConn.NewStream(ctx, desc, method, opts...) +} + +func TestWorkerCompletionTransportRoutesSessionStreams(t *testing.T) { + intake := &sessionStreamConn{} + completions := []*sessionStreamConn{{}, {}} + transport := &workerCompletionTransport{ + intake: intake, + completions: []grpc.ClientConnInterface{completions[0], completions[1]}, + } + desc := &protos.TaskHubSidecarService_ServiceDesc.Streams[1] + method := protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName + + var cancels []context.CancelFunc + for range 2 * workerSessionStreamsPerConnection { + ctx, cancel := context.WithCancel(context.Background()) + cancels = append(cancels, cancel) + _, err := transport.NewStream(ctx, desc, method) + require.NoError(t, err) + } + require.EqualValues(t, workerSessionStreamsPerConnection, completions[0].opened.Load()) + require.EqualValues(t, workerSessionStreamsPerConnection, completions[1].opened.Load()) + require.Zero(t, intake.opened.Load()) + + _, err := transport.NewStream(context.Background(), desc, protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName) + require.ErrorIs(t, err, errSessionStreamBudgetFull) + + // An ended stream returns its budget. + cancels[0]() + require.Eventually(t, func() bool { + ctx, cancel := context.WithCancel(context.Background()) + cancels = append(cancels, cancel) + _, err := transport.NewStream(ctx, desc, method) + return err == nil + }, time.Second, time.Millisecond) + + // Other streams stay on intake. + _, err = transport.NewStream(context.Background(), desc, protos.TaskHubSidecarService_GetWorkItems_FullMethodName) + require.NoError(t, err) + require.EqualValues(t, 1, intake.opened.Load()) + for _, cancel := range cancels { + cancel() + } + + require.True(t, isWorkerCompletionMethod(protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName)) + require.True(t, isWorkerCompletionMethod(protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName)) +} + +func TestBudgetedStreamReturnsBudgetOnRecvError(t *testing.T) { + transport := &workerCompletionTransport{ + intake: &fakeClientConn{}, + completions: []grpc.ClientConnInterface{&erroringStreamConn{}}, + } + stream, err := transport.NewStream(context.Background(), &grpc.StreamDesc{ServerStreams: true}, + protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName) + require.NoError(t, err) + require.Equal(t, []int{1}, transport.sessionStreams) + require.ErrorIs(t, stream.RecvMsg(nil), io.EOF) + require.Equal(t, []int{0}, transport.sessionStreams) +} + +type erroringStreamConn struct{ fakeClientConn } + +func (*erroringStreamConn) NewStream(context.Context, *grpc.StreamDesc, string, ...grpc.CallOption) (grpc.ClientStream, error) { + return erroringClientStream{}, nil +} + +type erroringClientStream struct{ grpc.ClientStream } + +func (erroringClientStream) RecvMsg(any) error { return io.EOF } + +func TestWorkerSessionBudgetFullAfterUncertainAttemptAbandons(t *testing.T) { + h := newSessionHarness(t, 2) + var opens atomic.Int32 + h.client.openErr = func() error { + if opens.Add(1) == 2 { + return errSessionStreamBudgetFull + } + return nil + } + h.dispatch("uncertain") + // The service may have applied this attempt before the stream broke. + h.client.nextCall(t).stream.end(status.Error(codes.Unavailable, "lost")) + require.Eventually(t, func() bool { return h.client.abandons() == 1 }, 5*time.Second, time.Millisecond) + require.EqualValues(t, 2, opens.Load()) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations(), "a unary completion after an uncertain attempt can leave the session open") +} + +func TestWorkerSessionOpenTimeoutRetries(t *testing.T) { + h := newSessionHarness(t, 1, WithWorkerRPCTimeout(200*time.Millisecond)) + var opens atomic.Int32 + h.client.openWait = func(ctx context.Context) error { + if opens.Add(1) == 1 { + <-ctx.Done() + return status.FromContextError(ctx.Err()).Err() + } + return nil + } + h.dispatch("slow-open") + retried := h.client.nextCall(t) + require.EqualValues(t, 2, opens.Load(), "a stuck open must time out and retry") + require.EqualValues(t, 0, retried.count()) + require.Zero(t, h.client.abandons()) +} + +func TestWorkerSessionOpenStopsWhenIntakeEnds(t *testing.T) { + h := newSessionHarness(t, 1, WithWorkerRPCTimeout(time.Minute)) + opening := make(chan struct{}) + unblocked := make(chan struct{}) + var opens atomic.Int32 + h.client.openWait = func(ctx context.Context) error { + if opens.Add(1) > 1 { + return nil + } + close(opening) + <-ctx.Done() + close(unblocked) + return status.FromContextError(ctx.Err()).Err() + } + h.dispatch("stuck") + <-opening + h.client.stream.results <- fakeWorkItemResult{err: status.Error(codes.Unavailable, "intake ended")} + select { + case <-unblocked: + case <-time.After(5 * time.Second): + t.Fatal("the stuck open did not stop when its intake generation ended") + } + require.Never(t, func() bool { + return h.client.abandons() != 0 || len(h.client.unaryOrchestrations()) != 0 + }, 50*time.Millisecond, 5*time.Millisecond) +} + +func TestWorkerSessionBatchLimitsRelease(t *testing.T) { + t.Run("events", func(t *testing.T) { + h := newSessionHarness(t, 2, WithMaxStreamedHistoryEvents(2)) + h.dispatch("events") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.chunk(helpers.NewOrchestratorStartedEvent(), helpers.NewEventRaisedEvent("x", nil)) + first.stream.chunk(helpers.NewTaskCompletedEvent(0, nil)) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + }) + t.Run("bytes", func(t *testing.T) { + h := newSessionHarness(t, 2, WithMaxStreamedHistoryBytes(256)) + h.dispatch("bytes") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.chunk(helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("x", wrapperspb.String(strings.Repeat("x", 512)))) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + }) +} + +func TestWorkerSessionReservesCompletionCountBytes(t *testing.T) { + for _, message := range []proto.Message{&protos.OrchestratorResponse{InstanceId: "x"}, &protos.EntityBatchResult{}} { + before := proto.Size(message) + switch m := message.(type) { + case *protos.OrchestratorResponse: + m.CompletionCount = math.MaxInt32 + case *protos.EntityBatchResult: + m.CompletionCount = math.MaxInt32 + } + require.LessOrEqual(t, proto.Size(message)-before, sessionCompletionCountReserve) + } + off := newSessionHarness(t, 1, WithMaxOrchestratorCompletionBytes(64*1024)).worker + off.options.extendedSessions = nil + require.Equal(t, 64*1024, off.orchestratorCompletionLimit()) + on := newSessionHarness(t, 1, WithMaxOrchestratorCompletionBytes(64*1024)).worker + require.Equal(t, 64*1024-sessionCompletionCountReserve, on.orchestratorCompletionLimit()) +} + +type countingPayloadStore struct { + *payload.MemoryStore + resolves atomic.Int32 +} + +func (s *countingPayloadStore) Resolve(ctx context.Context, location string) ([]byte, error) { + s.resolves.Add(1) + return s.MemoryStore.Resolve(ctx, location) +} + +func TestWorkerEntitySessionCachesStateBeforeExternalization(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("text", func(ctx *task.EntityContext) (any, error) { + var value string + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + return nil, ctx.SetState(value + "x") + })) + store := &countingPayloadStore{MemoryStore: payload.NewMemoryStore()} + client := newSessionSchedulerClient() + h := startSessionWorker(t, registry, client, WithWorkerLargePayloads(&api.LargePayloadOptions{ + Store: store, Resolver: store, ThresholdBytes: 1, MaxPayloadBytes: 1024, + })) + var requests []*protos.EntityBatchRequest + var mu sync.Mutex + base := h.worker.executor.(task.EntityExecutor) + h.worker.executor = &recordingExecutor{executeEntity: func(ctx context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + mu.Lock() + requests = append(requests, proto.Clone(request).(*protos.EntityBatchRequest)) + mu.Unlock() + return base.ExecuteEntity(ctx, request) + }} + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@text@a", entitySignal("add"))}, + CompletionToken: "entity-token", + }} + + first := client.nextCall(t) + require.True(t, api.IsLargePayloadReference(first.entity.EntityState.GetValue()), "the service must get the externalized state") + first.stream.ack(0) + first.stream.batch(helpers.NewOrchestratorStartedEvent(), entitySignal("add")) + + second := client.nextCall(t) + require.True(t, api.IsLargePayloadReference(second.entity.EntityState.GetValue())) + require.Zero(t, store.resolves.Load(), "a resumed batch must not download the cached state") + mu.Lock() + defer mu.Unlock() + require.Len(t, requests, 2) + require.Equal(t, `"x"`, requests[1].EntityState.GetValue()) +} + +func TestWorkerSessionUnimplementedAfterUncertainAttemptAbandons(t *testing.T) { + h := newSessionHarness(t, 2) + var opens atomic.Int32 + h.client.openErr = func() error { + if opens.Add(1) == 2 { + return status.Error(codes.Unimplemented, "switched off") + } + return nil + } + h.dispatch("uncertain") + h.client.nextCall(t).stream.end(status.Error(codes.Unavailable, "lost")) + require.Eventually(t, func() bool { return h.client.abandons() == 1 }, 5*time.Second, time.Millisecond) + require.Empty(t, h.client.unaryOrchestrations(), "a unary completion after an uncertain attempt can leave the session open") + + // Sessions stay off on this intake generation. + h.dispatch("next") + require.Len(t, h.waitUnary(t, 1), 1) + h.client.noCall(t) +} + +// detachedSessionScope returns a session scope whose intake stream the test +// ends by calling the returned function. +func detachedSessionScope() (*workerSessionScope, context.CancelFunc) { + streamCtx, endStream := context.WithCancel(context.Background()) + connection := &grpcWorkerConnection{streamCtx: streamCtx, cancelStream: endStream} + run := &grpcWorkerRun{intakeCtx: context.Background()} + return &workerSessionScope{run: run, connection: connection}, endStream +} + +func TestWorkerSessionRetryWaitStopsWhenIntakeEnds(t *testing.T) { + worker := newSessionHarness(t, 1, WithWorkerTransientRetryPolicy(3, time.Hour, time.Hour)).worker + scope, endStream := detachedSessionScope() + opened := make(chan struct{}, 3) + outcome := make(chan sessionOutcome, 1) + go func() { + result, _ := worker.awaitSessionBatch(context.Background(), scope, sessionCall{ + instanceID: "retry-wait", + open: func(context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + opened <- struct{}{} + return nil, status.Error(codes.Unavailable, "down") + }, + abandon: func(context.Context) error { return nil }, + }) + outcome <- result + }() + <-opened + endStream() + select { + case result := <-outcome: + require.Equal(t, sessionEnded, result) + case <-time.After(5 * time.Second): + t.Fatal("the retry wait did not stop when the intake stream ended") + } + require.Len(t, opened, 0, "no attempt may run after the intake stream ended") +} + +func TestWorkerSessionCleanupStopsWhenIntakeEnds(t *testing.T) { + worker := newSessionHarness(t, 1, WithWorkerTransientRetryPolicy(3, time.Hour, time.Hour)).worker + worker.options.sessionCleanupTimeout = time.Hour + scope, endStream := detachedSessionScope() + attempts := make(chan struct{}, 3) + done := make(chan struct{}) + go func() { + defer close(done) + worker.cleanupSessionWorkItem(context.Background(), scope, "cleanup", "release", func(context.Context) error { + attempts <- struct{}{} + return status.Error(codes.Unavailable, "down") + }) + }() + <-attempts + endStream() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not stop when the intake stream ended") + } + require.Len(t, attempts, 0) + require.False(t, scope.connection.sessionCleanupFailed.Load(), "an ended generation needs no extra reconnect") +} diff --git a/task/executor.go b/task/executor.go index 9548029..899563b 100644 --- a/task/executor.go +++ b/task/executor.go @@ -249,12 +249,68 @@ func (te *taskExecutor) ExecuteOrchestrator( newEvents []*protos.HistoryEvent, entityParameters *protos.OrchestratorEntityParameters, ) (*ExecutionResults, error) { + results, orchestrationCtx, err := te.prepareOrchestrator(id, oldEvents, newEvents, entityParameters) + if results != nil || err != nil { + return results, err + } + actions := orchestrationCtx.start() + return te.orchestratorResults(orchestrationCtx, actions), nil +} + +// StartOrchestratorSession implements SessionExecutor. It runs the first turn +// like ExecuteOrchestrator, but keeps the orchestration loaded when the turn +// can be continued with new events. +func (te *taskExecutor) StartOrchestratorSession( + _ context.Context, + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, +) (*ExecutionResults, OrchestratorSession, error) { + results, orchestrationCtx, err := te.prepareOrchestrator(id, oldEvents, newEvents, entityParameters) + if results != nil || err != nil { + return results, nil, err + } + session := &orchestratorSession{executor: te, ctx: orchestrationCtx} + orchestrationCtx.begin() + return session.finishTurn(orchestrationCtx.run()) +} + +// orchestratorResults builds the response for one completed turn. +func (te *taskExecutor) orchestratorResults( + orchestrationCtx *OrchestrationContext, + actions []*protos.OrchestratorAction, +) *ExecutionResults { + te.reportHistoryMetric(orchestrationCtx) + + results := &ExecutionResults{ + Response: &protos.OrchestratorResponse{ + InstanceId: string(orchestrationCtx.ID), + Actions: actions, + CustomStatus: wrapperspb.String(orchestrationCtx.customStatus), + }, + } + if orchestrationCtx.maxEventsPerTurnExceeded && !orchestrationCtx.hasCompletionAction() { + results.Response.NumEventsProcessed = wrapperspb.Int32(int32(orchestrationCtx.processedEventsThisTurn)) + } + return results +} + +// prepareOrchestrator checks the work item before any orchestrator code runs. +// It returns either a final result or error, or a new orchestration context. +func (te *taskExecutor) prepareOrchestrator( + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, +) (*ExecutionResults, *OrchestrationContext, error) { entitiesSupported, err := validateEntityParameters(entityParameters) if err != nil { - return nil, err + return nil, nil, err } if isRewindRequest(oldEvents, newEvents) { - return buildRewindResult(id, oldEvents, newEvents) + results, err := buildRewindResult(id, oldEvents, newEvents) + return results, nil, err } started := startedEvent(oldEvents, newEvents) name := started.GetName() @@ -262,7 +318,7 @@ func (te *taskExecutor) ExecuteOrchestrator( if versionErr := te.versioning.check(version); versionErr != nil && !allowsUnversionedTask(te.unversionedOrchestrators, name, version) { if te.versioning.FailureStrategy == VersionFailureReject { - return nil, versionErr + return nil, nil, versionErr } return &ExecutionResults{ Response: &protos.OrchestratorResponse{ @@ -277,11 +333,11 @@ func (te *taskExecutor) ExecuteOrchestrator( ), }, }, - }, nil + }, nil, nil } if te.orchestratorNotFound == OrchestratorNotFoundReject && name != "" { if !te.Registry.hasOrchestrator(name, version) { - return nil, newTaskNotRegisteredError( + return nil, nil, newTaskNotRegisteredError( orchestratorTaskNotFoundErrorType, name, version, @@ -301,20 +357,7 @@ func (te *taskExecutor) ExecuteOrchestrator( te.converter, entitiesSupported, ) - actions := orchestrationCtx.start() - te.reportHistoryMetric(orchestrationCtx) - - results := &ExecutionResults{ - Response: &protos.OrchestratorResponse{ - InstanceId: string(id), - Actions: actions, - CustomStatus: wrapperspb.String(orchestrationCtx.customStatus), - }, - } - if orchestrationCtx.maxEventsPerTurnExceeded && !orchestrationCtx.hasCompletionAction() { - results.Response.NumEventsProcessed = wrapperspb.Int32(int32(orchestrationCtx.processedEventsThisTurn)) - } - return results, nil + return nil, orchestrationCtx, nil } func validateEntityParameters(parameters *protos.OrchestratorEntityParameters) (bool, error) { @@ -545,9 +588,9 @@ func (te *taskExecutor) reportHistoryMetric(ctx *OrchestrationContext) { InstanceID: ctx.ID, OrchestrationName: ctx.Name, OrchestrationVersion: ctx.Version, - HistoryLength: ctx.HistoryLength(), + HistoryLength: ctx.historyLength(), ProcessedEvents: ctx.processedEventsThisTurn, - HistoryLimitExceeded: ctx.HistoryLimitExceeded(), + HistoryLimitExceeded: ctx.maxHistoryEventsExceeded, } defer func() { if recovered := recover(); recovered != nil { diff --git a/task/executor_contract.go b/task/executor_contract.go index 1d5c3d6..673a78a 100644 --- a/task/executor_contract.go +++ b/task/executor_contract.go @@ -36,3 +36,50 @@ type Executor interface { type EntityExecutor interface { ExecuteEntity(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) } + +// SessionExecutor is an optional extension of [Executor] for extended +// sessions. An extended session keeps an orchestration loaded in memory +// between turns, so the next turn only needs the new events instead of a +// replay of the full history. Like [Executor], it is an internal contract +// between the task runtime and the gRPC worker. +type SessionExecutor interface { + // StartOrchestratorSession runs one turn exactly like + // [Executor.ExecuteOrchestrator]. It also returns a session when the + // orchestration can continue in memory. A nil session means that the + // caller must complete the turn without a session. + StartOrchestratorSession( + ctx context.Context, + iid api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, + ) (*ExecutionResults, OrchestratorSession, error) +} + +// OrchestratorSession is an orchestration that stays loaded between turns. +// A session has one owner and is not safe for concurrent use. +// +// After each turn that returns a session, the owner must call Commit once +// the backend has accepted the turn's actions, and then either Resume or +// Close. +type OrchestratorSession interface { + // Commit marks the actions that the backend has durably applied. They + // are not sent again, in the same way that replayed history retires + // them. Pass exactly the actions that were sent for the last turn. + Commit(sent []*protos.OrchestratorAction) + // Resume runs the next turn with a batch of new events. The batch must + // start with an OrchestratorStarted event. historyBase is the number of + // history events that the backend has stored for the current execution. + // The returned session is nil when the orchestration can no longer stay + // loaded; the session is then already closed. On error the session is + // also closed and the caller must fall back to a full replay. + Resume( + ctx context.Context, + batch []*protos.HistoryEvent, + historyBase int, + ) (*ExecutionResults, OrchestratorSession, error) + // Close unloads the orchestration. Deferred orchestrator code that runs + // during the unload cannot schedule durable work. Close is safe to call + // more than once. + Close() +} diff --git a/task/orchestrator.go b/task/orchestrator.go index 63f7b9f..e1d1131 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -8,6 +8,7 @@ import ( "maps" "math" "runtime/debug" + "slices" "sort" "strings" "time" @@ -57,33 +58,40 @@ type OrchestrationContext struct { rawInput []byte oldEvents []*protos.HistoryEvent newEvents []*protos.HistoryEvent - suspendedEvents []replayEvent - resumedEvents []replayEvent - isSuspended bool - isTerminated bool - historyIndex int + // historyBase is the number of stored history events before newEvents. It + // equals len(oldEvents) for a normal turn. A resumed session turn has no + // oldEvents, so it uses the stored event count reported by the backend. + historyBase int + suspendedEvents []replayEvent + resumedEvents []replayEvent + isSuspended bool + isTerminated bool + historyIndex int // processedEventsThisTurn follows DTS numEventsProcessed semantics and // excludes orchestration control markers. processedEventsThisTurn int maxEventsPerTurnExceeded bool maxHistoryEventsExceeded bool - sequenceNumber int32 - newGuidCounter uint64 - pendingActions map[int32]*protos.OrchestratorAction - pendingTasks map[int32]*completableTask - pendingEntityTasks map[string]*completableTask - continuedAsNew bool - continuedAsNewInput any - continuedAsNewVersion *wrapperspb.StringValue - customStatus string - defaultVersion string - converter api.DataConverter - entitiesSupported bool - scheduler *coroutineScheduler - unwinding bool - root *OrchestrationContext - scope *cancellationScope - derived []*OrchestrationContext + // turnHistoryRead records that orchestrator code read HistoryLength or + // HistoryLimitExceeded, which an extended session cannot keep consistent. + turnHistoryRead bool + sequenceNumber int32 + newGuidCounter uint64 + pendingActions map[int32]*protos.OrchestratorAction + pendingTasks map[int32]*completableTask + pendingEntityTasks map[string]*completableTask + continuedAsNew bool + continuedAsNewInput any + continuedAsNewVersion *wrapperspb.StringValue + customStatus string + defaultVersion string + converter api.DataConverter + entitiesSupported bool + scheduler *coroutineScheduler + unwinding bool + root *OrchestrationContext + scope *cancellationScope + derived []*OrchestrationContext bufferedExternalEvents map[string]*list.List pendingExternalEventTasks map[string]*list.List @@ -285,17 +293,22 @@ func newOrchestrationContext( registry: registry, oldEvents: oldEvents, newEvents: newEvents, + historyBase: len(oldEvents), bufferedExternalEvents: make(map[string]*list.List), pendingExternalEventTasks: make(map[string]*list.List), eventChannels: make(map[string]any), eventWaiters: make(map[string]map[*coroutine]struct{}), } - ctx.maxHistoryEventsExceeded = options.MaxHistoryEvents > 0 && - len(oldEvents)+len(newEvents) > options.MaxHistoryEvents + ctx.maxHistoryEventsExceeded = ctx.historyLimitReached() ctx.scope = newCancellationScope(nil) return ctx } +func (ctx *OrchestrationContext) historyLimitReached() bool { + limit := ctx.orchestrationOptions.MaxHistoryEvents + return limit > 0 && ctx.historyBase+len(ctx.newEvents) > limit +} + func (ctx *OrchestrationContext) engineContext() *OrchestrationContext { if ctx.root != nil { return ctx.root @@ -367,7 +380,15 @@ func (ctx *OrchestrationContext) WithCancel() (*OrchestrationContext, func()) { return child, cancel } -func (ctx *OrchestrationContext) start() (actions []*protos.OrchestratorAction) { +// start runs one complete turn and then unloads the orchestration. +func (ctx *OrchestrationContext) start() []*protos.OrchestratorAction { + ctx.begin() + defer ctx.unload() + return ctx.run() +} + +// begin resets the per-execution state and creates the coroutine scheduler. +func (ctx *OrchestrationContext) begin() { ctx.historyIndex = 0 ctx.processedEventsThisTurn = 0 ctx.sequenceNumber = 0 @@ -378,10 +399,71 @@ func (ctx *OrchestrationContext) start() (actions []*protos.OrchestratorAction) ctx.pendingEntityTasks = make(map[string]*completableTask) ctx.clearCriticalSection() ctx.scheduler = newCoroutineScheduler(ctx) - defer func() { - ctx.scheduler.shutdown() - ctx.scheduler = nil - }() +} + +// unload stops all coroutines with the forced-unload (teardown) semantics. +// It is safe to call more than once. +func (ctx *OrchestrationContext) unload() { + if ctx.scheduler == nil { + return + } + ctx.scheduler.shutdown() + ctx.scheduler = nil +} + +// commit retires the actions that the backend has durably applied. This has +// the same effect as replaying the history events that those actions create. +func (ctx *OrchestrationContext) commit(sent []*protos.OrchestratorAction) { + for _, action := range sent { + if action == nil { + continue + } + if _, ok := ctx.pendingActions[action.Id]; !ok { + continue + } + if requested := action.GetSendEntityMessage().GetEntityLockRequested(); requested != nil && + ctx.criticalSectionID != "" && requested.CriticalSectionId == ctx.criticalSectionID { + ctx.criticalSectionRequestCommitted = true + } + delete(ctx.pendingActions, action.Id) + } +} + +// releaseProcessedHistory drops the events of a finished turn while the +// orchestration waits for its next batch. The live coroutines and the event +// buffers keep everything that later turns need, so a long session does not +// keep its full history in memory. +func (ctx *OrchestrationContext) releaseProcessedHistory() { + ctx.historyBase = ctx.historyLength() + ctx.oldEvents = nil + ctx.newEvents = nil + ctx.historyIndex = 0 + ctx.scheduler.dropCompleted() +} + +// resume runs one more turn on the loaded orchestration. The batch contains +// only new events, and historyBase is the number of events that the backend +// has already stored for this execution. +func (ctx *OrchestrationContext) resume(batch []*protos.HistoryEvent, historyBase int) []*protos.OrchestratorAction { + ctx.oldEvents = nil + ctx.newEvents = batch + ctx.historyBase = historyBase + ctx.historyIndex = 0 + ctx.IsReplaying = false + ctx.processedEventsThisTurn = 0 + ctx.maxEventsPerTurnExceeded = false + ctx.maxHistoryEventsExceeded = ctx.historyLimitReached() + // A full replay would see events buffered by an earlier suspension as + // old history, so they are replayed when the resume drains them. + for i := range ctx.suspendedEvents { + ctx.suspendedEvents[i].isReplaying = true + } + return ctx.run() +} + +// run processes the remaining events and runs coroutines until no more +// progress is possible in this turn. It returns the pending actions. +func (ctx *OrchestrationContext) run() (actions []*protos.OrchestratorAction) { defer func() { if value := recover(); value != nil { ctx.continuedAsNew = false @@ -522,18 +604,27 @@ func countsTowardTurnBudget(event *protos.HistoryEvent) bool { event.GetExecutionResumed() == nil } -// HistoryLength returns the total old and new history supplied to this execution. +// HistoryLength returns the total old and new history of this execution turn. +// A full replay reads it again with the value of a later turn, so with +// extended sessions, reading it ends the session after the current turn. func (ctx *OrchestrationContext) HistoryLength() int { engine := ctx.engineContext() - return len(engine.oldEvents) + len(engine.newEvents) + engine.turnHistoryRead = true + return engine.historyLength() } -// HistoryLimitExceeded reports whether MaxHistoryEvents was exceeded. +// HistoryLimitExceeded reports whether MaxHistoryEvents was exceeded. Like +// HistoryLength, reading it ends an extended session after the current turn. func (ctx *OrchestrationContext) HistoryLimitExceeded() bool { engine := ctx.engineContext() + engine.turnHistoryRead = true return engine.maxHistoryEventsExceeded } +func (ctx *OrchestrationContext) historyLength() int { + return ctx.historyBase + len(ctx.newEvents) +} + func (ctx *OrchestrationContext) processEvent(e *protos.HistoryEvent) error { defer ctx.syncDerivedContexts() // Buffer certain events if we're in a suspended state @@ -1662,7 +1753,7 @@ func (ctx *OrchestrationContext) enforceHistoryLimit() { InstanceID: ctx.ID, OrchestrationName: ctx.Name, OrchestrationVersion: ctx.Version, - HistoryLength: ctx.HistoryLength(), + HistoryLength: ctx.historyLength(), MaxHistoryEvents: ctx.orchestrationOptions.MaxHistoryEvents, UnprocessedEventCount: unprocessedEventCount, SerializedInput: string(ctx.rawInput), @@ -1702,7 +1793,7 @@ func invokeHistoryLimitHandler(handler HistoryLimitHandler, info HistoryLimitInf func (ctx *OrchestrationContext) newHistoryLimitError(policyError error) *HistoryLimitError { return &HistoryLimitError{ InstanceID: ctx.ID, - HistoryLength: ctx.HistoryLength(), + HistoryLength: ctx.historyLength(), MaxHistoryEvents: ctx.orchestrationOptions.MaxHistoryEvents, PolicyError: policyError, } @@ -1814,12 +1905,12 @@ func (ctx *OrchestrationContext) actions() []*protos.OrchestratorAction { return nil } - actions := make([]*protos.OrchestratorAction, 0, len(ctx.pendingActions)) - for id := int32(0); id < ctx.sequenceNumber; id++ { - a, ok := ctx.pendingActions[id] - if !ok { - continue - } + // Sort the pending IDs instead of scanning every ID ever used, because a + // long session retires most of them. + ids := slices.Sorted(maps.Keys(ctx.pendingActions)) + actions := make([]*protos.OrchestratorAction, 0, len(ids)) + for _, id := range ids { + a := ctx.pendingActions[id] actions = append(actions, a) // ContinueAsNew intent survives a later failure or termination, so check // the completion's actual status before carrying events forward. diff --git a/task/scheduler.go b/task/scheduler.go index 2911384..d854d74 100644 --- a/task/scheduler.go +++ b/task/scheduler.go @@ -3,6 +3,7 @@ package task import ( "container/heap" "fmt" + "slices" ) type coroutineQueue []*coroutine @@ -177,6 +178,23 @@ func (s *coroutineScheduler) isStopping() bool { return s.stopping } +// dropCompleted forgets finished coroutines between session turns, so a long +// session does not keep their closures. Cancellation and shutdown skip +// finished coroutines, so this changes no behavior. +func (s *coroutineScheduler) dropCompleted() { + finished := func(c *coroutine) bool { return c.state == coroutineCompleted && c != s.root } + scopes := make(map[*cancellationScope]struct{}) + for _, c := range s.all { + if finished(c) && c.scope != nil { + scopes[c.scope] = struct{}{} + } + } + for scope := range scopes { + scope.coroutines = slices.DeleteFunc(scope.coroutines, finished) + } + s.all = slices.DeleteFunc(s.all, finished) +} + func (s *coroutineScheduler) shutdown() { if s.stopping { return diff --git a/task/session.go b/task/session.go new file mode 100644 index 0000000..e1eaf69 --- /dev/null +++ b/task/session.go @@ -0,0 +1,130 @@ +package task + +import ( + "context" + "errors" + "fmt" + + "github.com/microsoft/durabletask-go/internal/protos" +) + +var _ SessionExecutor = (*taskExecutor)(nil) + +type orchestratorSessionState int + +const ( + sessionAwaitingCommit orchestratorSessionState = iota + sessionAwaitingResume + sessionClosed +) + +var errOrchestratorSessionClosed = errors.New("orchestrator session is closed") + +// orchestratorSession keeps one OrchestrationContext and its coroutines +// loaded between turns. +type orchestratorSession struct { + executor *taskExecutor + ctx *OrchestrationContext + state orchestratorSessionState + // closeErr explains why the session was closed by a misuse, if it was. + closeErr error +} + +func (s *orchestratorSession) Commit(sent []*protos.OrchestratorAction) { + if s.state != sessionAwaitingCommit { + s.closeWith(fmt.Errorf("orchestrator session commit is out of order")) + return + } + s.ctx.commit(sent) + s.state = sessionAwaitingResume +} + +func (s *orchestratorSession) Resume( + _ context.Context, + batch []*protos.HistoryEvent, + historyBase int, +) (*ExecutionResults, OrchestratorSession, error) { + switch s.state { + case sessionClosed: + if s.closeErr != nil { + return nil, nil, s.closeErr + } + return nil, nil, errOrchestratorSessionClosed + case sessionAwaitingCommit: + err := fmt.Errorf("orchestrator session resumed before the last turn was committed") + s.closeWith(err) + return nil, nil, err + } + if err := validateSessionBatch(batch, historyBase); err != nil { + s.closeWith(err) + return nil, nil, err + } + return s.finishTurn(s.ctx.resume(batch, historyBase)) +} + +func (s *orchestratorSession) Close() { + s.closeWith(nil) +} + +func (s *orchestratorSession) closeWith(err error) { + if s.state == sessionClosed { + return + } + s.state = sessionClosed + s.closeErr = err + s.ctx.unload() +} + +// finishTurn builds the turn result and decides whether the orchestration can +// stay loaded. Only a turn that ends with live, waiting orchestrator code and +// no unprocessed events can continue with new events. +func (s *orchestratorSession) finishTurn( + actions []*protos.OrchestratorAction, +) (*ExecutionResults, OrchestratorSession, error) { + results := s.executor.orchestratorResults(s.ctx, actions) + if !s.ctx.canContinueSession() { + s.Close() + return results, nil, nil + } + s.ctx.releaseProcessedHistory() + s.state = sessionAwaitingCommit + return results, s, nil +} + +// canContinueSession reports whether the orchestration can stay loaded after +// the current turn. +func (ctx *OrchestrationContext) canContinueSession() bool { + scheduler := ctx.scheduler + return scheduler != nil && + scheduler.root != nil && + !scheduler.isStopping() && + !ctx.isTerminated && + !ctx.hasCompletionAction() && + // The backend sends unprocessed events again in the next work item, so + // a partial turn must not keep them in memory. + !ctx.maxEventsPerTurnExceeded && + !ctx.maxHistoryEventsExceeded && + // A full replay would read these values again with a later turn's + // value, so the session could decide differently. + !ctx.turnHistoryRead +} + +func validateSessionBatch(batch []*protos.HistoryEvent, historyBase int) error { + if historyBase < 0 { + return fmt.Errorf("orchestrator session history base must not be negative: %d", historyBase) + } + if len(batch) == 0 || batch[0].GetOrchestratorStarted() == nil { + return fmt.Errorf("orchestrator session batch must start with an OrchestratorStarted event") + } + for _, event := range batch { + switch { + case event == nil: + return fmt.Errorf("orchestrator session batch must not contain nil events") + case event.GetExecutionStarted() != nil: + return fmt.Errorf("orchestrator session batch must not start a new execution") + case event.GetExecutionRewound() != nil: + return fmt.Errorf("orchestrator session batch must not contain a rewind request") + } + } + return nil +} diff --git a/task/session_test.go b/task/session_test.go new file mode 100644 index 0000000..5a99afd --- /dev/null +++ b/task/session_test.go @@ -0,0 +1,1199 @@ +package task + +import ( + "context" + "errors" + "fmt" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" +) + +var sessionTestStart = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + +// sessionScenario describes a multi-turn orchestration run against a fake +// backend. After each turn the backend stores the batch plus the history +// events created from the sent actions, then sends a new batch. +type sessionScenario struct { + orchestrator Orchestrator + input *wrapperspb.StringValue + options []TaskExecutorOption + // perTurn limits how many outstanding responses are delivered in one + // batch. Zero delivers all of them. + perTurn int + // extra adds events to the batch of the given resumed turn (1-based). + extra map[int][]*protos.HistoryEvent + // respond overrides the default response for an action. Return nil to + // use the default. + respond func(action *protos.OrchestratorAction) *protos.HistoryEvent + // step is the time between turns. Zero means one minute. + step time.Duration +} + +type sessionRun struct { + responses []*protos.OrchestratorResponse + // historyLengths are the HistoryLength values reported by the session path. + historyLengths []int + sessionEnded bool +} + +const sessionTestInstance = api.InstanceID("session-instance") + +func sessionOrchestratorStarted(at time.Time) *protos.HistoryEvent { + return &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.New(at), + EventType: &protos.HistoryEvent_OrchestratorStarted{OrchestratorStarted: &protos.OrchestratorStartedEvent{}}, + } +} + +func sessionExecutionStarted(name string, input *wrapperspb.StringValue) *protos.HistoryEvent { + event := helpers.NewExecutionStartedEvent(name, string(sessionTestInstance), input, nil, nil, nil) + event.Timestamp = timestamppb.New(sessionTestStart) + event.GetExecutionStarted().OrchestrationInstance.ExecutionId = wrapperspb.String("execution-1") + return event +} + +func newSessionTestExecutor( + registry *TaskRegistry, + lengths *[]int, + options ...TaskExecutorOption, +) *taskExecutor { + hooks := MetricsHooks{History: func(metric HistoryMetric) { + *lengths = append(*lengths, metric.HistoryLength) + }} + all := append([]TaskExecutorOption{WithMetricsHooks(hooks)}, options...) + return NewTaskExecutor(registry, all...).(*taskExecutor) +} + +// historyFromActions builds the history events that the backend stores for +// the actions sent in one turn. +func historyFromActions(t *testing.T, actions []*protos.OrchestratorAction) []*protos.HistoryEvent { + t.Helper() + events := make([]*protos.HistoryEvent, 0, len(actions)) + for _, action := range actions { + action = proto.Clone(action).(*protos.OrchestratorAction) + event := &protos.HistoryEvent{EventId: action.Id, Timestamp: timestamppb.New(sessionTestStart)} + switch { + case action.GetScheduleTask() != nil: + scheduled := action.GetScheduleTask() + event.EventType = &protos.HistoryEvent_TaskScheduled{TaskScheduled: &protos.TaskScheduledEvent{ + Name: scheduled.Name, Version: scheduled.Version, Input: scheduled.Input, Tags: scheduled.Tags, + }} + case action.GetCreateTimer() != nil: + event.EventType = &protos.HistoryEvent_TimerCreated{TimerCreated: &protos.TimerCreatedEvent{ + FireAt: action.GetCreateTimer().FireAt, + }} + case action.GetCreateSubOrchestration() != nil: + created := action.GetCreateSubOrchestration() + event.EventType = &protos.HistoryEvent_SubOrchestrationInstanceCreated{ + SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{ + InstanceId: created.InstanceId, Name: created.Name, Version: created.Version, Input: created.Input, + }, + } + case action.GetSendEvent() != nil: + sent := action.GetSendEvent() + event.EventType = &protos.HistoryEvent_EventSent{EventSent: &protos.EventSentEvent{ + InstanceId: sent.GetInstance().GetInstanceId(), Name: sent.Name, Input: sent.Data, + }} + case action.GetSendEntityMessage() != nil: + message := action.GetSendEntityMessage() + switch { + case message.GetEntityOperationCalled() != nil: + event.EventType = &protos.HistoryEvent_EntityOperationCalled{EntityOperationCalled: message.GetEntityOperationCalled()} + case message.GetEntityOperationSignaled() != nil: + event.EventType = &protos.HistoryEvent_EntityOperationSignaled{EntityOperationSignaled: message.GetEntityOperationSignaled()} + case message.GetEntityLockRequested() != nil: + event.EventType = &protos.HistoryEvent_EntityLockRequested{EntityLockRequested: message.GetEntityLockRequested()} + case message.GetEntityUnlockSent() != nil: + event.EventType = &protos.HistoryEvent_EntityUnlockSent{EntityUnlockSent: message.GetEntityUnlockSent()} + default: + t.Fatalf("unsupported entity message: %v", message) + } + case action.GetCompleteOrchestration() != nil: + continue + default: + t.Fatalf("unsupported action: %v", action) + } + events = append(events, event) + } + return events +} + +// defaultResponse returns the inbox event that answers an action, or nil if +// the action has no response. +func defaultResponse(action *protos.OrchestratorAction) *protos.HistoryEvent { + event := &protos.HistoryEvent{EventId: -1, Timestamp: timestamppb.New(sessionTestStart)} + switch { + case action.GetScheduleTask() != nil: + result := wrapperspb.String(fmt.Sprintf("%q", fmt.Sprintf("%s-%d", action.GetScheduleTask().Name, action.Id))) + event.EventType = &protos.HistoryEvent_TaskCompleted{TaskCompleted: &protos.TaskCompletedEvent{ + TaskScheduledId: action.Id, Result: result, + }} + case action.GetCreateTimer() != nil: + event.EventType = &protos.HistoryEvent_TimerFired{TimerFired: &protos.TimerFiredEvent{ + TimerId: action.Id, FireAt: action.GetCreateTimer().FireAt, + }} + case action.GetCreateSubOrchestration() != nil: + event.EventType = &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ + SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{ + TaskScheduledId: action.Id, Result: wrapperspb.String(`"child-done"`), + }, + } + case action.GetSendEntityMessage().GetEntityOperationCalled() != nil: + called := action.GetSendEntityMessage().GetEntityOperationCalled() + event.EventType = &protos.HistoryEvent_EntityOperationCompleted{ + EntityOperationCompleted: &protos.EntityOperationCompletedEvent{ + RequestId: called.RequestId, Output: wrapperspb.String(fmt.Sprintf("%q", called.Operation+"-result")), + }, + } + case action.GetSendEntityMessage().GetEntityLockRequested() != nil: + requested := action.GetSendEntityMessage().GetEntityLockRequested() + event.EventType = &protos.HistoryEvent_EntityLockGranted{ + EntityLockGranted: &protos.EntityLockGrantedEvent{CriticalSectionId: requested.CriticalSectionId}, + } + default: + return nil + } + return event +} + +// runSessionEquivalence runs the scenario through one session and, at every +// turn, through a fresh full replay of the same stored history. Both paths +// must produce the same response and the same history length. +func runSessionEquivalence(t *testing.T, scenario sessionScenario) sessionRun { + t.Helper() + baseline := runtime.NumGoroutine() + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", scenario.orchestrator)) + var sessionLengths, replayLengths []int + sessionExecutor := newSessionTestExecutor(registry, &sessionLengths, scenario.options...) + replayExecutor := newSessionTestExecutor(registry, &replayLengths, scenario.options...) + step := scenario.step + if step == 0 { + step = time.Minute + } + respond := func(action *protos.OrchestratorAction) *protos.HistoryEvent { + if scenario.respond != nil { + if event := scenario.respond(action); event != nil { + return event + } + } + return defaultResponse(action) + } + lastExtraTurn := 0 + for turn := range scenario.extra { + lastExtraTurn = max(lastExtraTurn, turn) + } + + ctx := context.Background() + params := supportedEntityParameters() + var stored, outstanding []*protos.HistoryEvent + batch := []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + sessionExecutionStarted("session", scenario.input), + } + result, session, err := sessionExecutor.StartOrchestratorSession(ctx, sessionTestInstance, nil, batch, params) + require.NoError(t, err) + run := sessionRun{} + for turn := 0; ; turn++ { + replay, replayErr := replayExecutor.ExecuteOrchestrator(ctx, sessionTestInstance, stored, batch, params) + require.NoError(t, replayErr) + requireSameResponse(t, turn, replay.Response, result.Response) + require.Equal(t, replayLengths[len(replayLengths)-1], sessionLengths[len(sessionLengths)-1], "turn %d history length", turn) + require.Equal(t, len(stored)+len(batch), sessionLengths[len(sessionLengths)-1], "turn %d history length", turn) + run.responses = append(run.responses, result.Response) + run.historyLengths = append(run.historyLengths, sessionLengths[len(sessionLengths)-1]) + + if session == nil { + run.sessionEnded = true + break + } + require.Nil(t, result.Response.NumEventsProcessed, "turn %d: a partial turn must end the session", turn) + require.Nil(t, completionOf(result.Response), "turn %d: a completed turn must end the session", turn) + + // The backend applies the turn and acknowledges it. + stored = append(stored, batch...) + stored = append(stored, historyFromActions(t, result.Response.Actions)...) + session.Commit(result.Response.Actions) + for _, action := range result.Response.Actions { + if response := respond(action); response != nil { + outstanding = append(outstanding, response) + } + } + + next := turn + 1 + if len(outstanding) == 0 && next > lastExtraTurn { + session.Close() + break + } + require.Less(t, next, 50, "scenario does not finish") + deliver := len(outstanding) + if scenario.perTurn > 0 { + deliver = min(deliver, scenario.perTurn) + } + batch = []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Duration(next) * step))} + batch = append(batch, outstanding[:deliver]...) + outstanding = outstanding[deliver:] + batch = append(batch, scenario.extra[next]...) + + result, session, err = session.Resume(ctx, batch, len(stored)) + require.NoError(t, err) + } + requireNoGoroutineLeak(t, baseline) + return run +} + +func requireSameResponse(t *testing.T, turn int, want, got *protos.OrchestratorResponse) { + t.Helper() + if !proto.Equal(want, got) { + t.Fatalf("turn %d: session response differs from full replay\nreplay: %v\nsession: %v", turn, want, got) + } +} + +func completionOf(response *protos.OrchestratorResponse) *protos.CompleteOrchestrationAction { + for _, action := range response.GetActions() { + if completed := action.GetCompleteOrchestration(); completed != nil { + return completed + } + } + return nil +} + +func requireNoGoroutineLeak(t *testing.T, baseline int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for runtime.NumGoroutine() > baseline && time.Now().Before(deadline) { + runtime.Gosched() + time.Sleep(5 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), baseline, "orchestration goroutines leaked") +} + +func requireCompletedWith(t *testing.T, run sessionRun, output string) { + t.Helper() + require.True(t, run.sessionEnded, "session should end with the orchestration") + completed := completionOf(run.responses[len(run.responses)-1]) + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, completed.OrchestrationStatus, + "failure: %v", completed.GetFailureDetails()) + require.Equal(t, output, completed.GetResult().GetValue()) +} + +func eventRaised(name, rawInput string) *protos.HistoryEvent { + event := helpers.NewEventRaisedEvent(name, wrapperspb.String(rawInput)) + event.Timestamp = timestamppb.New(sessionTestStart) + return event +} + +func suspended() *protos.HistoryEvent { + event := helpers.NewSuspendOrchestrationEvent("pause") + event.Timestamp = timestamppb.New(sessionTestStart) + return event +} + +func resumed() *protos.HistoryEvent { + event := helpers.NewResumeOrchestrationEvent("continue") + event.Timestamp = timestamppb.New(sessionTestStart) + return event +} + +func TestSessionReplayEquivalence(t *testing.T) { + counter := api.NewEntityID("counter", "a") + other := api.NewEntityID("counter", "b") + + tests := []struct { + name string + scenario sessionScenario + output string + check func(t *testing.T, run sessionRun) + }{ + { + name: "activities", + output: `["first-0","second-1","third-2"]`, + scenario: sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + results := make([]string, 0, 3) + for _, name := range []string{"first", "second", "third"} { + var result string + if err := ctx.CallActivity(name, WithActivityInput(len(results))).Await(&result); err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil + }}, + }, + { + name: "activity retry", + output: `"flaky-2"`, + scenario: func() sessionScenario { + failed := false + return sessionScenario{ + orchestrator: func(ctx *OrchestrationContext) (any, error) { + var result string + err := ctx.CallActivity("flaky", WithActivityRetryPolicy(&RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 10 * time.Second, + })).Await(&result) + return result, err + }, + respond: func(action *protos.OrchestratorAction) *protos.HistoryEvent { + if action.GetScheduleTask() == nil || failed { + return nil + } + failed = true + event := helpers.NewTaskFailedEvent(action.Id, &protos.TaskFailureDetails{ + ErrorType: "Transient", ErrorMessage: "try again", + }) + event.Timestamp = timestamppb.New(sessionTestStart) + return event + }, + } + }(), + }, + { + name: "timers and long timer chunks", + output: `"done"`, + scenario: sessionScenario{ + options: []TaskExecutorOption{WithMaximumTimerInterval(time.Minute)}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(30 * time.Second).Await(nil); err != nil { + return nil, err + } + if err := ctx.CreateTimer(150 * time.Second).Await(nil); err != nil { + return nil, err + } + return "done", nil + }, + }, + }, + { + name: "sub-orchestrations and send event", + output: `["child-done","child-done"]`, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + first := ctx.CallSubOrchestrator("child", WithSubOrchestratorInput(1)) + second := ctx.CallSubOrchestrator("child", WithSubOrchestrationInstanceID("fixed-child")) + if err := ctx.SendEvent("other", "hello", "payload"); err != nil { + return nil, err + } + results := make([]string, 2) + if err := first.Await(&results[0]); err != nil { + return nil, err + } + if err := second.Await(&results[1]); err != nil { + return nil, err + } + return results, nil + }}, + }, + { + name: "external events and event channels", + output: `["a","b","c","late"]`, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{ + 1: {eventRaised("ch", `"a"`)}, + 2: {eventRaised("ch", `"b"`), eventRaised("ch", `"c"`), eventRaised("single", `"late"`)}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[string](ctx, "ch") + values := []string{} + for range 3 { + values = append(values, channel.Receive(ctx)) + } + var single string + if err := ctx.WaitForSingleEvent("single", -1).Await(&single); err != nil { + return nil, err + } + return append(values, single), nil + }, + }, + }, + { + name: "Go, wait group, WhenAll and select", + output: `"left-0|right-1|all-2,all-3|picked:timer"`, + scenario: sessionScenario{ + perTurn: 1, + extra: map[int][]*protos.HistoryEvent{9: {eventRaised("late", `"x"`)}}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + results := make([]string, 2) + wg := ctx.NewWaitGroup() + wg.Add(2) + for index, name := range []string{"left", "right"} { + ctx.Go(func(child *OrchestrationContext) { + defer wg.Done() + _ = child.CallActivity(name).Await(&results[index]) + }) + } + wg.Wait(ctx) + all := []Task{ctx.CallActivity("all"), ctx.CallActivity("all")} + if err := ctx.WhenAll(all...); err != nil { + return nil, err + } + allResults := make([]string, 2) + for i, task := range all { + _ = task.Await(&allResults[i]) + } + picked := "" + channel := NewEventChannel[string](ctx, "late") + ctx.Select( + OnTask(ctx.CreateTimer(time.Second), func(Task) { picked = "timer" }), + OnEvent(channel, func(value string) { picked = "event:" + value }), + ) + return strings.Join([]string{results[0], results[1], strings.Join(allResults, ","), "picked:" + picked}, "|"), nil + }, + }, + }, + { + name: "cancellation scopes", + output: `"canceled:after-2"`, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + timer := ctx.CreateTimer(time.Second) + child, cancel := ctx.WithCancel() + work := child.CallActivity("slow") + winner := ctx.WhenAny(timer, work) + state := "finished" + if winner == timer { + cancel() + if err := work.Await(nil); errors.Is(err, ErrTaskCanceled) { + state = "canceled" + } + } + var after string + if err := ctx.CallActivity("after").Await(&after); err != nil { + return nil, err + } + return state + ":" + after, nil + }}, + }, + { + name: "entities and critical sections", + output: `"get-result|add-result"`, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + if err := ctx.SignalEntity(other, "reset"); err != nil { + return nil, err + } + var first string + if err := ctx.CallEntity(counter, "get").Await(&first); err != nil { + return nil, err + } + release, err := ctx.LockEntities(counter, other) + if err != nil { + return nil, err + } + var second string + if err := ctx.CallEntity(counter, "add", WithEntityInput(1)).Await(&second); err != nil { + return nil, err + } + release() + return first + "|" + second, nil + }}, + }, + { + name: "canceled lock request after commit", + output: `"lock canceled"`, + check: func(t *testing.T, run sessionRun) { + unlocks := 0 + for _, response := range run.responses { + for _, action := range response.Actions { + if action.GetSendEntityMessage().GetEntityUnlockSent() != nil { + unlocks++ + } + } + } + require.Equal(t, 1, unlocks, "the late grant must be released once") + }, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + timer := ctx.CreateTimer(time.Second) + child, cancel := ctx.WithCancel() + lockErr := make([]error, 1) + done := ctx.NewWaitGroup() + done.Add(1) + ctx.Go(func(*OrchestrationContext) { + defer done.Done() + _, lockErr[0] = child.LockEntities(counter) + }) + if err := timer.Await(nil); err != nil { + return nil, err + } + cancel() + done.Wait(ctx) + if !errors.Is(lockErr[0], ErrTaskCanceled) { + return nil, fmt.Errorf("unexpected lock result: %w", lockErr[0]) + } + // Wait long enough for the late grant to be released. + if err := ctx.CreateTimer(time.Second).Await(nil); err != nil { + return nil, err + } + return "lock canceled", nil + }}, + }, + { + name: "custom status and NewGuid", + scenario: sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + ids := []string{} + for range 3 { + ids = append(ids, ctx.NewGuid()) + ctx.SetCustomStatus(fmt.Sprintf("step-%d", len(ids))) + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + ids = append(ids, ctx.NewGuid()) + return ids, nil + }}, + }, + { + name: "suspend with actions created before suspension", + output: `"work-0"`, + check: func(t *testing.T, run sessionRun) { + require.Empty(t, run.responses[1].Actions, "a suspended turn sends no actions") + require.Len(t, run.responses[2].Actions, 1) + require.Equal(t, "work", run.responses[2].Actions[0].GetScheduleTask().GetName()) + }, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{ + 1: {eventRaised("go", `""`), suspended()}, + 2: {resumed()}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + if err := ctx.WaitForSingleEvent("go", -1).Await(nil); err != nil { + return nil, err + } + var result string + err := ctx.CallActivity("work").Await(&result) + return result, err + }, + }, + }, + { + name: "events buffered while suspended", + output: `"payload"`, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{ + 1: {suspended(), eventRaised("go", `"payload"`), suspended()}, + 2: {resumed()}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + var value string + err := ctx.WaitForSingleEvent("go", -1).Await(&value) + return value, err + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := runSessionEquivalence(t, tt.scenario) + require.Greater(t, len(run.responses), 2, "scenario should run several turns") + if tt.output != "" { + requireCompletedWith(t, run, tt.output) + } else { + require.True(t, run.sessionEnded) + require.NotNil(t, completionOf(run.responses[len(run.responses)-1])) + } + if tt.check != nil { + tt.check(t, run) + } + }) + } +} + +func TestSessionCustomStatusIsCarriedAcrossTurns(t *testing.T) { + run := runSessionEquivalence(t, sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus("waiting") + if err := ctx.CallActivity("a").Await(nil); err != nil { + return nil, err + } + return nil, ctx.CallActivity("b").Await(nil) + }}) + for _, response := range run.responses { + require.Equal(t, "waiting", response.GetCustomStatus().GetValue()) + } +} + +func TestSessionEndsWithOrchestration(t *testing.T) { + waitThen := func(end func(ctx *OrchestrationContext) (any, error)) Orchestrator { + return func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + return end(ctx) + } + } + tests := []struct { + name string + scenario sessionScenario + status protos.OrchestrationStatus + }{ + { + name: "failure", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + scenario: sessionScenario{orchestrator: waitThen(func(*OrchestrationContext) (any, error) { + return nil, errors.New("boom") + })}, + }, + { + name: "continue as new", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{1: {eventRaised("kept", `"x"`)}}, + orchestrator: waitThen(func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew("next", WithKeepUnprocessedEvents()) + return nil, nil + }), + }, + }, + { + name: "termination", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{1: {func() *protos.HistoryEvent { + event := helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"stop"`), false) + event.Timestamp = timestamppb.New(sessionTestStart) + return event + }()}}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }, + }, + }, + { + name: "history limit", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + scenario: sessionScenario{ + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 6})}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + for { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + }, + }, + }, + { + name: "history limit continue as new", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + scenario: sessionScenario{ + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 6, + OnHistoryLimitExceeded: func(info HistoryLimitInfo) (any, error) { + return info.HistoryLength, nil + }, + })}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + for { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := runSessionEquivalence(t, tt.scenario) + require.True(t, run.sessionEnded) + require.Greater(t, len(run.responses), 1) + completed := completionOf(run.responses[len(run.responses)-1]) + require.NotNil(t, completed) + require.Equal(t, tt.status, completed.OrchestrationStatus) + }) + } +} + +func TestSessionEndsOnPartialTurn(t *testing.T) { + run := runSessionEquivalence(t, sessionScenario{ + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 2})}, + extra: map[int][]*protos.HistoryEvent{ + 1: {eventRaised("e", `1`), eventRaised("e", `2`), eventRaised("e", `3`)}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "e") + sum := 0 + for range 3 { + sum += channel.Receive(ctx) + } + return sum, nil + }, + }) + require.True(t, run.sessionEnded) + last := run.responses[len(run.responses)-1] + require.NotNil(t, last.NumEventsProcessed) + require.Equal(t, int32(2), last.NumEventsProcessed.GetValue()) + require.Nil(t, completionOf(last)) +} + +func TestSessionEndsOnPanicDuringResume(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + panic("resume panic") + })) + baseline := runtime.NumGoroutine() + executor := NewTaskExecutor(registry).(*taskExecutor) + first := []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), sessionExecutionStarted("session", nil)} + result, session, err := executor.StartOrchestratorSession(context.Background(), sessionTestInstance, nil, first, nil) + require.NoError(t, err) + require.NotNil(t, session) + session.Commit(result.Response.Actions) + batch := []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + defaultResponse(result.Response.Actions[0]), + } + result, next, err := session.Resume(context.Background(), batch, len(first)+1) + require.NoError(t, err) + require.Nil(t, next) + completed := completionOf(result.Response) + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, completed.OrchestrationStatus) + require.Contains(t, completed.GetFailureDetails().GetErrorMessage(), "resume panic") + requireNoGoroutineLeak(t, baseline) +} + +// TestStartOrchestratorSessionMatchesExecuteOrchestrator checks first turns +// that cannot keep a session. They must return the same result or error as +// ExecuteOrchestrator and no session. +func TestStartOrchestratorSessionMatchesExecuteOrchestrator(t *testing.T) { + started := func(name string, version string) *protos.HistoryEvent { + event := sessionExecutionStarted(name, nil) + if version != "" { + event.GetExecutionStarted().Version = wrapperspb.String(version) + } + return event + } + tests := []struct { + name string + options []TaskExecutorOption + old []*protos.HistoryEvent + new []*protos.HistoryEvent + params *protos.OrchestratorEntityParameters + wantError bool + }{ + {name: "completes", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("done", "")}}, + {name: "fails", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("fail", "")}}, + {name: "panics", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("panic", "")}}, + {name: "continues as new", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("again", "")}}, + {name: "not registered", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("missing", "")}}, + { + name: "not registered reject", + options: []TaskExecutorOption{ + WithOrchestratorNotFoundStrategy(OrchestratorNotFoundReject), + }, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("missing", "")}, + wantError: true, + }, + { + name: "version fail", + options: []TaskExecutorOption{WithVersioning(VersioningOptions{Version: "2.0", MatchStrategy: VersionMatchStrict, FailureStrategy: VersionFailureFail})}, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("wait", "1.0")}, + }, + { + name: "version reject", + options: []TaskExecutorOption{WithVersioning(VersioningOptions{Version: "2.0", MatchStrategy: VersionMatchStrict, FailureStrategy: VersionFailureReject})}, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("wait", "1.0")}, + wantError: true, + }, + { + name: "unsupported entity parameters", + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("wait", "")}, + params: &protos.OrchestratorEntityParameters{}, + wantError: true, + }, + { + name: "non-deterministic replay", + old: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + started("wait", ""), + helpers.NewTimerCreatedEvent(0, timestamppb.New(sessionTestStart)), + }, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Minute))}, + }, + { + name: "partial turn", + options: []TaskExecutorOption{ + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 1}), + }, + new: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + started("wait", ""), + eventRaised("x", `1`), + }, + }, + { + name: "rewind", + old: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + started("fail", ""), + { + EventId: -1, + Timestamp: timestamppb.New(sessionTestStart), + EventType: &protos.HistoryEvent_ExecutionCompleted{ExecutionCompleted: &protos.ExecutionCompletedEvent{ + OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + FailureDetails: &protos.TaskFailureDetails{ErrorType: "x", ErrorMessage: "y"}, + }}, + }, + }, + new: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + { + EventId: -1, + Timestamp: timestamppb.New(sessionTestStart), + EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}, + }, + }, + }, + } + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("done", func(*OrchestrationContext) (any, error) { return "ok", nil })) + require.NoError(t, registry.AddOrchestratorN("fail", func(*OrchestrationContext) (any, error) { return nil, errors.New("failed") })) + require.NoError(t, registry.AddOrchestratorN("panic", func(*OrchestrationContext) (any, error) { panic("first turn") })) + require.NoError(t, registry.AddOrchestratorN("again", func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew(1) + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("wait", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + })) + require.NoError(t, registry.AddOrchestratorNVersion("wait", "1.0", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + })) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + params := tt.params + if params == nil { + params = supportedEntityParameters() + } + baseline := runtime.NumGoroutine() + executor := NewTaskExecutor(registry, tt.options...).(*taskExecutor) + want, wantErr := executor.ExecuteOrchestrator(context.Background(), sessionTestInstance, tt.old, tt.new, params) + got, session, gotErr := executor.StartOrchestratorSession(context.Background(), sessionTestInstance, tt.old, tt.new, params) + require.Nil(t, session) + if tt.wantError { + require.Error(t, wantErr) + require.EqualError(t, gotErr, wantErr.Error()) + require.Nil(t, got) + } else { + require.NoError(t, wantErr) + require.NoError(t, gotErr) + switch tt.name { + case "panics": + // Stack traces differ between the two calls. + require.Equal(t, completionOf(want.Response).OrchestrationStatus, completionOf(got.Response).OrchestrationStatus) + case "rewind": + // Rewind generates a new execution ID on every call. + require.Len(t, got.Response.Actions, len(want.Response.Actions)) + default: + requireSameResponse(t, 0, want.Response, got.Response) + } + } + requireNoGoroutineLeak(t, baseline) + }) + } +} + +func startWaitingSession(t *testing.T, orchestrator Orchestrator) (*OrchestrationContext, OrchestratorSession, *protos.OrchestratorResponse) { + t.Helper() + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", orchestrator)) + executor := NewTaskExecutor(registry).(*taskExecutor) + result, session, err := executor.StartOrchestratorSession( + context.Background(), + sessionTestInstance, + nil, + []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), sessionExecutionStarted("session", nil)}, + nil, + ) + require.NoError(t, err) + require.NotNil(t, session) + return session.(*orchestratorSession).ctx, session, result.Response +} + +func TestSessionStateMachine(t *testing.T) { + waiting := func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("work").Await(nil) + } + validBatch := []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Minute))} + + t.Run("resume before commit", func(t *testing.T) { + _, session, _ := startWaitingSession(t, waiting) + _, next, err := session.Resume(context.Background(), validBatch, 3) + require.ErrorContains(t, err, "before the last turn was committed") + require.Nil(t, next) + _, _, err = session.Resume(context.Background(), validBatch, 3) + require.Error(t, err) + }) + t.Run("commit twice", func(t *testing.T) { + _, session, response := startWaitingSession(t, waiting) + session.Commit(response.Actions) + session.Commit(response.Actions) + _, _, err := session.Resume(context.Background(), validBatch, 3) + require.ErrorContains(t, err, "commit is out of order") + }) + t.Run("resume after close", func(t *testing.T) { + _, session, response := startWaitingSession(t, waiting) + session.Commit(response.Actions) + session.Close() + session.Close() + _, _, err := session.Resume(context.Background(), validBatch, 3) + require.ErrorIs(t, err, errOrchestratorSessionClosed) + }) + invalid := []struct { + name string + batch []*protos.HistoryEvent + base int + message string + }{ + {name: "empty", batch: nil, base: 3, message: "must start with an OrchestratorStarted"}, + {name: "no orchestrator started", batch: []*protos.HistoryEvent{eventRaised("x", `1`)}, base: 3, message: "must start with an OrchestratorStarted"}, + {name: "negative base", batch: validBatch, base: -1, message: "must not be negative"}, + {name: "new execution", batch: []*protos.HistoryEvent{validBatch[0], sessionExecutionStarted("session", nil)}, base: 3, message: "must not start a new execution"}, + {name: "rewind", batch: []*protos.HistoryEvent{validBatch[0], {EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}}}, base: 3, message: "rewind"}, + {name: "nil event", batch: []*protos.HistoryEvent{validBatch[0], nil}, base: 3, message: "nil events"}, + } + for _, tt := range invalid { + t.Run("invalid batch "+tt.name, func(t *testing.T) { + ctx, session, response := startWaitingSession(t, waiting) + session.Commit(response.Actions) + _, next, err := session.Resume(context.Background(), tt.batch, tt.base) + require.ErrorContains(t, err, tt.message) + require.Nil(t, next) + require.Nil(t, ctx.scheduler, "an invalid batch must unload the session") + _, _, err = session.Resume(context.Background(), validBatch, 3) + require.Error(t, err) + }) + } +} + +func TestSessionCommitRetiresOnlySentActions(t *testing.T) { + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WhenAll(ctx.CallActivity("a"), ctx.CallActivity("b")) + }) + require.Len(t, response.Actions, 2) + session.Commit(response.Actions[:1]) + require.Len(t, ctx.pendingActions, 1) + result, next, err := session.Resume(context.Background(), []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + }, 4) + require.NoError(t, err) + require.NotNil(t, next) + require.Len(t, result.Response.Actions, 1) + require.True(t, proto.Equal(response.Actions[1], result.Response.Actions[0]), "an unsent action must be sent again") + next.Close() +} + +func TestSessionHistoryLengthUsesHistoryBase(t *testing.T) { + var lengths []int + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *OrchestrationContext) (any, error) { + for { + if err := ctx.WaitForSingleEvent("e", -1).Await(nil); err != nil { + return nil, err + } + } + })) + executor := newSessionTestExecutor(registry, &lengths, + WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 100})) + result, session, err := executor.StartOrchestratorSession(context.Background(), sessionTestInstance, nil, + []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), sessionExecutionStarted("session", nil)}, nil) + require.NoError(t, err) + require.Equal(t, []int{2}, lengths) + session.Commit(result.Response.Actions) + + batch := []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), eventRaised("e", `1`)} + result, session, err = session.Resume(context.Background(), batch, 98) + require.NoError(t, err) + require.NotNil(t, session) + require.Equal(t, []int{2, 100}, lengths) + session.Commit(result.Response.Actions) + + result, session, err = session.Resume(context.Background(), batch, 99) + require.NoError(t, err) + require.Nil(t, session, "the history limit must end the session") + completed := completionOf(result.Response) + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, completed.OrchestrationStatus) + require.Equal(t, []int{2, 100, 101}, lengths) +} + +func TestSessionDropsProcessedHistoryBetweenTurns(t *testing.T) { + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + var value string + if err := ctx.WaitForSingleEvent("go", -1).Await(&value); err != nil { + return nil, err + } + return value, ctx.CallActivity("work").Await(nil) + }) + require.Nil(t, ctx.oldEvents) + require.Nil(t, ctx.newEvents) + require.Equal(t, 2, ctx.historyBase) + session.Commit(response.Actions) + + // The event is buffered while suspended, and it must survive the cleanup. + batch := []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + suspended(), + eventRaised("go", `"payload"`), + } + result, next, err := session.Resume(context.Background(), batch, 2) + require.NoError(t, err) + require.NotNil(t, next) + require.Empty(t, result.Response.Actions) + require.Nil(t, ctx.newEvents) + require.Len(t, ctx.suspendedEvents, 1) + next.Commit(result.Response.Actions) + + result, next, err = next.Resume(context.Background(), + []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(2 * time.Minute)), resumed()}, 5) + require.NoError(t, err) + require.NotNil(t, next) + require.Len(t, result.Response.Actions, 1) + require.Equal(t, "work", result.Response.Actions[0].GetScheduleTask().GetName()) + require.Nil(t, ctx.newEvents) + next.Close() +} + +func TestSessionCloseEmitsNoDurableWork(t *testing.T) { + baseline := runtime.NumGoroutine() + deferredBlocked := 0 + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + for range 3 { + ctx.Go(func(child *OrchestrationContext) { + defer func() { + if recovered := recover(); recovered != nil { + if errors.Is(recovered.(error), ErrTaskBlocked) { + deferredBlocked++ + } + panic(recovered) + } + }() + defer func() { + _ = child.CallActivity("cleanup") + }() + _ = child.CreateTimer(time.Hour).Await(nil) + }) + } + defer func() { + ctx.SetCustomStatus("changed") + _ = ctx.SignalEntity(api.NewEntityID("counter", "x"), "op") + }() + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }) + require.Len(t, response.Actions, 3) + session.Commit(response.Actions) + sequence := ctx.sequenceNumber + status := ctx.customStatus + require.Greater(t, runtime.NumGoroutine(), baseline) + + session.Close() + require.Equal(t, sequence, ctx.sequenceNumber, "Close must not schedule durable work") + require.Empty(t, ctx.pendingActions) + require.Equal(t, status, ctx.customStatus) + require.Equal(t, 3, deferredBlocked) + requireNoGoroutineLeak(t, baseline) +} + +func TestSessionDropsFinishedCoroutinesBetweenTurns(t *testing.T) { + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + childDone := false + ctx.Go(func(child *OrchestrationContext) { + if child.WaitForSingleEvent("child", -1).Await(nil) == nil { + childDone = true + } + }) + for range 3 { + if err := ctx.WaitForSingleEvent("e", -1).Await(nil); err != nil { + return nil, err + } + ctx.Go(func(*OrchestrationContext) {}) + } + if err := ctx.WaitForSingleEvent("end", -1).Await(nil); err != nil { + return nil, err + } + return childDone, nil + }) + live := func(want int) { + t.Helper() + require.Len(t, ctx.scheduler.all, want) + for _, c := range ctx.scheduler.all { + if c.scope != nil { + for _, scoped := range c.scope.coroutines { + require.NotEqual(t, coroutineCompleted, scoped.state, "a finished coroutine is still in its scope") + } + } + } + } + live(2) + actions := response.Actions + historyBase := 2 + turn := func(event string) (*ExecutionResults, OrchestratorSession) { + t.Helper() + session.Commit(actions) + historyBase += 2 + result, next, err := session.Resume(context.Background(), []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Duration(historyBase) * time.Minute)), + eventRaised(event, `1`), + }, historyBase) + require.NoError(t, err) + actions = result.Response.Actions + return result, next + } + for range 3 { + _, next := turn("e") + require.NotNil(t, next) + live(2) + } + _, next := turn("child") + require.NotNil(t, next) + live(1) + result, next := turn("end") + require.Nil(t, next) + requireCompletedWith(t, sessionRun{responses: []*protos.OrchestratorResponse{result.Response}, sessionEnded: true}, `true`) +} + +// A full replay reads HistoryLength again with the latest turn's value. The +// session must end once orchestrator code reads it, or a later turn could +// decide differently from the replay. +func TestSessionEndsAfterTurnHistoryRead(t *testing.T) { + for _, tt := range []struct { + name string + read func(*OrchestrationContext) bool + }{ + {name: "HistoryLength", read: func(ctx *OrchestrationContext) bool { return ctx.HistoryLength() > 9 }}, + {name: "HistoryLimitExceeded", read: func(ctx *OrchestrationContext) bool { return ctx.HistoryLimitExceeded() }}, + } { + t.Run(tt.name, func(t *testing.T) { + run := runSessionEquivalence(t, sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + for i := 0; ; i++ { + // The first reads happen on turn 2, so the session must + // survive the earlier turns. + if i >= 2 && tt.read(ctx) { + return i, nil + } + if i == 6 { + return i, nil + } + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + }}) + require.True(t, run.sessionEnded) + require.Len(t, run.responses, 3, "the session must end on the turn that read the value") + require.Nil(t, completionOf(run.responses[2])) + }) + } +} diff --git a/tests/durabletaskscheduler/dts_extended_sessions_test.go b/tests/durabletaskscheduler/dts_extended_sessions_test.go new file mode 100644 index 0000000..b8568ff --- /dev/null +++ b/tests/durabletaskscheduler/dts_extended_sessions_test.go @@ -0,0 +1,579 @@ +package durabletaskscheduler_test + +import ( + "context" + "errors" + "fmt" + "io" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// sessionTracker observes extended-session streams and can inject faults. +// The fault switches are one-shot. +type sessionTracker struct { + acks atomic.Int32 + entityAcks atomic.Int32 + open atomic.Int32 + maxOpen atomic.Int32 + unaryOrchestrations atomic.Int32 + releases atomic.Int32 + + dropAck atomic.Bool + cutBatch atomic.Bool + breakWhenIdle atomic.Bool + + mu sync.Mutex + codes map[codes.Code]int +} + +func newSessionTracker() *sessionTracker { + return &sessionTracker{codes: make(map[codes.Code]int)} +} + +func (tr *sessionTracker) recordCode(err error) { + if err == nil || errors.Is(err, io.EOF) { + return + } + tr.mu.Lock() + tr.codes[status.Code(err)]++ + tr.mu.Unlock() +} + +func (tr *sessionTracker) code(code codes.Code) int { + tr.mu.Lock() + defer tr.mu.Unlock() + return tr.codes[code] +} + +func (tr *sessionTracker) streamInterceptor() grpc.StreamClientInterceptor { + return func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + stream, err := streamer(ctx, desc, cc, method, opts...) + entity := method == protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName + if err != nil || (!entity && method != protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName) { + tr.recordCode(err) + return stream, err + } + open := tr.open.Add(1) + for { + current := tr.maxOpen.Load() + if open <= current || tr.maxOpen.CompareAndSwap(current, open) { + break + } + } + tracked := &trackedSessionStream{ClientStream: stream, tracker: tr, entity: entity} + context.AfterFunc(ctx, tracked.finish) + return tracked, nil + } +} + +func (tr *sessionTracker) unaryInterceptor() grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + switch method { + case protos.TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName: + tr.unaryOrchestrations.Add(1) + case protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName: + tr.releases.Add(1) + } + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +type trackedSessionStream struct { + grpc.ClientStream + tracker *sessionTracker + entity bool + once sync.Once + broken bool +} + +func (s *trackedSessionStream) finish() { + s.once.Do(func() { s.tracker.open.Add(-1) }) +} + +func (s *trackedSessionStream) fail(message string) error { + s.finish() + err := status.Error(codes.Unavailable, message) + s.tracker.recordCode(err) + return err +} + +func (s *trackedSessionStream) RecvMsg(m any) error { + if s.broken { + return s.fail("injected: session stream broke") + } + err := s.ClientStream.RecvMsg(m) + if err != nil { + s.finish() + s.tracker.recordCode(err) + return err + } + response := m.(*protos.CompleteTaskWithExtendedSessionResponse) + switch { + case response.GetAck() != nil: + if s.tracker.dropAck.CompareAndSwap(true, false) { + return s.fail("injected: acknowledgement lost") + } + if s.entity { + s.tracker.entityAcks.Add(1) + } else { + s.tracker.acks.Add(1) + } + s.broken = s.tracker.breakWhenIdle.CompareAndSwap(true, false) + case response.GetNewMessages() != nil: + // Deliver the first chunk, then break the stream before the batch ends. + if s.tracker.cutBatch.CompareAndSwap(true, false) { + s.broken = true + } + } + return nil +} + +func startSessionEmulatorWorker( + t *testing.T, + registry *task.TaskRegistry, + tracker *sessionTracker, + sessions *durabletaskclient.ExtendedSessionOptions, + extra ...durabletaskclient.TaskHubGrpcWorkerOption, +) *durabletaskscheduler.Client { + t.Helper() + options := emulatorOptions(t) + options.StreamInterceptors = append(slices.Clone(options.StreamInterceptors), tracker.streamInterceptor()) + options.UnaryInterceptors = append(slices.Clone(options.UnaryInterceptors), tracker.unaryInterceptor()) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(8), + durabletaskclient.WithMaxConcurrentActivityWorkItems(8), + durabletaskclient.WithMaxConcurrentEntityWorkItems(8), + durabletaskclient.WithWorkerSilentDisconnectTimeout(15 * time.Second), + } + if sessions != nil { + workerOptions = append(workerOptions, durabletaskclient.WithExtendedSessions(*sessions)) + } + client, _ := startEmulatorWithOptions(t, options, registry, append(workerOptions, extra...)...) + return client +} + +var ( + sessionProbeOnce sync.Once + sessionProbeSupported bool + sessionProbeErr error +) + +// requireExtendedSessions skips the test when the endpoint does not support +// extended sessions, for example the public emulator before the feature. +func requireExtendedSessions(t *testing.T) { + t.Helper() + emulatorOptions(t) + sessionProbeOnce.Do(func() { + // A probe that stops early, for example on a failed require, must fail + // the tests instead of skipping them. + sessionProbeErr = errors.New("the extended-session probe did not finish") + registry := task.NewTaskRegistry() + if err := registry.AddActivityN("SessionProbeActivity", func(task.ActivityContext) (any, error) { return nil, nil }); err != nil { + sessionProbeErr = err + return + } + if err := registry.AddOrchestratorN("SessionProbe", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("SessionProbeActivity").Await(nil) + }); err != nil { + sessionProbeErr = err + return + } + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, "SessionProbe") + if err != nil { + sessionProbeErr = err + return + } + if _, err := client.WaitForOrchestrationCompletion(ctx, id); err != nil { + sessionProbeErr = err + return + } + sessionProbeErr = nil + sessionProbeSupported = tracker.acks.Load() > 0 + if !sessionProbeSupported && tracker.code(codes.Unimplemented) == 0 { + sessionProbeErr = fmt.Errorf("the session probe saw neither an acknowledgement nor Unimplemented") + } + }) + require.NoError(t, sessionProbeErr) + if !sessionProbeSupported { + t.Skip("the DTS endpoint returns Unimplemented for extended sessions") + } +} + +func sessionActivityChain(name string, starts *atomic.Int32, steps int) (*task.TaskRegistry, error) { + registry := task.NewTaskRegistry() + if err := registry.AddActivityN(name+"Step", func(ctx task.ActivityContext) (any, error) { + var input int + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + 1, nil + }); err != nil { + return nil, err + } + err := registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + starts.Add(1) + value := 0 + for range steps { + if err := ctx.CallActivity(name+"Step", task.WithActivityInput(value)).Await(&value); err != nil { + return nil, err + } + } + return value, nil + }) + return registry, err +} + +func runSessionOrchestration( + t *testing.T, + client *durabletaskscheduler.Client, + name string, + opts ...api.NewOrchestrationOptions, +) *api.OrchestrationMetadata { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, name, opts...) + require.NoError(t, err) + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + return metadata +} + +func requireCompletedValue(t *testing.T, metadata *api.OrchestrationMetadata, want int) { + t.Helper() + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus, + "failure: %v", metadata.FailureDetails) + var got int + require.NoError(t, metadata.ReadOutput(&got)) + require.Equal(t, want, got) +} + +// Scenario 1: a multi-turn session, then a terminal unary completion. +func TestDTSExtendedSessionMultiTurn(t *testing.T) { + requireExtendedSessions(t) + starts := new(atomic.Int32) + registry, err := sessionActivityChain("SessionMultiTurn", starts, 4) + require.NoError(t, err) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + + requireCompletedValue(t, runSessionOrchestration(t, client, "SessionMultiTurn"), 4) + require.EqualValues(t, 1, starts.Load(), "the session path must run every turn without a replay") + require.GreaterOrEqual(t, tracker.acks.Load(), int32(4)) + require.GreaterOrEqual(t, tracker.unaryOrchestrations.Load(), int32(1)) +} + +// Scenario 2: a session, continue-as-new, and a terminal completion. +func TestDTSExtendedSessionContinueAsNew(t *testing.T) { + requireExtendedSessions(t) + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("SessionCANStep", func(task.ActivityContext) (any, error) { return nil, nil })) + require.NoError(t, registry.AddOrchestratorN("SessionCAN", func(ctx *task.OrchestrationContext) (any, error) { + starts.Add(1) + var generation int + if err := ctx.GetInput(&generation); err != nil { + return nil, err + } + for range 2 { + if err := ctx.CallActivity("SessionCANStep").Await(nil); err != nil { + return nil, err + } + } + if generation < 2 { + ctx.ContinueAsNew(generation + 1) + return nil, nil + } + return generation, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + + requireCompletedValue(t, runSessionOrchestration(t, client, "SessionCAN", api.WithInput(0)), 2) + require.EqualValues(t, 3, starts.Load(), "each execution must start once and then continue in its session") + require.GreaterOrEqual(t, tracker.acks.Load(), int32(6)) +} + +// Scenarios 3, 4 and 5: lost acknowledgement, a batch cut off after its +// first chunk, and a session stream that breaks while idle. The worker +// recovers with a full replay and the result is the same. +func TestDTSExtendedSessionRecovery(t *testing.T) { + requireExtendedSessions(t) + tests := []struct { + name string + inject func(*sessionTracker) + check func(*testing.T, *sessionTracker) + }{ + { + name: "lost acknowledgement", + inject: func(tr *sessionTracker) { tr.dropAck.Store(true) }, + check: func(t *testing.T, tr *sessionTracker) { + require.False(t, tr.dropAck.Load(), "the fault was not injected") + require.GreaterOrEqual(t, tr.code(codes.NotFound), 1, "a retry with the old count must get NotFound") + }, + }, + { + name: "batch cut off after the first chunk", + inject: func(tr *sessionTracker) { tr.cutBatch.Store(true) }, + check: func(t *testing.T, tr *sessionTracker) { + require.False(t, tr.cutBatch.Load(), "the fault was not injected") + require.GreaterOrEqual(t, tr.releases.Load(), int32(1)) + }, + }, + { + name: "stream breaks while idle", + inject: func(tr *sessionTracker) { tr.breakWhenIdle.Store(true) }, + check: func(t *testing.T, tr *sessionTracker) { + require.False(t, tr.breakWhenIdle.Load(), "the fault was not injected") + require.GreaterOrEqual(t, tr.releases.Load(), int32(1)) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := "SessionRecovery" + strings.ReplaceAll(strings.ToUpper(tt.name[:1])+tt.name[1:], " ", "") + starts := new(atomic.Int32) + registry, err := sessionActivityChain(name, starts, 3) + require.NoError(t, err) + tracker := newSessionTracker() + tt.inject(tracker) + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + requireCompletedValue(t, runSessionOrchestration(t, client, name), 3) + tt.check(t, tracker) + require.Greater(t, starts.Load(), int32(1), "recovery must use a full replay") + }) + } +} + +// Scenario 7: the worker releases idle sessions while new messages arrive. +func TestDTSExtendedSessionReleaseRace(t *testing.T) { + requireExtendedSessions(t) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("SessionReleaseRace", func(ctx *task.OrchestrationContext) (any, error) { + sum := 0 + for range 8 { + var value int + if err := ctx.WaitForSingleEvent("value", -1).Await(&value); err != nil { + return nil, err + } + sum += value + } + return sum, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, + &durabletaskclient.ExtendedSessionOptions{IdleTimeout: 100 * time.Millisecond}) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, "SessionReleaseRace") + require.NoError(t, err) + _, err = client.WaitForOrchestrationStart(ctx, id) + require.NoError(t, err) + // The idle timer starts at the acknowledgement, not at RaiseEvent, so wait + // for the first idle release instead of relying on the delays below. + require.Eventually(t, func() bool { return tracker.releases.Load() >= 1 }, time.Minute, 10*time.Millisecond, + "the idle timeout must release the waiting session") + for i := 1; i <= 8; i++ { + time.Sleep(time.Duration(40*(i%4)) * time.Millisecond) + require.NoError(t, client.RaiseEvent(ctx, id, "value", api.WithEventPayload(i))) + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + requireCompletedValue(t, metadata, 36) +} + +// Scenario 9: entity operations across turns, a failed batch, and deletion. +func TestDTSExtendedSessionEntities(t *testing.T) { + requireExtendedSessions(t) + var failBatches atomic.Bool + var failedBatches atomic.Int32 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityFactoryN("SessionCounter", func(task.EntityFactoryContext) (task.EntityBatch, error) { + if failBatches.Load() { + failedBatches.Add(1) + return task.EntityBatch{}, errors.New("injected entity batch failure") + } + return task.EntityBatch{Entity: func(ctx *task.EntityContext) (any, error) { + if ctx.Operation == "delete" { + ctx.DeleteState() + return nil, nil + } + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + return nil, ctx.SetState(value + 1) + }}, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + entityID := api.NewEntityID("SessionCounter", uuid.NewString()) + requireState := func(want string) { + t.Helper() + require.Eventually(t, func() bool { + metadata, err := client.GetEntity(ctx, entityID) + if err != nil { + return false + } + if want == "" { + return metadata == nil || !metadata.HasState + } + return metadata != nil && metadata.SerializedState == want + }, time.Minute, 50*time.Millisecond, "entity state did not become %q", want) + } + + for range 3 { + require.NoError(t, client.SignalEntity(ctx, entityID, "add")) + time.Sleep(300 * time.Millisecond) + } + requireState("3") + require.GreaterOrEqual(t, tracker.entityAcks.Load(), int32(1), "entity batches must use sessions") + + failBatches.Store(true) + require.NoError(t, client.SignalEntity(ctx, entityID, "add")) + require.Eventually(t, func() bool { return failedBatches.Load() > 0 }, time.Minute, 10*time.Millisecond) + failBatches.Store(false) + requireState("4") + + require.NoError(t, client.SignalEntity(ctx, entityID, "delete")) + requireState("") +} + +// Scenario 10: a session crosses the history limit, and the result equals a +// worker without sessions. +func TestDTSExtendedSessionHistoryLimitMatchesReplay(t *testing.T) { + requireExtendedSessions(t) + run := func(sessions *durabletaskclient.ExtendedSessionOptions) *api.OrchestrationMetadata { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("SessionLimitStep", func(task.ActivityContext) (any, error) { return nil, nil })) + require.NoError(t, registry.AddOrchestratorN("SessionHistoryLimit", func(ctx *task.OrchestrationContext) (any, error) { + for { + if err := ctx.CallActivity("SessionLimitStep").Await(nil); err != nil { + return nil, err + } + } + })) + tracker := newSessionTracker() + options := emulatorOptions(t) + options.StreamInterceptors = append(slices.Clone(options.StreamInterceptors), tracker.streamInterceptor()) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(4), + durabletaskclient.WithTaskExecutorOptions(task.WithOrchestrationOptions(task.OrchestrationOptions{MaxHistoryEvents: 30})), + } + if sessions != nil { + workerOptions = append(workerOptions, durabletaskclient.WithExtendedSessions(*sessions)) + } + client, worker := startEmulatorWithOptions(t, options, registry, workerOptions...) + metadata := runSessionOrchestration(t, client, "SessionHistoryLimit") + if sessions != nil { + require.Greater(t, tracker.acks.Load(), int32(1), "the first run must use sessions") + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + return metadata + } + withSessions := run(&durabletaskclient.ExtendedSessionOptions{}) + withoutSessions := run(nil) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, withSessions.RuntimeStatus) + require.Equal(t, withoutSessions.RuntimeStatus, withSessions.RuntimeStatus) + require.NotNil(t, withSessions.FailureDetails) + require.Equal(t, withoutSessions.FailureDetails.ErrorType, withSessions.FailureDetails.ErrorType) + normalize := func(metadata *api.OrchestrationMetadata) string { + return strings.ReplaceAll(metadata.FailureDetails.ErrorMessage, string(metadata.InstanceID), "") + } + require.Equal(t, normalize(withoutSessions), normalize(withSessions)) +} + +// Scenario 11: when the stream budget is full, turns use the unary RPC, and +// sessions start again once streams are free. +func TestDTSExtendedSessionStreamBudget(t *testing.T) { + requireExtendedSessions(t) + const instances = 56 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("SessionBudget", func(ctx *task.OrchestrationContext) (any, error) { + var value int + if err := ctx.WaitForSingleEvent("go", -1).Await(&value); err != nil { + return nil, err + } + return value, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}, + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(2*instances), + durabletaskclient.WithWorkerCompletionConnections(1), + ) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + ids := make([]api.InstanceID, 0, instances) + for range instances { + id, err := client.ScheduleNewOrchestration(ctx, "SessionBudget") + require.NoError(t, err) + ids = append(ids, id) + } + require.Eventually(t, func() bool { + return tracker.acks.Load()+tracker.unaryOrchestrations.Load() >= instances + }, 2*time.Minute, 50*time.Millisecond) + require.LessOrEqual(t, tracker.maxOpen.Load(), int32(50), "one connection must not exceed its session stream budget") + require.Greater(t, tracker.unaryOrchestrations.Load(), int32(0), "turns over the budget must use the unary RPC") + for i, id := range ids { + require.NoError(t, client.RaiseEvent(ctx, id, "go", api.WithEventPayload(i))) + } + for i, id := range ids { + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + requireCompletedValue(t, metadata, i) + } + + // With free streams, a new instance uses a session again. + acksBefore := tracker.acks.Load() + id, err := client.ScheduleNewOrchestration(ctx, "SessionBudget") + require.NoError(t, err) + require.Eventually(t, func() bool { return tracker.acks.Load() > acksBefore }, time.Minute, 20*time.Millisecond) + require.NoError(t, client.RaiseEvent(ctx, id, "go", api.WithEventPayload(7))) + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + requireCompletedValue(t, metadata, 7) +} diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go new file mode 100644 index 0000000..c1f71ff --- /dev/null +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -0,0 +1,1330 @@ +//go:build stress && unix + +// Stress and performance tests for extended sessions. They are not part of +// normal CI. Run them by hand against a session-enabled DTS endpoint: +// +// DTS_EMULATOR_ENDPOINT=http://127.0.0.1:8080 \ +// DTS_STRESS_RESTART=/path/to/restart-backend.sh \ +// go test -tags stress -run 'TestDTSPerf|TestDTSStress' -timeout 6h -v ./tests/durabletaskscheduler +// +// Settings: +// - DTS_STRESS_RESTART: a shell command that restarts the backend. The +// harness then waits until the endpoint answers. The perf test restarts +// the backend before every run. The backend-restart stress test needs it. +// - DTS_STRESS_PERSISTENT=1: the backend keeps its state across restarts, +// so the backend-restart test checks every instance and not only new ones. +// - DTS_STRESS_SCALE: multiplies instance, step, and signal counts (default 1). +// - DTS_STRESS_REPEAT: perf repetitions. The table shows the median (default 3). +// - DTS_STRESS_SESSIONS=off: run the stress tests without extended sessions. +// - DTS_STRESS_FAULT_RATE: the chance of a fault on each session stream in +// the chaos test (default 0.1). +// - DTS_STRESS_VERBOSE=1: print worker warnings, not only errors. +// - DTS_STRESS_TRACE=1: print every work item the worker receives and every +// completion, abandon, and release it sends, with the completion token. +// +// Use 127.0.0.1 and not localhost. On some machines every localhost +// connection waits for an IPv6 attempt first. +// +// The CPU and memory numbers are for the whole test process, so they also +// include the client. + +package durabletaskscheduler_test + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "math" + "math/rand/v2" + "os" + "os/exec" + "runtime" + "runtime/pprof" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type stressFault int + +const ( + noFault stressFault = iota + faultCutBeforeSend + faultDropAck + faultCutMidBatch + faultCutAfterAck + faultDelayAfterAck + stressFaultKinds +) + +var stressFaultNames = [stressFaultKinds]string{ + "none", "cut-before-send", "drop-ack", "cut-mid-batch", "cut-after-ack", "delay-after-ack", +} + +// stressProbe counts what the worker sends and receives, and can inject +// faults on session streams. +type stressProbe struct { + faultRate float64 + idleTimeout time.Duration + + payloadBytes atomic.Int64 + workItems atomic.Int64 + replays atomic.Int64 + resumes atomic.Int64 + acks atomic.Int64 + unary atomic.Int64 + abandons atomic.Int64 + releases atomic.Int64 + notFound atomic.Int64 + open atomic.Int64 + maxOpen atomic.Int64 + faults [stressFaultKinds]atomic.Int64 + // delayHolds counts held messages, including holds that a worker + // restart ended before the idle timeout. + delayHolds atomic.Int64 + + mu sync.Mutex + rng *rand.Rand + turnStart map[string]time.Time + latencies []time.Duration +} + +func newStressProbe(faultRate float64, idleTimeout time.Duration) *stressProbe { + return &stressProbe{ + faultRate: faultRate, + idleTimeout: idleTimeout, + rng: rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 1)), + turnStart: make(map[string]time.Time), + } +} + +func (p *stressProbe) pickFault() stressFault { + if p.faultRate <= 0 { + return noFault + } + p.mu.Lock() + defer p.mu.Unlock() + if p.rng.Float64() >= p.faultRate { + return noFault + } + return stressFault(1 + p.rng.IntN(int(stressFaultKinds)-1)) +} + +func (p *stressProbe) recordErr(err error) { + if status.Code(err) == codes.NotFound { + p.notFound.Add(1) + } +} + +// startTurn marks the time when the input of a turn is ready. +func (p *stressProbe) startTurn(token string) { + p.mu.Lock() + p.turnStart[token] = time.Now() + p.mu.Unlock() +} + +// finishTurn records the worker time of a turn when its completion is sent. +func (p *stressProbe) finishTurn(token string) { + p.mu.Lock() + defer p.mu.Unlock() + if start, ok := p.turnStart[token]; ok { + p.latencies = append(p.latencies, time.Since(start)) + delete(p.turnStart, token) + } +} + +func (p *stressProbe) faultCount() int64 { + var total int64 + for i := range p.faults { + total += p.faults[i].Load() + } + return total +} + +func (p *stressProbe) streamInterceptor() grpc.StreamClientInterceptor { + return func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + switch method { + case protos.TaskHubSidecarService_GetWorkItems_FullMethodName, + protos.TaskHubSidecarService_StreamInstanceHistory_FullMethodName: + stream, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + p.recordErr(err) + return nil, err + } + return &probedReadStream{ClientStream: stream, probe: p}, nil + case protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName, + protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName: + streamCtx, cancel := context.WithCancel(ctx) + stream, err := streamer(streamCtx, desc, cc, method, opts...) + if err != nil { + cancel() + p.recordErr(err) + return nil, err + } + open := p.open.Add(1) + for current := p.maxOpen.Load(); open > current && !p.maxOpen.CompareAndSwap(current, open); current = p.maxOpen.Load() { + } + s := &probedSessionStream{ClientStream: stream, probe: p, ctx: streamCtx, cancel: cancel, fault: p.pickFault()} + context.AfterFunc(streamCtx, s.finish) + return s, nil + default: + return streamer(ctx, desc, cc, method, opts...) + } + } +} + +func (p *stressProbe) unaryInterceptor() grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + switch method { + case protos.TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName: + p.unary.Add(1) + p.finishTurn(req.(*protos.OrchestratorResponse).GetCompletionToken()) + case protos.TaskHubSidecarService_CompleteEntityTask_FullMethodName: + p.unary.Add(1) + p.finishTurn(req.(*protos.EntityBatchResult).GetCompletionToken()) + case protos.TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName, + protos.TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName: + p.abandons.Add(1) + case protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName: + p.releases.Add(1) + } + err := invoker(ctx, method, req, reply, cc, opts...) + if token, ok := req.(interface{ GetCompletionToken() string }); ok { + stressTracef("%s %s: %v", method[strings.LastIndex(method, "/")+1:], token.GetCompletionToken(), err) + } + p.recordErr(err) + return err + } +} + +type probedReadStream struct { + grpc.ClientStream + probe *stressProbe +} + +func (s *probedReadStream) RecvMsg(m any) error { + err := s.ClientStream.RecvMsg(m) + if err != nil { + s.probe.recordErr(err) + return err + } + switch message := m.(type) { + case *protos.HistoryChunk: + s.probe.payloadBytes.Add(int64(proto.Size(message))) + case *protos.WorkItem: + if request := message.GetActivityRequest(); request != nil { + stressTracef("received activity %s %s", message.GetCompletionToken(), request.GetOrchestrationInstance().GetInstanceId()) + } + switch { + case message.GetOrchestratorRequest() != nil: + request := message.GetOrchestratorRequest() + if len(request.GetPastEvents()) > 0 || request.GetRequiresHistoryStreaming() { + s.probe.replays.Add(1) + } + case message.GetEntityRequestV2() != nil, message.GetEntityRequest() != nil: + default: + return nil + } + stressTracef("received %s %s%s", message.GetCompletionToken(), + message.GetOrchestratorRequest().GetInstanceId(), message.GetEntityRequestV2().GetInstanceId()) + s.probe.workItems.Add(1) + s.probe.payloadBytes.Add(int64(proto.Size(message))) + s.probe.startTurn(message.GetCompletionToken()) + } + return nil +} + +type probedSessionStream struct { + grpc.ClientStream + probe *stressProbe + ctx context.Context + cancel context.CancelFunc + fault stressFault + once sync.Once + token string + batch bool + broken bool + delay bool + ackAt time.Time +} + +func (s *probedSessionStream) finish() { + s.once.Do(func() { s.probe.open.Add(-1) }) +} + +// inject breaks the stream on both sides, so the service also sees it end. +func (s *probedSessionStream) inject() error { + stressTracef("session fault %s: %s", s.token, stressFaultNames[s.fault]) + s.probe.faults[s.fault].Add(1) + s.cancel() + s.finish() + return status.Error(codes.Unavailable, "injected: "+stressFaultNames[s.fault]) +} + +func (s *probedSessionStream) SendMsg(m any) error { + if s.fault == faultCutBeforeSend { + return s.inject() + } + switch request := m.(type) { + case *protos.OrchestratorResponse: + s.token = request.GetCompletionToken() + case *protos.EntityBatchResult: + s.token = request.GetCompletionToken() + } + s.probe.finishTurn(s.token) + stressTracef("session send %s", s.token) + return s.ClientStream.SendMsg(m) +} + +func (s *probedSessionStream) RecvMsg(m any) error { + if s.broken { + return s.inject() + } + if s.delay { + // Hold back the next message past the worker's idle timeout. The + // worker usually ends the stream first. The fault counts only when + // the idle timeout passed since the acknowledgement, which is when + // the worker's idle timer starts. + s.delay = false + s.probe.delayHolds.Add(1) + var cancelled bool + select { + case <-time.After(s.probe.idleTimeout + time.Second): + case <-s.ctx.Done(): + cancelled = true + } + if time.Since(s.ackAt) >= s.probe.idleTimeout { + s.probe.faults[faultDelayAfterAck].Add(1) + } + if cancelled { + s.finish() + return status.FromContextError(s.ctx.Err()).Err() + } + } + err := s.ClientStream.RecvMsg(m) + if err != nil { + if errors.Is(err, io.EOF) && s.batch { + s.probe.resumes.Add(1) + s.probe.startTurn(s.token) + } + s.probe.recordErr(err) + stressTracef("session end %s batch=%v: %v", s.token, s.batch, err) + s.cancel() + s.finish() + return err + } + response := m.(*protos.CompleteTaskWithExtendedSessionResponse) + switch { + case response.GetAck() != nil: + if s.fault == faultDropAck { + return s.inject() + } + s.probe.acks.Add(1) + s.ackAt = time.Now() + s.broken = s.fault == faultCutAfterAck + s.delay = s.fault == faultDelayAfterAck + case response.GetNewMessages() != nil: + s.probe.payloadBytes.Add(int64(proto.Size(response.GetNewMessages()))) + s.batch = true + s.broken = s.fault == faultCutMidBatch + } + return nil +} + +var stressTrace = os.Getenv("DTS_STRESS_TRACE") == "1" + +func stressTracef(format string, args ...any) { + if stressTrace { + fmt.Fprintf(os.Stderr, "TRACE %s "+format+"\n", append([]any{time.Now().Format("15:04:05.000000")}, args...)...) + } +} + +func envFloat(name string, fallback float64) float64 { + if value, err := strconv.ParseFloat(os.Getenv(name), 64); err == nil { + return value + } + return fallback +} + +func scaled(n int) int { + return max(1, int(math.Round(float64(n)*envFloat("DTS_STRESS_SCALE", 1)))) +} + +func stressLogger() api.Logger { + level := slog.LevelError + if os.Getenv("DTS_STRESS_VERBOSE") == "1" { + level = slog.LevelWarn + } + return api.NewSlogLogger(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))) +} + +// restartBackend runs DTS_STRESS_RESTART and waits until the endpoint +// answers. It does nothing when no restart command is set. +func restartBackend(t *testing.T) error { + command := os.Getenv("DTS_STRESS_RESTART") + if command == "" { + return nil + } + if output, err := exec.Command("sh", "-c", command).CombinedOutput(); err != nil { + return fmt.Errorf("restart command failed: %w: %s", err, output) + } + // NewClient sends a Hello, so it is part of the readiness check too. + var err error + for deadline := time.Now().Add(3 * time.Minute); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) { + if err = backendAnswers(t); err == nil { + return nil + } + } + return fmt.Errorf("the backend did not answer after the restart: %w", err) +} + +func backendAnswers(t *testing.T) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := durabletaskscheduler.NewClient(ctx, emulatorOptions(t), stressLogger()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + _, err = client.FetchOrchestrationMetadata(ctx, api.InstanceID("stress-ready-"+uuid.NewString())) + if errors.Is(err, api.ErrInstanceNotFound) { + return nil + } + return err +} + +// stressHarness owns one management client and a worker that can be +// replaced while the test runs. +type stressHarness struct { + t *testing.T + registry *task.TaskRegistry + probe *stressProbe + workerOptions []durabletaskclient.TaskHubGrpcWorkerOption + client *durabletaskscheduler.Client + + mu sync.Mutex + worker *durabletaskclient.TaskHubGrpcWorker + restarts int +} + +func newStressHarness( + t *testing.T, + registry *task.TaskRegistry, + probe *stressProbe, + sessions *durabletaskclient.ExtendedSessionOptions, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) *stressHarness { + t.Helper() + client, err := durabletaskscheduler.NewClient(context.Background(), emulatorOptions(t), stressLogger()) + require.NoError(t, err) + options := append([]durabletaskclient.TaskHubGrpcWorkerOption{durabletaskclient.WithAutoWorkItemFilters()}, workerOptions...) + if sessions != nil { + options = append(options, durabletaskclient.WithExtendedSessions(*sessions)) + } + h := &stressHarness{t: t, registry: registry, probe: probe, workerOptions: options, client: client} + require.NoError(t, h.startWorker()) + return h +} + +func (h *stressHarness) startWorker() error { + options := emulatorOptions(h.t) + options.StreamInterceptors = append(slices.Clone(options.StreamInterceptors), h.probe.streamInterceptor()) + options.UnaryInterceptors = append(slices.Clone(options.UnaryInterceptors), h.probe.unaryInterceptor()) + worker, err := durabletaskscheduler.NewWorker(options, h.registry, stressLogger(), h.workerOptions...) + if err != nil { + return err + } + if err := worker.Start(context.Background()); err != nil { + return err + } + h.mu.Lock() + h.worker = worker + h.mu.Unlock() + return nil +} + +func (h *stressHarness) stopWorker() { + h.mu.Lock() + worker := h.worker + h.worker = nil + h.mu.Unlock() + if worker == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + assert.NoError(h.t, worker.Shutdown(ctx), "worker shutdown") +} + +// restartWorker replaces the worker. It is safe to call from any goroutine. +func (h *stressHarness) restartWorker() { + h.stopWorker() + assert.NoError(h.t, h.startWorker()) + h.mu.Lock() + h.restarts++ + h.mu.Unlock() +} + +func (h *stressHarness) close() { + h.stopWorker() + require.NoError(h.t, h.client.Close()) +} + +func stressRegistry(t *testing.T) *task.TaskRegistry { + t.Helper() + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("PerfStep", func(ctx task.ActivityContext) (any, error) { + var input int + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + 1, nil + })) + require.NoError(t, registry.AddOrchestratorN("PerfChain", func(ctx *task.OrchestrationContext) (any, error) { + var steps int + if err := ctx.GetInput(&steps); err != nil { + return nil, err + } + value := 0 + for range steps { + if err := ctx.CallActivity("PerfStep", task.WithActivityInput(value)).Await(&value); err != nil { + return nil, err + } + } + return value, nil + })) + require.NoError(t, registry.AddOrchestratorN("PerfEvents", func(ctx *task.OrchestrationContext) (any, error) { + var events int + if err := ctx.GetInput(&events); err != nil { + return nil, err + } + sum := 0 + for i := range events { + var value int + if err := ctx.WaitForSingleEvent("e", -1).Await(&value); err != nil { + return nil, err + } + sum += value + if err := ctx.SetCustomStatusValue(i + 1); err != nil { + return nil, err + } + } + return sum, nil + })) + require.NoError(t, registry.AddOrchestratorN("PerfFanOut", func(ctx *task.OrchestrationContext) (any, error) { + var width int + if err := ctx.GetInput(&width); err != nil { + return nil, err + } + return fanOut(ctx, 0, width) + })) + require.NoError(t, registry.AddEntityN("StressCounter", func(ctx *task.EntityContext) (any, error) { + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + if ctx.Operation != "add" { + return nil, fmt.Errorf("unknown operation %q", ctx.Operation) + } + var amount int + if err := ctx.GetInput(&amount); err != nil { + return nil, err + } + value += amount + return value, ctx.SetState(value) + })) + require.NoError(t, registry.AddOrchestratorN("StressMixed", stressMixed)) + return registry +} + +func fanOut(ctx *task.OrchestrationContext, base, width int) (int, error) { + tasks := make([]task.Task, width) + for i := range tasks { + tasks[i] = ctx.CallActivity("PerfStep", task.WithActivityInput(base+i)) + } + if err := ctx.WhenAll(tasks...); err != nil { + return 0, err + } + sum := 0 + for _, pending := range tasks { + var value int + if err := pending.Await(&value); err != nil { + return 0, err + } + sum += value + } + return sum, nil +} + +type stressInput struct { + Index int + Entities int + Prefix string +} + +const stressEventsPerInstance = 3 + +func stressEntityID(prefix string, index int) api.EntityID { + return api.NewEntityID("StressCounter", fmt.Sprintf("%s-%d", prefix, index)) +} + +// stressMixed runs one of six shapes, chosen by the instance index. +// stressExpected returns its result. +func stressMixed(ctx *task.OrchestrationContext) (any, error) { + var input stressInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + i := input.Index + switch i % 6 { + case 0: + value := i + for range 3 + i%5 { + if err := ctx.CallActivity("PerfStep", task.WithActivityInput(value)).Await(&value); err != nil { + return nil, err + } + } + return value, nil + case 1: + return fanOut(ctx, i, 10) + case 2: + sum := 0 + for range stressEventsPerInstance { + var value int + if err := ctx.WaitForSingleEvent("e", 30*time.Minute).Await(&value); err != nil { + return nil, err + } + sum += value + } + return sum, nil + case 3: + sum := 0 + for range 2 { + var value int + if err := ctx.CallSubOrchestrator("PerfChain", task.WithSubOrchestratorInput(3)).Await(&value); err != nil { + return nil, err + } + sum += value + } + return sum, nil + case 4: + if err := ctx.CreateTimer(50 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + var value int + err := ctx.CallActivity("PerfStep", task.WithActivityInput(i)).Await(&value) + return value, err + default: + entityID := stressEntityID(input.Prefix, i%input.Entities) + // i%6 == 5 here, so i is always odd. Use i/6 to alternate the shapes. + if (i/6)%2 == 1 { + unlock, err := ctx.LockEntities(entityID) + if err != nil { + return nil, err + } + defer unlock() + } + return 1, ctx.CallEntity(entityID, "add", task.WithEntityInput(1)).Await(nil) + } +} + +func stressExpected(i int) int { + switch i % 6 { + case 0: + return i + 3 + i%5 + case 1: + return 10*i + 45 + 10 + case 2: + return stressEventsPerInstance * i + case 3: + return 6 + case 4: + return i + 1 + default: + return 1 + } +} + +// ---------------------------------------------------------------------------- +// Performance + +type perfResult struct { + wall time.Duration + units int + cpu time.Duration + totalAlloc uint64 + peakHeap uint64 + payloadBytes int64 + workItems int64 + replays int64 + resumes int64 + acks int64 + unary int64 + p50 time.Duration + p95 time.Duration + p99 time.Duration +} + +type perfWorkload struct { + name string + units string + // serial workloads drive one instance through many turns, so with + // sessions on it must load once and resume for almost every turn. + serial bool + run func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int +} + +func processCPU() time.Duration { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + return 0 + } + return time.Duration(usage.Utime.Nano() + usage.Stime.Nano()) +} + +func percentile(sorted []time.Duration, q float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + return sorted[min(len(sorted)-1, int(math.Ceil(q*float64(len(sorted))))-1)] +} + +// waitCompleted is safe to call from any goroutine. +func waitCompleted(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, want int) { + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if !assert.NoError(t, err) { + return + } + var got int + if assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus, "failure: %v", metadata.FailureDetails) && + assert.NoError(t, metadata.ReadOutput(&got)) { + assert.Equal(t, want, got, "%s output", id) + } +} + +func perfWorkloads() []perfWorkload { + return []perfWorkload{ + {name: "chain", units: "activities", serial: true, run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + steps := scaled(500) + id, err := client.ScheduleNewOrchestration(ctx, "PerfChain", api.WithInput(steps)) + require.NoError(t, err) + waitCompleted(t, ctx, client, id, steps) + return steps + }}, + {name: "events", units: "events", serial: true, run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + events := scaled(200) + id, err := client.ScheduleNewOrchestration(ctx, "PerfEvents", api.WithInput(events)) + require.NoError(t, err) + for i := range events { + require.NoError(t, client.RaiseEvent(ctx, id, "e", api.WithEventPayload(i))) + // Raise the next event only after this one was processed, so + // every event is its own turn. + for { + metadata, err := client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + var processed int + if metadata.SerializedCustomStatus != "" { + require.NoError(t, metadata.ReadCustomStatus(&processed)) + } + if processed > i { + break + } + time.Sleep(time.Millisecond) + } + } + waitCompleted(t, ctx, client, id, events*(events-1)/2) + return events + }}, + {name: "fan-out", units: "activities", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + width := scaled(200) + id, err := client.ScheduleNewOrchestration(ctx, "PerfFanOut", api.WithInput(width)) + require.NoError(t, err) + waitCompleted(t, ctx, client, id, width*(width+1)/2) + return width + }}, + {name: "entity", units: "signals", serial: true, run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + signals := scaled(1000) + entityID := stressEntityID("perf-"+uuid.NewString(), 0) + for range signals { + require.NoError(t, client.SignalEntity(ctx, entityID, "add", api.WithSignalInput(1))) + } + want := strconv.Itoa(signals) + require.Eventually(t, func() bool { + metadata, err := client.GetEntity(ctx, entityID) + return err == nil && metadata != nil && metadata.SerializedState == want + }, 10*time.Minute, 5*time.Millisecond) + return signals + }}, + {name: "short", units: "activities", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + instances := scaled(1000) + ids := make([]api.InstanceID, instances) + runParallel(instances, 32, func(i int) { + id, err := client.ScheduleNewOrchestration(ctx, "PerfChain", api.WithInput(5)) + assert.NoError(t, err) + ids[i] = id + }) + require.False(t, t.Failed()) + runParallel(instances, 32, func(i int) { waitCompleted(t, ctx, client, ids[i], 5) }) + return 5 * instances + }}, + } +} + +func runParallel(n, workers int, fn func(int)) { + var next atomic.Int64 + var wg sync.WaitGroup + for range min(n, workers) { + wg.Go(func() { + for i := int(next.Add(1)) - 1; i < n; i = int(next.Add(1)) - 1 { + fn(i) + } + }) + } + wg.Wait() +} + +func runPerf(t *testing.T, workload perfWorkload, sessions bool) perfResult { + t.Helper() + var sessionOptions *durabletaskclient.ExtendedSessionOptions + if sessions { + sessionOptions = &durabletaskclient.ExtendedSessionOptions{} + } + probe := newStressProbe(0, 0) + h := newStressHarness(t, stressRegistry(t), probe, sessionOptions, + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(16), + durabletaskclient.WithMaxConcurrentActivityWorkItems(32), + durabletaskclient.WithMaxConcurrentEntityWorkItems(16), + ) + defer h.close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + var peakHeap atomic.Uint64 + stopSampler := make(chan struct{}) + sampled := make(chan struct{}) + go func() { + defer close(sampled) + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + if stats.HeapInuse > peakHeap.Load() { + peakHeap.Store(stats.HeapInuse) + } + select { + case <-stopSampler: + return + case <-ticker.C: + } + } + }() + var stopOnce sync.Once + stop := func() { + stopOnce.Do(func() { + close(stopSampler) + <-sampled + }) + } + // A failed workload exits through runtime.Goexit, so stop the sampler then too. + defer stop() + cpuStart := processCPU() + start := time.Now() + units := workload.run(t, ctx, h.client) + wall := time.Since(start) + cpu := processCPU() - cpuStart + stop() + var after runtime.MemStats + runtime.ReadMemStats(&after) + + probe.mu.Lock() + latencies := slices.Clone(probe.latencies) + probe.mu.Unlock() + slices.Sort(latencies) + if sessions { + require.Positive(t, probe.acks.Load(), "the sessions-on run did not use sessions") + if workload.serial { + // DTS may combine entity signals into a few batches, so check that + // every turn after the first two loads resumed a session, not a + // share of turns. + turns := probe.acks.Load() + probe.unary.Load() + require.LessOrEqual(t, probe.workItems.Load(), int64(2), "a serial workload loaded its instance more than twice") + require.GreaterOrEqual(t, probe.resumes.Load(), turns-2, "only %d of %d turns resumed a session", probe.resumes.Load(), turns) + } + } else { + require.Zero(t, probe.acks.Load(), "the sessions-off run used sessions") + } + return perfResult{ + wall: wall, + units: units, + cpu: cpu, + totalAlloc: after.TotalAlloc - before.TotalAlloc, + peakHeap: peakHeap.Load() - min(peakHeap.Load(), before.HeapInuse), + payloadBytes: probe.payloadBytes.Load(), + workItems: probe.workItems.Load(), + replays: probe.replays.Load(), + resumes: probe.resumes.Load(), + acks: probe.acks.Load(), + unary: probe.unary.Load(), + p50: percentile(latencies, 0.50), + p95: percentile(latencies, 0.95), + p99: percentile(latencies, 0.99), + } +} + +func medianOf[T int64 | uint64 | time.Duration](results []perfResult, field func(perfResult) T) T { + values := make([]T, len(results)) + for i, result := range results { + values[i] = field(result) + } + slices.Sort(values) + return values[len(values)/2] +} + +func formatPerfRows(workload perfWorkload, off, on []perfResult) string { + var b strings.Builder + row := func(metric string, offValue, onValue float64, format func(float64) string) { + change := "n/a" + if offValue != 0 { + change = fmt.Sprintf("%+.1f%%", 100*(onValue-offValue)/offValue) + } + fmt.Fprintf(&b, "| %s | %s | %s | %s | %s |\n", workload.name, metric, format(offValue), format(onValue), change) + } + ms := func(v float64) string { return fmt.Sprintf("%.2f ms", v/float64(time.Millisecond)) } + mib := func(v float64) string { + if v < 1<<20 { + return fmt.Sprintf("%.1f KiB", v/(1<<10)) + } + return fmt.Sprintf("%.1f MiB", v/(1<<20)) + } + count := func(v float64) string { return strconv.FormatFloat(v, 'f', 0, 64) } + perSecond := func(v float64) string { return fmt.Sprintf("%.0f %s/s", v, workload.units) } + durationRow := func(metric string, field func(perfResult) time.Duration) { + row(metric, float64(medianOf(off, field)), float64(medianOf(on, field)), ms) + } + intRow := func(metric string, field func(perfResult) int64, format func(float64) string) { + row(metric, float64(medianOf(off, field)), float64(medianOf(on, field)), format) + } + throughput := func(results []perfResult) float64 { + wall := medianOf(results, func(r perfResult) time.Duration { return r.wall }) + return float64(results[0].units) / wall.Seconds() + } + durationRow("wall time", func(r perfResult) time.Duration { return r.wall }) + row("throughput", throughput(off), throughput(on), perSecond) + durationRow("turn p50", func(r perfResult) time.Duration { return r.p50 }) + durationRow("turn p95", func(r perfResult) time.Duration { return r.p95 }) + durationRow("turn p99", func(r perfResult) time.Duration { return r.p99 }) + durationRow("process CPU", func(r perfResult) time.Duration { return r.cpu }) + row("allocated", float64(medianOf(off, func(r perfResult) uint64 { return r.totalAlloc })), + float64(medianOf(on, func(r perfResult) uint64 { return r.totalAlloc })), mib) + row("peak heap growth", float64(medianOf(off, func(r perfResult) uint64 { return r.peakHeap })), + float64(medianOf(on, func(r perfResult) uint64 { return r.peakHeap })), mib) + intRow("worker protobuf payload bytes", func(r perfResult) int64 { return r.payloadBytes }, mib) + intRow("work items (loads)", func(r perfResult) int64 { return r.workItems }, count) + intRow("replays", func(r perfResult) int64 { return r.replays }, count) + intRow("session resumes", func(r perfResult) int64 { return r.resumes }, count) + intRow("session turns (acks)", func(r perfResult) int64 { return r.acks }, count) + intRow("unary completions", func(r perfResult) int64 { return r.unary }, count) + return b.String() +} + +// TestDTSPerf compares the same workloads with sessions off and on. With +// DTS_STRESS_RESTART set, every run starts on a fresh backend. +func TestDTSPerf(t *testing.T) { + emulatorOptions(t) + repeat := max(1, int(envFloat("DTS_STRESS_REPEAT", 3))) + const header = "| workload | metric | sessions off | sessions on | change |\n|---|---|---|---|---|\n" + var table strings.Builder + for _, workload := range perfWorkloads() { + t.Run(workload.name, func(t *testing.T) { + results := map[bool][]perfResult{} + for i := range repeat { + // Alternate the order, so warm-up does not favor one mode. + for _, sessions := range []bool{i%2 == 1, i%2 == 0} { + require.NoError(t, restartBackend(t)) + result := runPerf(t, workload, sessions) + t.Logf("sessions=%v wall=%v acks=%d unary=%d replays=%d resumes=%d", + sessions, result.wall, result.acks, result.unary, result.replays, result.resumes) + results[sessions] = append(results[sessions], result) + } + } + rows := formatPerfRows(workload, results[false], results[true]) + t.Logf("median of %d runs:\n%s%s", repeat, header, rows) + table.WriteString(rows) + }) + } + t.Logf("median of %d runs:\n%s%s", repeat, header, table.String()) +} + +// ---------------------------------------------------------------------------- +// Stress + +type stressConfig struct { + instances int + entities int + signalsPerEntity int + orchestrations int + activities int + entityConcurrency int + completionConnections int + faultRate float64 + workerRestarts int + backendRestart bool +} + +type stressBatch struct { + prefix string + ids []api.InstanceID + entities int + expected map[int]int +} + +func stressSessionsOn() bool { + return os.Getenv("DTS_STRESS_SESSIONS") != "off" +} + +// scheduleStressBatch starts the instances, raises their events, and sends +// the entity signals. +func scheduleStressBatch(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client, config stressConfig) *stressBatch { + t.Helper() + batch := &stressBatch{ + prefix: "stress-" + uuid.NewString()[:8], + ids: make([]api.InstanceID, config.instances), + entities: config.entities, + expected: make(map[int]int), + } + for i := range config.instances { + if i%6 == 5 { + batch.expected[i%config.entities]++ + } + } + for e := range config.entities { + batch.expected[e] += config.signalsPerEntity + } + runParallel(config.instances, 32, func(i int) { + id := api.InstanceID(fmt.Sprintf("%s-%d", batch.prefix, i)) + _, err := client.ScheduleNewOrchestration(ctx, "StressMixed", api.WithInstanceID(id), + api.WithInput(stressInput{Index: i, Entities: config.entities, Prefix: batch.prefix})) + assert.NoError(t, err) + batch.ids[i] = id + if i%6 == 2 { + for range stressEventsPerInstance { + assert.NoError(t, client.RaiseEvent(ctx, id, "e", api.WithEventPayload(i))) + } + } + }) + runParallel(config.entities*config.signalsPerEntity, 32, func(n int) { + assert.NoError(t, client.SignalEntity(ctx, stressEntityID(batch.prefix, n%config.entities), "add", api.WithSignalInput(1))) + }) + require.False(t, t.Failed(), "scheduling failed") + return batch +} + +// verifyStressBatch waits for every instance and entity and returns the +// problems it found. completed counts the finished instances. +func verifyStressBatch(ctx context.Context, client *durabletaskscheduler.Client, batch *stressBatch, completed *atomic.Int64) []string { + var mu sync.Mutex + var problems []string + report := func(format string, args ...any) { + mu.Lock() + problems = append(problems, fmt.Sprintf(format, args...)) + mu.Unlock() + } + runParallel(len(batch.ids), 32, func(i int) { + for { + metadata, err := client.WaitForOrchestrationCompletion(ctx, batch.ids[i], api.WithFetchPayloads(true)) + if err != nil { + if ctx.Err() != nil { + status := "unknown" + lookupCtx, cancelLookup := context.WithTimeout(context.Background(), 10*time.Second) + if latest, fetchErr := client.FetchOrchestrationMetadata(lookupCtx, batch.ids[i]); fetchErr == nil && latest != nil { + status = latest.RuntimeStatus.String() + } + cancelLookup() + report("%s: not finished (%s): %v", batch.ids[i], status, err) + return + } + // The backend may be restarting. + time.Sleep(time.Second) + continue + } + completed.Add(1) + var got int + if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { + report("%s: status %s: %v", batch.ids[i], metadata.RuntimeStatus, metadata.FailureDetails) + } else if err := metadata.ReadOutput(&got); err != nil || got != stressExpected(i) { + report("%s: output %d, want %d (%v)", batch.ids[i], got, stressExpected(i), err) + } + return + } + }) + // Signals can still be running after the last instance completes. + entityDeadline := time.Now().Add(3 * time.Minute) + for e := range batch.entities { + entityID := stressEntityID(batch.prefix, e) + want := strconv.Itoa(batch.expected[e]) + got := "" + for ctx.Err() == nil && time.Now().Before(entityDeadline) { + metadata, err := client.GetEntity(ctx, entityID) + if err == nil && metadata != nil { + got = metadata.SerializedState + } + if got == want { + break + } + time.Sleep(50 * time.Millisecond) + } + if got != want { + report("%s: state %q, want %s", entityID, got, want) + } + } + return problems +} + +func runStress(t *testing.T, config stressConfig) { + sessionsOn := stressSessionsOn() + if config.backendRestart && os.Getenv("DTS_STRESS_RESTART") == "" { + t.Skip("set DTS_STRESS_RESTART to run the backend-restart test") + } + baseline := runtime.NumGoroutine() + idleTimeout := 3 * time.Second + var sessions *durabletaskclient.ExtendedSessionOptions + if sessionsOn { + sessions = &durabletaskclient.ExtendedSessionOptions{IdleTimeout: idleTimeout} + } + probe := newStressProbe(config.faultRate, idleTimeout) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(config.orchestrations), + durabletaskclient.WithMaxConcurrentActivityWorkItems(config.activities), + durabletaskclient.WithMaxConcurrentEntityWorkItems(config.entityConcurrency), + } + if config.completionConnections > 0 { + workerOptions = append(workerOptions, durabletaskclient.WithWorkerCompletionConnections(config.completionConnections)) + } + h := newStressHarness(t, stressRegistry(t), probe, sessions, workerOptions...) + closed := false + defer func() { + if !closed { + h.close() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + start := time.Now() + batch := scheduleStressBatch(t, ctx, h.client, config) + t.Logf("scheduled %d instances and %d signals in %v", config.instances, config.entities*config.signalsPerEntity, time.Since(start)) + + var completed atomic.Int64 + done := make(chan struct{}) + var background sync.WaitGroup + var stopOnce sync.Once + stopBackground := func() { + stopOnce.Do(func() { close(done) }) + background.Wait() + } + // A failed check exits through runtime.Goexit, so stop the restarts then too. + defer stopBackground() + // waitProgress waits until the given number of instances completed. It + // returns false when the run ends first. + waitProgress := func(target int64) bool { + for completed.Load() < target { + select { + case <-done: + return false + case <-time.After(10 * time.Millisecond): + } + } + return true + } + if config.workerRestarts > 0 { + background.Go(func() { + for n := 1; n <= config.workerRestarts; n++ { + if !waitProgress(int64(config.instances) * int64(n) / int64(config.workerRestarts+1)) { + return + } + h.restartWorker() + } + }) + } + persistent := os.Getenv("DTS_STRESS_PERSISTENT") == "1" + restartAt := int64(config.instances) * 3 / 10 + if config.backendRestart { + background.Go(func() { + if !waitProgress(restartAt) { + return + } + t.Logf("restarting the backend after %d of %d instances", completed.Load(), config.instances) + assert.NoError(t, restartBackend(t)) + t.Logf("the backend is back after %v", time.Since(start)) + }) + } + var problems []string + if config.backendRestart && !persistent { + // All state is lost on restart. Check the instances that finish + // before it, then check that new work completes after it. + for i := range restartAt { + metadata, err := h.client.WaitForOrchestrationCompletion(ctx, batch.ids[i], api.WithFetchPayloads(true)) + require.NoError(t, err) + var got int + if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { + problems = append(problems, fmt.Sprintf("%s: status %s: %v", batch.ids[i], metadata.RuntimeStatus, metadata.FailureDetails)) + } else if err := metadata.ReadOutput(&got); err != nil || got != stressExpected(int(i)) { + problems = append(problems, fmt.Sprintf("%s: output %d, want %d (%v)", batch.ids[i], got, stressExpected(int(i)), err)) + } + completed.Add(1) + } + background.Wait() + fresh := config + fresh.instances, fresh.entities, fresh.signalsPerEntity = scaled(120), scaled(12), 5 + freshBatch := scheduleStressBatch(t, ctx, h.client, fresh) + problems = append(problems, verifyStressBatch(ctx, h.client, freshBatch, new(atomic.Int64))...) + } else { + problems = verifyStressBatch(ctx, h.client, batch, &completed) + } + stopBackground() + elapsed := time.Since(start) + + t.Logf("finished in %v: worker restarts=%d acks=%d unary=%d replays=%d resumes=%d releases=%d abandons=%d notFound=%d maxOpenStreams=%d", + elapsed, h.restarts, probe.acks.Load(), probe.unary.Load(), probe.replays.Load(), probe.resumes.Load(), + probe.releases.Load(), probe.abandons.Load(), probe.notFound.Load(), probe.maxOpen.Load()) + for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { + if n := probe.faults[kind].Load(); n > 0 { + t.Logf("injected %q: %d", stressFaultNames[kind], n) + } + } + for _, problem := range problems[:min(len(problems), 20)] { + t.Error(problem) + } + require.Empty(t, problems, "%d problems", len(problems)) + + if sessionsOn { + require.Positive(t, probe.acks.Load(), "sessions were not used") + connections := config.completionConnections + if connections == 0 { + connections = 3 + } + require.LessOrEqual(t, probe.maxOpen.Load(), int64(50*connections), "a connection exceeded its session stream budget") + // Idle sessions end after the idle timeout and return their streams. + require.Eventually(t, func() bool { return probe.open.Load() == 0 }, idleTimeout+30*time.Second, 50*time.Millisecond, + "%d session streams are still open", probe.open.Load()) + } else { + require.Zero(t, probe.acks.Load(), "sessions were used while off") + } + h.close() + closed = true + require.Zero(t, probe.open.Load(), "session streams are open after shutdown") + // Poll here: assert.Eventually runs its condition in another goroutine. + for deadline := time.Now().Add(20 * time.Second); runtime.NumGoroutine() > baseline && time.Now().Before(deadline); { + time.Sleep(100 * time.Millisecond) + } + if runtime.NumGoroutine() > baseline { + var stacks strings.Builder + _ = pprof.Lookup("goroutine").WriteTo(&stacks, 1) + t.Fatalf("goroutines: %d after shutdown, %d before:\n%s", runtime.NumGoroutine(), baseline, stacks.String()) + } + if probe.faultCount() == 0 && h.restarts == 0 && !config.backendRestart { + assert.Zero(t, probe.abandons.Load(), "unexpected abandons") + assert.Zero(t, probe.notFound.Load(), "unexpected NotFound errors") + } + if config.faultRate > 0 && sessionsOn && !config.backendRestart { + // Each stream gets each fault kind with a chance of faultRate/5. With + // 10 expected faults of each kind, a missing kind is a real problem, + // not bad luck (the default rate of 0.1 needs 500 turns). + if float64(probe.acks.Load())*config.faultRate/float64(stressFaultKinds-1) >= 10 { + for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { + applied := probe.faults[kind].Load() + if kind == faultDelayAfterAck { + applied = probe.delayHolds.Load() + } + assert.Positive(t, applied, "no %q fault was applied", stressFaultNames[kind]) + } + } + // Upper bounds that faults and restarts explain: + // - Only a failure before the acknowledgement abandons a session work + // item: one abandon per cut-before-send or drop-ack fault at most. + // - A worker restart can strand at most every in-flight work item: + // one abandon each, plus a NotFound for its completion and one for + // its abandon or release. + // - Every other fault ends one stream, with at most one NotFound from + // the retry or the release that follows. + inFlight := int64(h.restarts) * int64(config.orchestrations+config.activities+config.entityConcurrency) + preAck := probe.faults[faultCutBeforeSend].Load() + probe.faults[faultDropAck].Load() + assert.LessOrEqual(t, probe.abandons.Load(), preAck+inFlight, "more abandons than faults and restarts explain") + assert.LessOrEqual(t, probe.notFound.Load(), probe.faultCount()+2*inFlight, "more NotFound errors than faults and restarts explain") + } +} + +// TestDTSStress checks correctness under load and faults. Every subtest +// checks results, entity values, open session streams, and goroutines. +func TestDTSStress(t *testing.T) { + emulatorOptions(t) + instances, entities := scaled(2000), scaled(200) + t.Run("load", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 16, activities: 32, entityConcurrency: 16, + }) + }) + t.Run("one-completion-connection", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 160, activities: 64, entityConcurrency: 120, completionConnections: 1, + }) + }) + t.Run("chaos", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 32, activities: 32, entityConcurrency: 16, + faultRate: envFloat("DTS_STRESS_FAULT_RATE", 0.1), workerRestarts: 3, + }) + }) + t.Run("backend-restart", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 32, activities: 32, entityConcurrency: 16, + faultRate: envFloat("DTS_STRESS_FAULT_RATE", 0.1) / 2, backendRestart: true, + }) + }) +}