Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
03269a3
Add executor support for extended orchestration sessions
torosent Nov 6, 2026
ce4b7af
Drop processed history while an orchestration session waits
torosent Nov 7, 2026
7e90ee2
Route extended-session streams to completion connections
torosent Nov 7, 2026
5671231
Add extended sessions to the gRPC worker
torosent Nov 7, 2026
e3e04f2
Add DTS emulator conformance tests for extended sessions
torosent Nov 7, 2026
c7c4ade
Bound abandons in later session turns by the cleanup deadline
torosent Nov 7, 2026
686e2f4
Fix extended-session review findings in the worker
torosent Nov 7, 2026
d73a16a
Deliver the first chunk before cutting the E2E session batch
torosent Nov 7, 2026
3b2cb8f
Fail session E2E tests when the session probe stops early
torosent Nov 7, 2026
ce92a2d
Add an extended-session stress and performance harness
torosent Nov 7, 2026
9e78e10
Trace work items in the stress harness and use the default silent tim…
torosent Nov 7, 2026
14d0026
Tighten the stress harness checks
torosent Nov 7, 2026
f2ee933
Count delay faults from the acknowledgement in the stress harness
torosent Nov 7, 2026
a17cb3e
Document that a waiting extended session keeps its orchestration slot
torosent Nov 7, 2026
0c344d7
Fix a timer race in the graceful-stop shutdown-deadline test
torosent Nov 7, 2026
90bab64
Bound session stream opens and batches, and fix review findings
torosent Nov 7, 2026
97de25c
Drop finished coroutines and sort pending action IDs between session …
torosent Nov 7, 2026
5e3380f
Make stress harness cleanup and gates safe on failures and low fault …
torosent Nov 7, 2026
b9ceaf7
End a session after orchestrator code reads turn-wide history values
torosent Nov 7, 2026
9209c6e
Stop session retries and cleanup when the intake stream ends
torosent Nov 7, 2026
1bec35e
Make session E2E and stress readiness checks robust to timing
torosent Nov 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- Added opt-in extended sessions to gRPC workers with `WithExtendedSessions`. A session keeps an orchestration or entity loaded between turns, so DTS sends only the new messages and the worker skips the full replay. Sessions are off by default. A waiting session keeps its orchestration slot, the same as in Durable Functions, so raise `WithMaxConcurrentOrchestrationWorkItems` for slow backends or many short orchestrations. When the service returns `Unimplemented`, the worker completes work items without sessions until it reconnects.
- Added required completion-transport isolation to gRPC workers: one intake and three dedicated completion/abandon connections by default, with a configurable 1–8 completion budget and bounded generation-safe retirement. `WithWorkerCompletionConnections(0)` is invalid. Borrowed workers and client listeners require separate caller-owned channels via `WithWorkerCompletionTransports`; owning factories must return distinct connections and non-nil closers for every invocation.
- Added `ResourceId` connection-string support for every DTS authentication mode and optional `AuthorityHost` configuration for SDK-created DefaultAzure, WorkloadIdentity, Environment, and InteractiveBrowser credentials. Caller-supplied credentials own their authority; managed identity and developer-tool cloud configuration remain separate.
- Added a flat catalogue of directly runnable DTS samples with outcome assertions, owned-resource cleanup, and process-level E2E validation selected by sample name using Go's `-run` flag.
Expand Down
29 changes: 28 additions & 1 deletion client/grpc_worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"time"

"github.com/microsoft/durabletask-go/api"
Expand Down Expand Up @@ -118,6 +119,10 @@ type taskHubGrpcWorkerOptions struct {
unversionedOrchestrators map[string]struct{}
unversionedActivities map[string]struct{}
reconnectRandom randomInt64N
extendedSessions *ExtendedSessionOptions
sessionCleanupTimeout time.Duration
// sessionTimerFn replaces the session timers in tests.
sessionTimerFn func(sessionTimerKind, time.Duration) <-chan time.Time
// waitFn overrides every delay the worker imposes on itself: reconnect
// backoff, transient RPC retry backoff, and work-item abandon delays. It is
// only set by tests so the deterministic delay schedule can be observed
Expand All @@ -144,6 +149,7 @@ func defaultTaskHubGrpcWorkerOptions() taskHubGrpcWorkerOptions {
maxStreamedHistoryBytes: DefaultMaxStreamedHistoryBytes,
maxOrchestratorCompletionBytes: DefaultMaxOrchestratorCompletionBytes,
reconnectRandom: rand.Int64N,
sessionCleanupTimeout: defaultSessionCleanupTimeout,
capabilities: []WorkerCapability{WorkerCapabilityHistoryStreaming},
}
}
Expand Down Expand Up @@ -537,6 +543,7 @@ type grpcWorkerRun struct {
orchestrationSlots chan struct{}
activitySlots chan struct{}
entitySlots chan struct{}
sessions *sessionManager
pending sync.WaitGroup
retired sync.WaitGroup
err error
Expand All @@ -547,6 +554,11 @@ type grpcWorkerConnection struct {
stream workItemsStream
cancelStream context.CancelFunc
closer io.Closer
// streamCtx ends with this intake generation. Sessions dispatched by the
// generation end with it.
streamCtx context.Context
sessionsUnsupported atomic.Bool
sessionCleanupFailed atomic.Bool

pendingMu sync.Mutex
pendingChanged *sync.Cond
Expand Down Expand Up @@ -590,6 +602,14 @@ func (c *grpcWorkerConnection) waitForWorkItems() {
c.waitForWorkItemsLocked()
}

// abortForSessionCleanup ends the intake stream after a session work item
// could not be released or abandoned. The service then abandons every work
// item of this generation, and the worker reconnects.
func (c *grpcWorkerConnection) abortForSessionCleanup() {
c.sessionCleanupFailed.Store(true)
c.cancelStream()
}

func (c *grpcWorkerConnection) stopIntake() {
c.pendingMu.Lock()
defer c.pendingMu.Unlock()
Expand Down Expand Up @@ -707,6 +727,9 @@ func newTaskHubGrpcWorker(
return nil, err
}
}
if err := options.resolveExtendedSessionLimits(); err != nil {
return nil, err
}
hasLargePayloadCapability := slices.Contains(options.capabilities, WorkerCapabilityLargePayloads)
if hasLargePayloadCapability && options.largePayloads == nil {
return nil, fmt.Errorf("large-payload capability requires worker large-payload options")
Expand Down Expand Up @@ -1067,6 +1090,9 @@ func (w *TaskHubGrpcWorker) begin(ctx context.Context) (*grpcWorkerRun, *grpcWor
activitySlots: make(chan struct{}, w.options.maxConcurrentActivities),
entitySlots: make(chan struct{}, w.options.maxConcurrentEntities),
}
if w.options.extendedSessions != nil {
run.sessions = newSessionManager(w.options.extendedSessions)
}
w.run = run
w.lastErr = nil
w.mu.Unlock()
Expand Down Expand Up @@ -1236,6 +1262,7 @@ func (w *TaskHubGrpcWorker) connect(ctx context.Context) (*grpcWorkerConnection,
stream: stream,
cancelStream: cancelStream,
closer: closer,
streamCtx: streamCtx,
}, nil
}

Expand Down Expand Up @@ -1415,7 +1442,7 @@ func isTransientWorkerError(err error) bool {
if errors.Is(err, ErrStreamedHistoryLimitExceeded) {
return false
}
if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) {
if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) || errors.Is(err, errSessionCleanupFailed) {
return true
}
grpcStatus, ok := status.FromError(err)
Expand Down
85 changes: 82 additions & 3 deletions client/grpc_worker_connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,16 @@ import (

const maxWorkerCompletionConnections = 8

// workerSessionStreamsPerConnection is the extended-session stream budget of
// one completion connection: half of the 100 HTTP/2 streams per connection
// that DTS documents, so other completions always have room.
const workerSessionStreamsPerConnection = 50

// errSessionStreamBudgetFull means that no completion connection can open
// another extended-session stream. Nothing was sent, so the caller completes
// the work item with the unary RPC instead.
var errSessionStreamBudgetFull = errors.New("worker extended-session stream budget is full")

func validateWorkerCompletionBudget(n int) error {
if n < 1 || n > maxWorkerCompletionConnections {
return fmt.Errorf("worker completion connections must be between 1 and %d", maxWorkerCompletionConnections)
Expand Down Expand Up @@ -67,7 +77,10 @@ type workerCompletionTransport struct {
closers []io.Closer
next atomic.Uint64
closeOnce sync.Once
closeErr error

sessionMu sync.Mutex
sessionStreams []int
closeErr error
}

func newWorkerCompletionTransport(
Expand Down Expand Up @@ -158,7 +171,9 @@ func isWorkerCompletionMethod(method string) bool {
protos.TaskHubSidecarService_CompleteEntityTask_FullMethodName,
protos.TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName,
protos.TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName,
protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName:
protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName,
protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName,
protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName:
return true
default:
return false
Expand All @@ -171,7 +186,71 @@ func (c *workerCompletionTransport) NewStream(
method string,
opts ...grpc.CallOption,
) (grpc.ClientStream, error) {
return c.intake.NewStream(ctx, desc, method, opts...)
if !isWorkerSessionStreamMethod(method) {
return c.intake.NewStream(ctx, desc, method, opts...)
}
index, ok := c.reserveSessionStream()
if !ok {
return nil, errSessionStreamBudgetFull
}
stream, err := c.completions[index].NewStream(ctx, desc, method, opts...)
if err != nil {
c.releaseSessionStream(index)
return nil, err
}
budgeted := &budgetedClientStream{ClientStream: stream}
var once sync.Once
budgeted.release = func() { once.Do(func() { c.releaseSessionStream(index) }) }
// Callers cancel the stream context when they finish with the stream.
context.AfterFunc(ctx, budgeted.release)
return budgeted, nil
}

func isWorkerSessionStreamMethod(method string) bool {
return method == protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName ||
method == protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName
}

// reserveSessionStream picks the least-loaded completion connection that
// still has session stream budget.
func (c *workerCompletionTransport) reserveSessionStream() (int, bool) {
c.sessionMu.Lock()
defer c.sessionMu.Unlock()
if c.sessionStreams == nil {
c.sessionStreams = make([]int, len(c.completions))
}
best := -1
for i, streams := range c.sessionStreams {
if streams < workerSessionStreamsPerConnection && (best < 0 || streams < c.sessionStreams[best]) {
best = i
}
}
if best < 0 {
return 0, false
}
c.sessionStreams[best]++
return best, true
}

func (c *workerCompletionTransport) releaseSessionStream(index int) {
c.sessionMu.Lock()
c.sessionStreams[index]--
c.sessionMu.Unlock()
}

// budgetedClientStream returns its session stream budget when the stream
// ends with any status or when its context ends.
type budgetedClientStream struct {
grpc.ClientStream
release func()
}

func (s *budgetedClientStream) RecvMsg(m any) error {
err := s.ClientStream.RecvMsg(m)
if err != nil {
s.release()
}
return err
}

func (c *workerCompletionTransport) Close() error {
Expand Down
4 changes: 3 additions & 1 deletion client/grpc_worker_drain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,9 @@ func TestWorkerGracefulStopKeepsRejectedItemLeaseUntilAbandoned(t *testing.T) {
require.True(t, worker.Running())

if stop == "shutdown deadline" {
shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), time.Second)
// Stay below the 1 s RPC timeout of the blocked abandon. With equal
// timeouts, synctest fires both at once, and the run can end first.
shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer stopShutdown()
require.ErrorIs(t, worker.Shutdown(shutdownCtx), context.DeadlineExceeded)
} else {
Expand Down
Loading
Loading