From df4312f00297d70cfa436e1bb8ed584aa1ac9c4b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Nov 2026 16:45:11 -0800 Subject: [PATCH 1/6] Fix failed-message buffer cleanup in Azure Storage Release failed dequeue and deduplication buffer slots, retry storage timeouts, and abandon sessions that never handed off a work item without disturbing deferred or replacement-session ownership. Add focused regressions for these failure paths. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AzureStorageOrchestrationService.cs | 21 ++- .../Messaging/ControlQueue.cs | 13 +- .../OrchestrationSessionManager.cs | 26 +++- .../AzureStorageScenarioTests.cs | 132 ++++++++++++++++++ .../MessageManagerTests.cs | 94 +++++++++++++ .../OrchestrationStatePrefetchTests.cs | 35 +++++ .../TestHelpers.cs | 43 ++++++ 7 files changed, 354 insertions(+), 10 deletions(-) diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 54228ff3c..dc6150ee3 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -901,10 +901,12 @@ await this.AbandonAndReleaseSessionAsync( session?.Instance.ExecutionId ?? string.Empty, e.ToString()); - if (orchestrationWorkItem != null) + if (session != null) { - // The work-item needs to be released so that it can be retried later. - await this.ReleaseTaskOrchestrationWorkItemAsync(orchestrationWorkItem); + // No work item was handed off. Release the session and its messages. + await this.AbandonAndReleaseSessionAsync( + session, + "Failed to prepare the orchestration work item."); } throw; @@ -1144,6 +1146,19 @@ await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( async Task AbandonAndReleaseSessionAsync(OrchestrationSession session, string details) { + // An earlier cleanup may already have released this session. + if (!this.orchestrationSessionManager.TryGetExistingSession(session.Instance.InstanceId, out OrchestrationSession currentSession) || + !ReferenceEquals(session, currentSession)) + { + return; + } + + // Deferred messages stay buffered for the next session. + foreach (MessageData message in session.DeferredMessages) + { + session.CurrentMessageBatch.Remove(message); + } + try { await this.AbandonSessionAsync(session, details); diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index 1c56f7fa9..b4072bfb6 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -75,6 +75,7 @@ public async Task> GetMessagesAsync(CancellationToken } pendingOrchestratorMessageLimitReached = false; + var addedMessageIds = new ConcurrentBag(); try { @@ -134,7 +135,11 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } // Check to see whether we've already dequeued this message. - if (!this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1)) + if (this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1)) + { + addedMessageIds.Add(queueMessage.MessageId); + } + else { // This message is already loaded in memory and is therefore a duplicate. // We will continue to process it because we need the updated pop receipt. @@ -170,6 +175,12 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } catch (Exception e) { + // This batch was not handed off. Keep entries from earlier batches. + foreach (string messageId in addedMessageIds) + { + this.stats.PendingOrchestratorMessages.TryRemove(messageId, out _); + } + if (!linkedCts.IsCancellationRequested) { this.settings.Logger.MessageFailure( diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index 9f0105353..4ed10a915 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -151,11 +151,25 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati if (messages.Count > 0) { // De-dupe any execution started messages - IEnumerable filteredMessages = await this.DedupeExecutionStartedMessagesAsync( - controlQueue, - messages, - traceActivityId, - cancellationToken); + IEnumerable filteredMessages; + try + { + filteredMessages = await this.DedupeExecutionStartedMessagesAsync( + controlQueue, + messages, + traceActivityId, + cancellationToken); + } + catch + { + // These messages were not handed off. Release their buffer slots. + foreach (MessageData message in messages) + { + this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.MessageId, out _); + } + + throw; + } this.AddMessageToPendingOrchestration(controlQueue, filteredMessages, traceActivityId, cancellationToken); } @@ -566,7 +580,7 @@ async Task ScheduleOrchestrationStatePrefetch( this.orchestrationsReadyForProcessingQueue.Enqueue(node); } } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { // shutting down } diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 90acbe99e..1dd58c610 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -13,15 +13,19 @@ namespace DurableTask.AzureStorage.Tests { + using Azure; using Azure.Data.Tables; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; + using Azure.Storage.Queues.Models; + using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Settings; + using DurableTask.Core.Tracking; using Microsoft.Practices.EnterpriseLibrary.SemanticLogging.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -4467,6 +4471,134 @@ await service.SendTaskOrchestrationMessageAsync( } } + [TestMethod] + public async Task SessionCleanup_PreservesDeferredMessagesAndReplacementSession() + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + var instanceStore = new Mock(MockBehavior.Strict); + instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); + using var queue = service.AllControlQueues.Single(); + var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; + var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); + string payload = await messageManager.SerializeMessageDataAsync(new MessageData( + new TaskMessage { OrchestrationInstance = instance, Event = new EventRaisedEvent(-1, "event") }, + Guid.NewGuid(), queue.Name, null, new OrchestrationInstance())); + DateTimeOffset now = DateTimeOffset.UtcNow; + queueClient.Setup(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(new[] + { + QueuesModelFactory.QueueMessage("current", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + QueuesModelFactory.QueueMessage("deferred", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + }, Mock.Of())); + IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); + manager.AddMessageToPendingOrchestration(queue, messages, Guid.NewGuid(), shutdown.Token); + OrchestrationSession session = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); + MessageData deferredMessage = messages.Single(message => message.OriginalQueueMessage.MessageId == "deferred"); + session.DeferMessage(deferredMessage); + MethodInfo cleanupMethod = typeof(AzureStorageOrchestrationService).GetMethod( + "AbandonAndReleaseSessionAsync", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.IsNotNull(cleanupMethod); + + await (Task)cleanupMethod.Invoke(service, new object[] { session, "Simulated preparation failure." }); + OrchestrationSession replacement = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); + + Assert.AreSame(deferredMessage, replacement.CurrentMessageBatch.Single()); + manager.GetStats(out _, out int bufferedMessages, out _); + Assert.AreEqual(1, bufferedMessages, "The deferred message must keep its buffer slot."); + queueClient.Verify(client => client.UpdateMessageAsync( + "current", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + queueClient.Verify(client => client.UpdateMessageAsync( + "deferred", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + + await (Task)cleanupMethod.Invoke(service, new object[] { session, "Repeated cleanup of the old session." }); + + Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession currentSession)); + Assert.AreSame(replacement, currentSession, "Cleanup of a released session must not release its replacement."); + } + + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task WorkItemPreparationFailure_AbandonsMessagesAndReleasesSession(bool failAfterCreatingWorkItem) + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + var instanceStore = new Mock(MockBehavior.Strict); + instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + instanceStore.Setup(store => store.GetOrchestrationStateAsync(It.IsAny(), false)) + .ThrowsAsync(new InvalidOperationException("Simulated work item validation failure.")); + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); + using var queue = service.AllControlQueues.Single(); + var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; + var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); + var taskMessage = new TaskMessage + { + OrchestrationInstance = instance, + Event = failAfterCreatingWorkItem + ? new ExecutionTerminatedEvent(-1, "terminate") + : new EventRaisedEvent(-1, "event"), + }; + string payload = await messageManager.SerializeMessageDataAsync(new MessageData( + taskMessage, Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); + if (!failAfterCreatingWorkItem) + { + JObject envelope = JObject.Parse(payload); + envelope.Remove(nameof(MessageData.Sender)); + payload = envelope.ToString(Formatting.None); + } + + DateTimeOffset now = DateTimeOffset.UtcNow; + QueueMessage queueMessage = QueuesModelFactory.QueueMessage( + "failed", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)); + queueClient.Setup(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(new[] { queueMessage }, Mock.Of())); + IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); + manager.AddMessageToPendingOrchestration(queue, messages, Guid.NewGuid(), shutdown.Token); + + if (failAfterCreatingWorkItem) + { + await Assert.ThrowsExceptionAsync( + () => service.LockNextTaskOrchestrationWorkItemAsync(TimeSpan.FromSeconds(5), shutdown.Token)); + } + else + { + // Missing sender metadata faults before a work item is constructed. + await Assert.ThrowsExceptionAsync( + () => service.LockNextTaskOrchestrationWorkItemAsync(TimeSpan.FromSeconds(5), shutdown.Token)); + } + + manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); + Assert.AreEqual(0, activeSessions, "No session may remain owned when no work item was handed off."); + Assert.AreEqual(0, bufferedMessages, "Failed messages must release their buffer slots."); + Assert.AreEqual(0, pendingInstances); + queueClient.Verify(client => client.UpdateMessageAsync( + "failed", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + static AzureStorageOrchestrationService CreateServiceForSessionFailureTest( + AzureStorageOrchestrationServiceSettings settings, + IOrchestrationServiceInstanceStore instanceStore, + CancellationTokenSource shutdown, + out OrchestrationSessionManager manager) + { + var service = new AzureStorageOrchestrationService(settings, instanceStore); + const BindingFlags Flags = BindingFlags.Instance | BindingFlags.NonPublic; + manager = (OrchestrationSessionManager)typeof(AzureStorageOrchestrationService) + .GetField("orchestrationSessionManager", Flags).GetValue(service); + typeof(AzureStorageOrchestrationService).GetField("shutdownSource", Flags).SetValue(service, shutdown); + + // Skip task hub creation and listener startup; all queue I/O in these tests is mocked. + object creator = typeof(AzureStorageOrchestrationService).GetField("taskHubCreator", Flags).GetValue(service); + creator.GetType().GetField("lazy", Flags).SetValue(creator, new Lazy(() => Task.CompletedTask)); + return service; + } + [TestMethod] public async Task WorkerDoesNotDiscardExternalEventBatchedWithMessageForMissingExecution() { diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index 855f68d62..ea1ab93f4 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -13,9 +13,13 @@ #nullable enable namespace DurableTask.AzureStorage.Tests { + using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; + using Azure.Storage.Queues.Models; + using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Storage; + using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.History; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -25,7 +29,9 @@ namespace DurableTask.AzureStorage.Tests using System; using System.Collections.Generic; using System.Linq; + using System.Reflection; using System.Text; + using System.Threading; using System.Threading.Tasks; [TestClass] @@ -176,6 +182,94 @@ public async Task SerializeMessageDataAsync_UsesExpectedStoragePathAtThreshold( } } + [TestMethod] + public async Task ControlQueue_FailedBatch_ReleasesOnlyNewBufferEntries() + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + settings.ControlQueueBufferThreshold = 3; + var client = new AzureStorageClient(settings); + var messageManager = new MessageManager(settings, client, "buffer-largemessages"); + using var queue = new ControlQueue(client, queueClient.Object.Name, messageManager); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + string payload = await messageManager.SerializeMessageDataAsync(CreateMessageData("valid")); + MessageData malformedMessage = CreateMessageData("malformed"); + malformedMessage.TaskMessage.Event = null; + string malformedPayload = await messageManager.SerializeMessageDataAsync(malformedMessage); + DateTimeOffset now = DateTimeOffset.UtcNow; + QueueMessage[] batch = + { + QueuesModelFactory.QueueMessage("earlier", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + QueuesModelFactory.QueueMessage("new", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + QueuesModelFactory.QueueMessage("malformed", "receipt", malformedPayload, 1, now, now.AddDays(1), now.AddMinutes(5)), + }; + client.Stats.PendingOrchestratorMessages.TryAdd("earlier", 1); + int receiveCalls = 0; + queueClient.Setup(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + if (++receiveCalls == 1) + { + return Task.FromResult(Response.FromValue(batch, Mock.Of())); + } + + timeout.Cancel(); + return Task.FromResult(Response.FromValue(Array.Empty(), Mock.Of())); + }); + + IReadOnlyList messages = await queue.GetMessagesAsync(timeout.Token); + + Assert.AreEqual(0, messages.Count, "A failed batch must not be handed off."); + CollectionAssert.AreEquivalent(new[] { "earlier" }, client.Stats.PendingOrchestratorMessages.Keys.ToArray()); + Assert.AreEqual(2, receiveCalls, "The failed batch must not exhaust the buffer and block the next receive."); + } + + [TestMethod] + public async Task DequeueLoop_DeduplicationFailure_ReleasesUnhandedBufferEntries() + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + var client = new AzureStorageClient(settings); + var messageManager = new MessageManager(settings, client, "buffer-largemessages"); + using var queue = new ControlQueue(client, queueClient.Object.Name, messageManager); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + MessageData message = CreateMessageData("start"); + message.TaskMessage.Event = new ExecutionStartedEvent(-1, string.Empty) + { + Name = "orchestration", + OrchestrationInstance = message.TaskMessage.OrchestrationInstance, + }; + DateTimeOffset now = DateTimeOffset.UtcNow; + QueueMessage queueMessage = QueuesModelFactory.QueueMessage( + "start", "receipt", await messageManager.SerializeMessageDataAsync(message), 1, + now, now.AddDays(1), now.AddMinutes(5)); + queueClient.Setup(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(new[] { queueMessage }, Mock.Of())); + client.Stats.PendingOrchestratorMessages.TryAdd("earlier", 1); + + var trackingStore = new Mock(MockBehavior.Strict); + trackingStore.Setup(store => store.FetchInstanceStatusAsync( + It.IsAny>(), It.IsAny())) + .Callback(() => queue.Release(null, "Stop after the simulated deduplication failure.")) + .Throws(new InvalidOperationException("Simulated instance status lookup failure.")); + using var manager = new OrchestrationSessionManager( + client.QueueAccountName, settings, client.Stats, trackingStore.Object); + MethodInfo dequeueLoop = typeof(OrchestrationSessionManager).GetMethod( + "DequeueLoop", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new MissingMethodException("DequeueLoop"); + + await (Task)(dequeueLoop.Invoke(manager, new object[] { queue.Name, queue, timeout.Token }) + ?? throw new InvalidOperationException("The dequeue loop did not return a task.")); + + CollectionAssert.AreEquivalent(new[] { "earlier" }, client.Stats.PendingOrchestratorMessages.Keys.ToArray()); + manager.GetStats(out int pendingInstances, out _, out int activeSessions); + Assert.AreEqual(0, pendingInstances); + Assert.AreEqual(0, activeSessions); + trackingStore.Verify(store => store.FetchInstanceStatusAsync( + It.IsAny>(), It.IsAny()), Times.Once); + } + [DataTestMethod] [DataRow("blob.bin", "blob.bin")] [DataRow("@#$%!", "%40%23%24%25%21")] diff --git a/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs b/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs index dbe582b80..6e2100279 100644 --- a/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs +++ b/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs @@ -255,6 +255,41 @@ public async Task PrefetchRetry_HistoryAlreadyLoaded_DoesNotLoadHistoryAgain() Assert.AreSame(history.Events[0], session.RuntimeState.ExecutionStartedEvent); } + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task PrefetchRetry_StorageTimeout_RetriesWithoutLosingBatch(bool timeoutFetchingInstanceStatus) + { + this.settings.UseInstanceTableEtag = timeoutFetchingInstanceStatus; + OrchestrationHistory history = CreateHistory(); + var instanceETag = new ETag("instance-etag"); + var trackingStore = new FakeTrackingStore + { + OnGetHistory = attempt => !timeoutFetchingInstanceStatus && attempt == 1 + ? throw new OperationCanceledException("Simulated history request timeout.") + : Task.FromResult(history), + OnFetchInstanceStatus = attempt => attempt == 1 + ? throw new TaskCanceledException("Simulated instance status request timeout.") + : Task.FromResult(new InstanceStatus(new OrchestrationState(), instanceETag)), + }; + MessageData message = this.CreateMessage(); + using var manager = this.CreateManager(trackingStore); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + manager.AddMessageToPendingOrchestration(this.controlQueue, new[] { message }, Guid.NewGuid(), timeout.Token); + + OrchestrationSession session = await manager.GetNextSessionAsync(entitiesOnly: false, timeout.Token); + + Assert.IsNotNull(session, "A storage timeout must not strand the buffered message."); + Assert.AreEqual(timeoutFetchingInstanceStatus ? 1 : 2, trackingStore.HistoryCalls); + Assert.AreEqual(timeoutFetchingInstanceStatus ? 2 : 0, trackingStore.InstanceStatusCalls); + Assert.AreSame(message, session.CurrentMessageBatch.Single()); + Assert.AreSame(history.Events[0], session.RuntimeState.ExecutionStartedEvent); + if (timeoutFetchingInstanceStatus) + { + Assert.AreEqual(instanceETag, session.ETags.InstanceETag); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] static bool[] GetLiveStatesAfterFullCollection(WeakReference[] references) { diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index 7ed63f64d..e31e01333 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -13,17 +13,60 @@ #nullable enable namespace DurableTask.AzureStorage.Tests { + using Azure; + using Azure.Storage.Queues; + using Azure.Storage.Queues.Models; using DurableTask.Core.Logging; using DurableTask.Core.Settings; using Microsoft.Extensions.Logging; + using Moq; using System; using System.Configuration; using System.Diagnostics; using System.Diagnostics.Tracing; + using System.Threading; using System.Threading.Tasks; static class TestHelpers { + public static AzureStorageOrchestrationServiceSettings GetMockQueueSettings(out Mock queueClient) + { + const string ConnectionString = "UseDevelopmentStorage=true"; + const string TaskHubName = "BufferFailureTests"; + string queueName = AzureStorageOrchestrationService.GetControlQueueName(TaskHubName, 0); + + queueClient = new Mock(MockBehavior.Strict, ConnectionString, queueName); + queueClient.SetupGet(client => client.Name).Returns(queueName); + queueClient.SetupGet(client => client.Uri).Returns(new Uri($"http://127.0.0.1:10001/devstoreaccount1/{queueName}")); + queueClient.Setup(client => client.UpdateMessageAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(Response.FromValue( + QueuesModelFactory.UpdateReceipt("retry-receipt", DateTimeOffset.UtcNow.AddMinutes(1)), + Mock.Of())); + + var serviceClient = new Mock(MockBehavior.Strict, ConnectionString); + serviceClient.SetupGet(client => client.AccountName).Returns("devstoreaccount1"); + serviceClient.Setup(client => client.GetQueueClient(It.IsAny())).Returns(queueClient.Object); + + var queueProvider = new Mock>(); + queueProvider.Setup(provider => provider.CreateOptions()).Returns(() => new QueueClientOptions()); + queueProvider.Setup(provider => provider.CreateClient(It.IsAny())).Returns(serviceClient.Object); + + return new AzureStorageOrchestrationServiceSettings + { + TaskHubName = TaskHubName, + PartitionCount = 1, + StorageAccountClientProvider = new StorageAccountClientProvider( + StorageServiceClientProvider.ForBlob(ConnectionString), + queueProvider.Object, + StorageServiceClientProvider.ForTable(ConnectionString)), + }; + } + public static TestOrchestrationHost GetTestOrchestrationHost( bool enableExtendedSessions, int extendedSessionTimeoutInSeconds = 30, From cc3d449ef1f25f04fef9a5e5a6ed09a5e457b7f1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Nov 2026 21:53:55 -0800 Subject: [PATCH 2/6] Preserve existing buffer reservations when deduplication fails Carry only newly reserved message IDs through execution-start status checks so failures do not remove buffer entries owned by earlier copies of duplicates. Cover duplicates whose original messages remain in pending batches or active sessions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Messaging/ControlQueue.cs | 5 ++- .../OrchestrationSessionManager.cs | 9 ++--- .../MessageManagerTests.cs | 35 +++++++++++++++---- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index b4072bfb6..7918f7745 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -47,7 +47,9 @@ public ControlQueue( protected override TimeSpan MessageVisibilityTimeout => this.settings.ControlQueueVisibilityTimeout; - public async Task> GetMessagesAsync(CancellationToken cancellationToken) + public async Task> GetMessagesAsync( + CancellationToken cancellationToken, + List? newlyReservedMessageIds = null) { using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(this.releaseCancellationToken, cancellationToken)) { @@ -171,6 +173,7 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) this.storageAccountName); } + newlyReservedMessageIds?.AddRange(addedMessageIds); return sortedMessages; } catch (Exception e) diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index 4ed10a915..e64ab83d1 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -147,7 +147,8 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false); // This will block until either new messages arrive or the queue is released. - IReadOnlyList messages = await controlQueue.GetMessagesAsync(cancellationToken); + var newlyReservedMessageIds = new List(); + IReadOnlyList messages = await controlQueue.GetMessagesAsync(cancellationToken, newlyReservedMessageIds); if (messages.Count > 0) { // De-dupe any execution started messages @@ -162,10 +163,10 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati } catch { - // These messages were not handed off. Release their buffer slots. - foreach (MessageData message in messages) + // Keep reservations owned by earlier copies of duplicate messages. + foreach (string messageId in newlyReservedMessageIds) { - this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.MessageId, out _); + this.stats.PendingOrchestratorMessages.TryRemove(messageId, out _); } throw; diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index ea1ab93f4..96c0cc30c 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -225,8 +225,10 @@ public async Task ControlQueue_FailedBatch_ReleasesOnlyNewBufferEntries() Assert.AreEqual(2, receiveCalls, "The failed batch must not exhaust the buffer and block the next receive."); } - [TestMethod] - public async Task DequeueLoop_DeduplicationFailure_ReleasesUnhandedBufferEntries() + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task DequeueLoop_DeduplicationFailure_ReleasesOnlyNewBufferEntries(bool activeSession) { var settings = TestHelpers.GetMockQueueSettings(out var queueClient); var client = new AzureStorageClient(settings); @@ -240,21 +242,36 @@ public async Task DequeueLoop_DeduplicationFailure_ReleasesUnhandedBufferEntries OrchestrationInstance = message.TaskMessage.OrchestrationInstance, }; DateTimeOffset now = DateTimeOffset.UtcNow; + string originalPayload = await messageManager.SerializeMessageDataAsync(CreateMessageData("earlier")); + QueueMessage originalQueueMessage = QueuesModelFactory.QueueMessage( + "earlier", "original-receipt", originalPayload, 1, now, now.AddDays(1), now.AddMinutes(5)); + QueueMessage duplicateQueueMessage = QueuesModelFactory.QueueMessage( + "earlier", "duplicate-receipt", originalPayload, 2, now, now.AddDays(1), now.AddMinutes(5)); QueueMessage queueMessage = QueuesModelFactory.QueueMessage( "start", "receipt", await messageManager.SerializeMessageDataAsync(message), 1, now, now.AddDays(1), now.AddMinutes(5)); - queueClient.Setup(client => client.ReceiveMessagesAsync( + queueClient.SetupSequence(client => client.ReceiveMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(new[] { queueMessage }, Mock.Of())); - client.Stats.PendingOrchestratorMessages.TryAdd("earlier", 1); + .ReturnsAsync(Response.FromValue(new[] { originalQueueMessage }, Mock.Of())) + .ReturnsAsync(Response.FromValue(new[] { duplicateQueueMessage, queueMessage }, Mock.Of())); var trackingStore = new Mock(MockBehavior.Strict); + trackingStore.Setup(store => store.GetHistoryEventsAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new OrchestrationHistory(Array.Empty())); trackingStore.Setup(store => store.FetchInstanceStatusAsync( It.IsAny>(), It.IsAny())) .Callback(() => queue.Release(null, "Stop after the simulated deduplication failure.")) .Throws(new InvalidOperationException("Simulated instance status lookup failure.")); using var manager = new OrchestrationSessionManager( client.QueueAccountName, settings, client.Stats, trackingStore.Object); + IReadOnlyList originalBatch = await queue.GetMessagesAsync(timeout.Token); + MessageData originalMessage = originalBatch.Single(); + manager.AddMessageToPendingOrchestration(queue, originalBatch, Guid.NewGuid(), timeout.Token); + OrchestrationSession? originalSession = activeSession + ? await manager.GetNextSessionAsync(entitiesOnly: false, timeout.Token) + ?? throw new InvalidOperationException("The original session was not acquired.") + : null; MethodInfo dequeueLoop = typeof(OrchestrationSessionManager).GetMethod( "DequeueLoop", BindingFlags.Instance | BindingFlags.NonPublic) ?? throw new MissingMethodException("DequeueLoop"); @@ -264,8 +281,12 @@ public async Task DequeueLoop_DeduplicationFailure_ReleasesUnhandedBufferEntries CollectionAssert.AreEquivalent(new[] { "earlier" }, client.Stats.PendingOrchestratorMessages.Keys.ToArray()); manager.GetStats(out int pendingInstances, out _, out int activeSessions); - Assert.AreEqual(0, pendingInstances); - Assert.AreEqual(0, activeSessions); + Assert.AreEqual(activeSession ? 0 : 1, pendingInstances); + Assert.AreEqual(activeSession ? 1 : 0, activeSessions); + OrchestrationSession retainedSession = originalSession + ?? await manager.GetNextSessionAsync(entitiesOnly: false, timeout.Token) + ?? throw new InvalidOperationException("The original message's session was lost."); + Assert.AreSame(originalMessage, retainedSession.CurrentMessageBatch.Single()); trackingStore.Verify(store => store.FetchInstanceStatusAsync( It.IsAny>(), It.IsAny()), Times.Once); } From afe5a86db1b1be8cc16f7de7f16fc22a59b282ff Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Nov 2026 22:20:48 -0800 Subject: [PATCH 3/6] Restore retained message reservations during session release Re-establish buffer entries for pending and deferred messages at the locked session handoff. This preserves a reservation when abandonment of a current message removes the entry shared with a pending duplicate. Add a deterministic preparation-failure regression that injects the duplicate while storage validation is awaiting completion. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OrchestrationSessionManager.cs | 9 ++- .../AzureStorageScenarioTests.cs | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index e64ab83d1..1f8d7fb73 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -720,10 +720,17 @@ public bool TryReleaseSession(string instanceId, CancellationToken cancellationT if (this.activeOrchestrationSessions.TryGetValue(instanceId, out session) && this.activeOrchestrationSessions.Remove(instanceId)) { + IEnumerable retainedMessages = session.PendingMessages.Concat(session.DeferredMessages); + // Abandoning a current message can remove a reservation still needed by a pending duplicate. + foreach (MessageData message in retainedMessages) + { + this.stats.PendingOrchestratorMessages.TryAdd(message.OriginalQueueMessage.MessageId, 1); + } + // Put any unprocessed messages back into the pending buffer. this.AddMessageToPendingOrchestration( session.ControlQueue, - session.PendingMessages.Concat(session.DeferredMessages), + retainedMessages, session.TraceActivityId, cancellationToken); return true; diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 1dd58c610..231b72459 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -4581,6 +4581,75 @@ await Assert.ThrowsExceptionAsync( "failed", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } + [TestMethod] + public async Task WorkItemPreparationFailure_PreservesPendingDuplicateReservation() + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + var statusLookupStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var statusLookup = new TaskCompletionSource>( + TaskCreationOptions.RunContinuationsAsynchronously); + var instanceStore = new Mock(MockBehavior.Strict); + instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + instanceStore.Setup(store => store.GetOrchestrationStateAsync(It.IsAny(), false)) + .Callback(() => statusLookupStarted.TrySetResult(true)) + .Returns(statusLookup.Task); + using var shutdown = new CancellationTokenSource(StandardTimeout); + using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); + using var queue = service.AllControlQueues.Single(); + var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; + var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); + string payload = await messageManager.SerializeMessageDataAsync(new MessageData( + new TaskMessage { OrchestrationInstance = instance, Event = new ExecutionTerminatedEvent(-1, "terminate") }, + Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); + DateTimeOffset now = DateTimeOffset.UtcNow; + queueClient.SetupSequence(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(new[] + { + QueuesModelFactory.QueueMessage("current", "original-receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + }, Mock.Of())) + .ReturnsAsync(Response.FromValue(new[] + { + QueuesModelFactory.QueueMessage("current", "duplicate-receipt", payload, 2, now, now.AddDays(1), now.AddMinutes(5)), + }, Mock.Of())); + IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); + manager.AddMessageToPendingOrchestration(queue, messages, Guid.NewGuid(), shutdown.Token); + Task preparation = service.LockNextTaskOrchestrationWorkItemAsync(StandardTimeout, shutdown.Token); + + try + { + await TestHelpers.WaitFor(() => statusLookupStarted.Task.IsCompleted, StandardTimeout); + Assert.IsFalse(preparation.IsCompleted, "Preparation must still be awaiting the storage lookup."); + Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession session)); + + IReadOnlyList duplicates = await queue.GetMessagesAsync(shutdown.Token); + MessageData duplicate = duplicates.Single(); + manager.AddMessageToPendingOrchestration(queue, duplicates, Guid.NewGuid(), shutdown.Token); + Assert.AreSame(duplicate, session.PendingMessages.Single()); + manager.GetStats(out _, out int bufferedBeforeFailure, out _); + Assert.AreEqual(1, bufferedBeforeFailure, "The duplicate must share the original message's reservation."); + + statusLookup.SetException(new InvalidOperationException("Simulated work item validation failure.")); + await Assert.ThrowsExceptionAsync(() => preparation); + + manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); + Assert.AreEqual(0, activeSessions); + Assert.AreEqual(1, pendingInstances); + Assert.AreEqual(1, bufferedMessages, "The retained pending duplicate must keep its buffer reservation."); + OrchestrationSession replacement = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); + Assert.AreNotSame(session, replacement); + Assert.AreSame(duplicate, replacement.CurrentMessageBatch.Single()); + queueClient.Verify(client => client.UpdateMessageAsync( + "current", "original-receipt", It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + finally + { + shutdown.Cancel(); + statusLookup.TrySetCanceled(); + } + } + static AzureStorageOrchestrationService CreateServiceForSessionFailureTest( AzureStorageOrchestrationServiceSettings settings, IOrchestrationServiceInstanceStore instanceStore, From 64acf867e7ea7b126cc78a802bebbb20d063e3de Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Nov 2026 22:47:53 -0800 Subject: [PATCH 4/6] Release aborted session buffer reservations during forced shutdown Remove reservations for messages owned by aborted sessions while preserving IDs still held by independent pending batches. Keep late cleanup from touching replacement sessions. Cover delayed preparation cancellation and failure, all session message collections, resumed receiving, and shared pending reservations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OrchestrationSessionTests.cs | 20 ++- .../OrchestrationSessionManager.cs | 20 ++- .../AzureStorageScenarioTests.cs | 142 ++++++++++++++++++ 3 files changed, 177 insertions(+), 5 deletions(-) diff --git a/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs b/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs index 126c4b9bc..18adc5c2a 100644 --- a/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs +++ b/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs @@ -22,7 +22,9 @@ namespace DurableTask.AzureStorage.Tests using System.Threading.Tasks; using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Monitoring; + using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; + using DurableTask.Core; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -171,7 +173,13 @@ public async Task WaitAsync_AlreadySignaledAndCancelled_ReturnsTrue() [TestMethod] public void AbortAllSessions_ClearsActiveSessions() { - var settings = new AzureStorageOrchestrationServiceSettings(); + var settings = new AzureStorageOrchestrationServiceSettings + { + StorageAccountClientProvider = new StorageAccountClientProvider("UseDevelopmentStorage=true"), + }; + var client = new AzureStorageClient(settings); + var messageManager = new MessageManager(settings, client, "abort-largemessages"); + using var queue = new ControlQueue(client, "abort-control-00", messageManager); var stats = new AzureStorageOrchestrationServiceStats(); var trackingStore = new Mock(); @@ -189,9 +197,13 @@ public void AbortAllSessions_ClearsActiveSessions() manager.GetStats(out _, out _, out int initialCount); Assert.AreEqual(0, initialCount, "Should start with no active sessions"); - sessions["instance1"] = null; - sessions["instance2"] = null; - sessions["instance3"] = null; + foreach (string instanceId in new[] { "instance1", "instance2", "instance3" }) + { + sessions[instanceId] = new OrchestrationSession( + settings, "testaccount", new OrchestrationInstance { InstanceId = instanceId }, + queue, new List(), new OrchestrationRuntimeState(), new OrchestrationETags(), + DateTime.UtcNow, null, TimeSpan.Zero, CancellationToken.None, Guid.NewGuid()); + } manager.GetStats(out _, out _, out int activeCount); Assert.AreEqual(3, activeCount, "Should have 3 active sessions"); diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index 1f8d7fb73..9ce06e6f6 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -690,7 +690,8 @@ async Task ScheduleOrchestrationStatePrefetch( } /// - /// Immediately removes all active sessions, causing + /// Immediately removes all active sessions and releases their unretained message reservations, + /// causing /// to return false for all partitions. This unblocks so that /// a forced shutdown can complete without waiting for sessions to drain naturally. /// @@ -698,6 +699,23 @@ public void AbortAllSessions() { lock (this.messageAndSessionLock) { + var pendingMessageIds = new HashSet(this.pendingOrchestrationMessageBatches + .SelectMany(batch => batch.Messages) + .Select(message => message.OriginalQueueMessage.MessageId)); + foreach (OrchestrationSession session in this.activeOrchestrationSessions.Values) + { + foreach (MessageData message in session.PendingMessages + .Concat(session.CurrentMessageBatch) + .Concat(session.DeferredMessages) + .Concat(session.DiscardedMessages)) + { + if (!pendingMessageIds.Contains(message.OriginalQueueMessage.MessageId)) + { + this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.MessageId, out _); + } + } + } + this.activeOrchestrationSessions.Clear(); } } diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 231b72459..ac82e336b 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -4650,6 +4650,148 @@ public async Task WorkItemPreparationFailure_PreservesPendingDuplicateReservatio } } + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task WorkItemPreparation_ForcedAbortReleasesReservations(bool cancelStorageRequest) + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + settings.ControlQueueBufferThreshold = 5; + var statusLookupStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var statusLookup = new TaskCompletionSource>( + TaskCreationOptions.RunContinuationsAsynchronously); + var instanceStore = new Mock(MockBehavior.Strict); + instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + instanceStore.Setup(store => store.GetOrchestrationStateAsync(It.IsAny(), false)) + .Callback(() => statusLookupStarted.TrySetResult(true)) + .Returns(statusLookup.Task); + using var shutdown = new CancellationTokenSource(StandardTimeout); + using var restarted = new CancellationTokenSource(StandardTimeout); + using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); + using var queue = service.AllControlQueues.Single(); + var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; + var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); + string payload = await messageManager.SerializeMessageDataAsync(new MessageData( + new TaskMessage { OrchestrationInstance = instance, Event = new ExecutionTerminatedEvent(-1, "terminate") }, + Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); + string waitingPayload = await messageManager.SerializeMessageDataAsync(new MessageData( + new TaskMessage + { + OrchestrationInstance = new OrchestrationInstance { InstanceId = "waiting", ExecutionId = "execution" }, + Event = new EventRaisedEvent(-1, "event"), + }, + Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); + DateTimeOffset now = DateTimeOffset.UtcNow; + QueueMessage[] originalBatch = new[] { "current", "pending", "deferred", "discarded" } + .Select(id => QueuesModelFactory.QueueMessage(id, "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5))) + .Concat(new[] { QueuesModelFactory.QueueMessage("waiting", "receipt", waitingPayload, 1, now, now.AddDays(1), now.AddMinutes(5)) }) + .ToArray(); + queueClient.SetupSequence(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(originalBatch, Mock.Of())) + .ReturnsAsync(Response.FromValue(new[] + { + QueuesModelFactory.QueueMessage("current", "replacement-receipt", payload, 2, now, now.AddDays(1), now.AddMinutes(5)), + }, Mock.Of())); + IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); + manager.AddMessageToPendingOrchestration( + queue, messages.Where(message => message.Id == "current"), Guid.NewGuid(), shutdown.Token); + Task preparation = service.LockNextTaskOrchestrationWorkItemAsync(StandardTimeout, shutdown.Token); + + try + { + await TestHelpers.WaitFor(() => statusLookupStarted.Task.IsCompleted, StandardTimeout); + Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession session)); + session.DeferMessage(messages.Single(message => message.Id == "deferred")); + session.DiscardMessage(messages.Single(message => message.Id == "discarded")); + manager.AddMessageToPendingOrchestration( + queue, messages.Where(message => message.Id == "pending" || message.Id == "waiting"), Guid.NewGuid(), shutdown.Token); + manager.GetStats(out _, out int bufferedBeforeAbort, out _); + Assert.AreEqual(5, bufferedBeforeAbort); + + shutdown.Cancel(); + manager.AbortAllSessions(); + manager.AbortAllSessions(); + + manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); + Assert.AreEqual(0, activeSessions); + Assert.AreEqual(1, pendingInstances); + Assert.AreEqual(1, bufferedMessages, "Only the independent waiting batch must retain its reservation."); + + IReadOnlyList replacementMessages = await queue.GetMessagesAsync(restarted.Token); + manager.AddMessageToPendingOrchestration(queue, replacementMessages, Guid.NewGuid(), restarted.Token); + await manager.GetNextSessionAsync(entitiesOnly: false, restarted.Token); + await manager.GetNextSessionAsync(entitiesOnly: false, restarted.Token); + Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession replacement)); + Assert.AreNotSame(session, replacement); + + if (cancelStorageRequest) + { + statusLookup.SetCanceled(); + Assert.IsNull(await preparation); + } + else + { + statusLookup.SetException(new InvalidOperationException("Storage failed after forced shutdown.")); + await Assert.ThrowsExceptionAsync(() => preparation); + } + + Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession currentSession)); + Assert.AreSame(replacement, currentSession, "Late cleanup must not release a replacement session."); + Assert.AreSame(replacementMessages.Single(), replacement.CurrentMessageBatch.Single()); + manager.GetStats(out _, out int remainingReservations, out int remainingSessions); + Assert.AreEqual(2, remainingReservations); + Assert.AreEqual(2, remainingSessions); + queueClient.Verify(client => client.UpdateMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + finally + { + shutdown.Cancel(); + statusLookup.TrySetCanceled(); + } + } + + [TestMethod] + public async Task AbortAllSessions_PreservesSharedPendingReservation() + { + var settings = TestHelpers.GetMockQueueSettings(out var queueClient); + var instanceStore = new Mock(MockBehavior.Strict); + instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + using var shutdown = new CancellationTokenSource(StandardTimeout); + using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); + using var queue = service.AllControlQueues.Single(); + var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; + string payload = await new MessageManager(settings, new AzureStorageClient(settings), "abort-largemessages") + .SerializeMessageDataAsync(new MessageData( + new TaskMessage { OrchestrationInstance = instance, Event = new ExecutionStartedEvent(-1, string.Empty) }, + Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); + DateTimeOffset now = DateTimeOffset.UtcNow; + queueClient.Setup(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(new[] + { + QueuesModelFactory.QueueMessage("start", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + }, Mock.Of())); + IReadOnlyList originalBatch = await queue.GetMessagesAsync(shutdown.Token); + manager.AddMessageToPendingOrchestration(queue, originalBatch, Guid.NewGuid(), shutdown.Token); + OrchestrationSession originalSession = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); + IReadOnlyList duplicateBatch = await queue.GetMessagesAsync(shutdown.Token); + manager.AddMessageToPendingOrchestration(queue, duplicateBatch, Guid.NewGuid(), shutdown.Token); + + manager.AbortAllSessions(); + + manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); + Assert.AreEqual(0, activeSessions); + Assert.AreEqual(1, pendingInstances); + Assert.AreEqual(1, bufferedMessages, "A duplicate retained in a pending batch must keep its shared reservation."); + OrchestrationSession replacement = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); + Assert.AreNotSame(originalSession, replacement); + Assert.AreSame(duplicateBatch.Single(), replacement.CurrentMessageBatch.Single()); + } + static AzureStorageOrchestrationService CreateServiceForSessionFailureTest( AzureStorageOrchestrationServiceSettings settings, IOrchestrationServiceInstanceStore instanceStore, From ed9376e3275f91576705b5a336f3a70a3858c6c2 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Nov 2026 23:22:32 -0800 Subject: [PATCH 5/6] Remove optional forced-shutdown reservation cleanup Restore the existing AbortAllSessions behavior rather than enumerate message collections without synchronization shared with processing code. Remove the dedicated cleanup tests and their now-unneeded test adaptation. Keep the earlier failed-dequeue, deduplication, and work-item-preparation fixes unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OrchestrationSessionTests.cs | 20 +-- .../OrchestrationSessionManager.cs | 20 +-- .../AzureStorageScenarioTests.cs | 142 ------------------ 3 files changed, 5 insertions(+), 177 deletions(-) diff --git a/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs b/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs index 18adc5c2a..126c4b9bc 100644 --- a/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs +++ b/Test/DurableTask.AzureStorage.Tests/OrchestrationSessionTests.cs @@ -22,9 +22,7 @@ namespace DurableTask.AzureStorage.Tests using System.Threading.Tasks; using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Monitoring; - using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; - using DurableTask.Core; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -173,13 +171,7 @@ public async Task WaitAsync_AlreadySignaledAndCancelled_ReturnsTrue() [TestMethod] public void AbortAllSessions_ClearsActiveSessions() { - var settings = new AzureStorageOrchestrationServiceSettings - { - StorageAccountClientProvider = new StorageAccountClientProvider("UseDevelopmentStorage=true"), - }; - var client = new AzureStorageClient(settings); - var messageManager = new MessageManager(settings, client, "abort-largemessages"); - using var queue = new ControlQueue(client, "abort-control-00", messageManager); + var settings = new AzureStorageOrchestrationServiceSettings(); var stats = new AzureStorageOrchestrationServiceStats(); var trackingStore = new Mock(); @@ -197,13 +189,9 @@ public void AbortAllSessions_ClearsActiveSessions() manager.GetStats(out _, out _, out int initialCount); Assert.AreEqual(0, initialCount, "Should start with no active sessions"); - foreach (string instanceId in new[] { "instance1", "instance2", "instance3" }) - { - sessions[instanceId] = new OrchestrationSession( - settings, "testaccount", new OrchestrationInstance { InstanceId = instanceId }, - queue, new List(), new OrchestrationRuntimeState(), new OrchestrationETags(), - DateTime.UtcNow, null, TimeSpan.Zero, CancellationToken.None, Guid.NewGuid()); - } + sessions["instance1"] = null; + sessions["instance2"] = null; + sessions["instance3"] = null; manager.GetStats(out _, out _, out int activeCount); Assert.AreEqual(3, activeCount, "Should have 3 active sessions"); diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index 9ce06e6f6..1f8d7fb73 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -690,8 +690,7 @@ async Task ScheduleOrchestrationStatePrefetch( } /// - /// Immediately removes all active sessions and releases their unretained message reservations, - /// causing + /// Immediately removes all active sessions, causing /// to return false for all partitions. This unblocks so that /// a forced shutdown can complete without waiting for sessions to drain naturally. /// @@ -699,23 +698,6 @@ public void AbortAllSessions() { lock (this.messageAndSessionLock) { - var pendingMessageIds = new HashSet(this.pendingOrchestrationMessageBatches - .SelectMany(batch => batch.Messages) - .Select(message => message.OriginalQueueMessage.MessageId)); - foreach (OrchestrationSession session in this.activeOrchestrationSessions.Values) - { - foreach (MessageData message in session.PendingMessages - .Concat(session.CurrentMessageBatch) - .Concat(session.DeferredMessages) - .Concat(session.DiscardedMessages)) - { - if (!pendingMessageIds.Contains(message.OriginalQueueMessage.MessageId)) - { - this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.MessageId, out _); - } - } - } - this.activeOrchestrationSessions.Clear(); } } diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index ac82e336b..231b72459 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -4650,148 +4650,6 @@ public async Task WorkItemPreparationFailure_PreservesPendingDuplicateReservatio } } - [DataTestMethod] - [DataRow(false)] - [DataRow(true)] - public async Task WorkItemPreparation_ForcedAbortReleasesReservations(bool cancelStorageRequest) - { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - settings.ControlQueueBufferThreshold = 5; - var statusLookupStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var statusLookup = new TaskCompletionSource>( - TaskCreationOptions.RunContinuationsAsynchronously); - var instanceStore = new Mock(MockBehavior.Strict); - instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(Array.Empty()); - instanceStore.Setup(store => store.GetOrchestrationStateAsync(It.IsAny(), false)) - .Callback(() => statusLookupStarted.TrySetResult(true)) - .Returns(statusLookup.Task); - using var shutdown = new CancellationTokenSource(StandardTimeout); - using var restarted = new CancellationTokenSource(StandardTimeout); - using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); - using var queue = service.AllControlQueues.Single(); - var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; - var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); - string payload = await messageManager.SerializeMessageDataAsync(new MessageData( - new TaskMessage { OrchestrationInstance = instance, Event = new ExecutionTerminatedEvent(-1, "terminate") }, - Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); - string waitingPayload = await messageManager.SerializeMessageDataAsync(new MessageData( - new TaskMessage - { - OrchestrationInstance = new OrchestrationInstance { InstanceId = "waiting", ExecutionId = "execution" }, - Event = new EventRaisedEvent(-1, "event"), - }, - Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); - DateTimeOffset now = DateTimeOffset.UtcNow; - QueueMessage[] originalBatch = new[] { "current", "pending", "deferred", "discarded" } - .Select(id => QueuesModelFactory.QueueMessage(id, "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5))) - .Concat(new[] { QueuesModelFactory.QueueMessage("waiting", "receipt", waitingPayload, 1, now, now.AddDays(1), now.AddMinutes(5)) }) - .ToArray(); - queueClient.SetupSequence(client => client.ReceiveMessagesAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(originalBatch, Mock.Of())) - .ReturnsAsync(Response.FromValue(new[] - { - QueuesModelFactory.QueueMessage("current", "replacement-receipt", payload, 2, now, now.AddDays(1), now.AddMinutes(5)), - }, Mock.Of())); - IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); - manager.AddMessageToPendingOrchestration( - queue, messages.Where(message => message.Id == "current"), Guid.NewGuid(), shutdown.Token); - Task preparation = service.LockNextTaskOrchestrationWorkItemAsync(StandardTimeout, shutdown.Token); - - try - { - await TestHelpers.WaitFor(() => statusLookupStarted.Task.IsCompleted, StandardTimeout); - Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession session)); - session.DeferMessage(messages.Single(message => message.Id == "deferred")); - session.DiscardMessage(messages.Single(message => message.Id == "discarded")); - manager.AddMessageToPendingOrchestration( - queue, messages.Where(message => message.Id == "pending" || message.Id == "waiting"), Guid.NewGuid(), shutdown.Token); - manager.GetStats(out _, out int bufferedBeforeAbort, out _); - Assert.AreEqual(5, bufferedBeforeAbort); - - shutdown.Cancel(); - manager.AbortAllSessions(); - manager.AbortAllSessions(); - - manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); - Assert.AreEqual(0, activeSessions); - Assert.AreEqual(1, pendingInstances); - Assert.AreEqual(1, bufferedMessages, "Only the independent waiting batch must retain its reservation."); - - IReadOnlyList replacementMessages = await queue.GetMessagesAsync(restarted.Token); - manager.AddMessageToPendingOrchestration(queue, replacementMessages, Guid.NewGuid(), restarted.Token); - await manager.GetNextSessionAsync(entitiesOnly: false, restarted.Token); - await manager.GetNextSessionAsync(entitiesOnly: false, restarted.Token); - Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession replacement)); - Assert.AreNotSame(session, replacement); - - if (cancelStorageRequest) - { - statusLookup.SetCanceled(); - Assert.IsNull(await preparation); - } - else - { - statusLookup.SetException(new InvalidOperationException("Storage failed after forced shutdown.")); - await Assert.ThrowsExceptionAsync(() => preparation); - } - - Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession currentSession)); - Assert.AreSame(replacement, currentSession, "Late cleanup must not release a replacement session."); - Assert.AreSame(replacementMessages.Single(), replacement.CurrentMessageBatch.Single()); - manager.GetStats(out _, out int remainingReservations, out int remainingSessions); - Assert.AreEqual(2, remainingReservations); - Assert.AreEqual(2, remainingSessions); - queueClient.Verify(client => client.UpdateMessageAsync( - It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); - } - finally - { - shutdown.Cancel(); - statusLookup.TrySetCanceled(); - } - } - - [TestMethod] - public async Task AbortAllSessions_PreservesSharedPendingReservation() - { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - var instanceStore = new Mock(MockBehavior.Strict); - instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(Array.Empty()); - using var shutdown = new CancellationTokenSource(StandardTimeout); - using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); - using var queue = service.AllControlQueues.Single(); - var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; - string payload = await new MessageManager(settings, new AzureStorageClient(settings), "abort-largemessages") - .SerializeMessageDataAsync(new MessageData( - new TaskMessage { OrchestrationInstance = instance, Event = new ExecutionStartedEvent(-1, string.Empty) }, - Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); - DateTimeOffset now = DateTimeOffset.UtcNow; - queueClient.Setup(client => client.ReceiveMessagesAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(new[] - { - QueuesModelFactory.QueueMessage("start", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), - }, Mock.Of())); - IReadOnlyList originalBatch = await queue.GetMessagesAsync(shutdown.Token); - manager.AddMessageToPendingOrchestration(queue, originalBatch, Guid.NewGuid(), shutdown.Token); - OrchestrationSession originalSession = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); - IReadOnlyList duplicateBatch = await queue.GetMessagesAsync(shutdown.Token); - manager.AddMessageToPendingOrchestration(queue, duplicateBatch, Guid.NewGuid(), shutdown.Token); - - manager.AbortAllSessions(); - - manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); - Assert.AreEqual(0, activeSessions); - Assert.AreEqual(1, pendingInstances); - Assert.AreEqual(1, bufferedMessages, "A duplicate retained in a pending batch must keep its shared reservation."); - OrchestrationSession replacement = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); - Assert.AreNotSame(originalSession, replacement); - Assert.AreSame(duplicateBatch.Single(), replacement.CurrentMessageBatch.Single()); - } - static AzureStorageOrchestrationService CreateServiceForSessionFailureTest( AzureStorageOrchestrationServiceSettings settings, IOrchestrationServiceInstanceStore instanceStore, From 71e83fd41e1417f7938f033b09c7af047b1826fe Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 12 Nov 2026 13:09:59 -0800 Subject: [PATCH 6/6] Limit failed-message fix to ControlQueue batch cleanup Keep only rollback of buffer IDs newly inserted by a failed ControlQueue receive batch and a focused mocked regression test. Remove session management, work-item preparation, prefetch retry and handoff changes along with their tests and shared helper additions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AzureStorageOrchestrationService.cs | 21 +- .../Messaging/ControlQueue.cs | 5 +- .../OrchestrationSessionManager.cs | 38 +--- .../AzureStorageScenarioTests.cs | 201 ------------------ .../MessageManagerTests.cs | 89 ++------ .../OrchestrationStatePrefetchTests.cs | 35 --- .../TestHelpers.cs | 43 ---- 7 files changed, 30 insertions(+), 402 deletions(-) diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index dc6150ee3..54228ff3c 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -901,12 +901,10 @@ await this.AbandonAndReleaseSessionAsync( session?.Instance.ExecutionId ?? string.Empty, e.ToString()); - if (session != null) + if (orchestrationWorkItem != null) { - // No work item was handed off. Release the session and its messages. - await this.AbandonAndReleaseSessionAsync( - session, - "Failed to prepare the orchestration work item."); + // The work-item needs to be released so that it can be retried later. + await this.ReleaseTaskOrchestrationWorkItemAsync(orchestrationWorkItem); } throw; @@ -1146,19 +1144,6 @@ await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( async Task AbandonAndReleaseSessionAsync(OrchestrationSession session, string details) { - // An earlier cleanup may already have released this session. - if (!this.orchestrationSessionManager.TryGetExistingSession(session.Instance.InstanceId, out OrchestrationSession currentSession) || - !ReferenceEquals(session, currentSession)) - { - return; - } - - // Deferred messages stay buffered for the next session. - foreach (MessageData message in session.DeferredMessages) - { - session.CurrentMessageBatch.Remove(message); - } - try { await this.AbandonSessionAsync(session, details); diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index 7918f7745..b4072bfb6 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -47,9 +47,7 @@ public ControlQueue( protected override TimeSpan MessageVisibilityTimeout => this.settings.ControlQueueVisibilityTimeout; - public async Task> GetMessagesAsync( - CancellationToken cancellationToken, - List? newlyReservedMessageIds = null) + public async Task> GetMessagesAsync(CancellationToken cancellationToken) { using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(this.releaseCancellationToken, cancellationToken)) { @@ -173,7 +171,6 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) this.storageAccountName); } - newlyReservedMessageIds?.AddRange(addedMessageIds); return sortedMessages; } catch (Exception e) diff --git a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs index 1f8d7fb73..9f0105353 100644 --- a/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs +++ b/src/DurableTask.AzureStorage/OrchestrationSessionManager.cs @@ -147,30 +147,15 @@ async Task DequeueLoop(string partitionId, ControlQueue controlQueue, Cancellati Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false); // This will block until either new messages arrive or the queue is released. - var newlyReservedMessageIds = new List(); - IReadOnlyList messages = await controlQueue.GetMessagesAsync(cancellationToken, newlyReservedMessageIds); + IReadOnlyList messages = await controlQueue.GetMessagesAsync(cancellationToken); if (messages.Count > 0) { // De-dupe any execution started messages - IEnumerable filteredMessages; - try - { - filteredMessages = await this.DedupeExecutionStartedMessagesAsync( - controlQueue, - messages, - traceActivityId, - cancellationToken); - } - catch - { - // Keep reservations owned by earlier copies of duplicate messages. - foreach (string messageId in newlyReservedMessageIds) - { - this.stats.PendingOrchestratorMessages.TryRemove(messageId, out _); - } - - throw; - } + IEnumerable filteredMessages = await this.DedupeExecutionStartedMessagesAsync( + controlQueue, + messages, + traceActivityId, + cancellationToken); this.AddMessageToPendingOrchestration(controlQueue, filteredMessages, traceActivityId, cancellationToken); } @@ -581,7 +566,7 @@ async Task ScheduleOrchestrationStatePrefetch( this.orchestrationsReadyForProcessingQueue.Enqueue(node); } } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) { // shutting down } @@ -720,17 +705,10 @@ public bool TryReleaseSession(string instanceId, CancellationToken cancellationT if (this.activeOrchestrationSessions.TryGetValue(instanceId, out session) && this.activeOrchestrationSessions.Remove(instanceId)) { - IEnumerable retainedMessages = session.PendingMessages.Concat(session.DeferredMessages); - // Abandoning a current message can remove a reservation still needed by a pending duplicate. - foreach (MessageData message in retainedMessages) - { - this.stats.PendingOrchestratorMessages.TryAdd(message.OriginalQueueMessage.MessageId, 1); - } - // Put any unprocessed messages back into the pending buffer. this.AddMessageToPendingOrchestration( session.ControlQueue, - retainedMessages, + session.PendingMessages.Concat(session.DeferredMessages), session.TraceActivityId, cancellationToken); return true; diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 231b72459..90acbe99e 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -13,19 +13,15 @@ namespace DurableTask.AzureStorage.Tests { - using Azure; using Azure.Data.Tables; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; - using Azure.Storage.Queues.Models; - using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Storage; using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Settings; - using DurableTask.Core.Tracking; using Microsoft.Practices.EnterpriseLibrary.SemanticLogging.Utility; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -4471,203 +4467,6 @@ await service.SendTaskOrchestrationMessageAsync( } } - [TestMethod] - public async Task SessionCleanup_PreservesDeferredMessagesAndReplacementSession() - { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - var instanceStore = new Mock(MockBehavior.Strict); - instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(Array.Empty()); - using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); - using var queue = service.AllControlQueues.Single(); - var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; - var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); - string payload = await messageManager.SerializeMessageDataAsync(new MessageData( - new TaskMessage { OrchestrationInstance = instance, Event = new EventRaisedEvent(-1, "event") }, - Guid.NewGuid(), queue.Name, null, new OrchestrationInstance())); - DateTimeOffset now = DateTimeOffset.UtcNow; - queueClient.Setup(client => client.ReceiveMessagesAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(new[] - { - QueuesModelFactory.QueueMessage("current", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), - QueuesModelFactory.QueueMessage("deferred", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), - }, Mock.Of())); - IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); - manager.AddMessageToPendingOrchestration(queue, messages, Guid.NewGuid(), shutdown.Token); - OrchestrationSession session = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); - MessageData deferredMessage = messages.Single(message => message.OriginalQueueMessage.MessageId == "deferred"); - session.DeferMessage(deferredMessage); - MethodInfo cleanupMethod = typeof(AzureStorageOrchestrationService).GetMethod( - "AbandonAndReleaseSessionAsync", BindingFlags.Instance | BindingFlags.NonPublic); - Assert.IsNotNull(cleanupMethod); - - await (Task)cleanupMethod.Invoke(service, new object[] { session, "Simulated preparation failure." }); - OrchestrationSession replacement = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); - - Assert.AreSame(deferredMessage, replacement.CurrentMessageBatch.Single()); - manager.GetStats(out _, out int bufferedMessages, out _); - Assert.AreEqual(1, bufferedMessages, "The deferred message must keep its buffer slot."); - queueClient.Verify(client => client.UpdateMessageAsync( - "current", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - queueClient.Verify(client => client.UpdateMessageAsync( - "deferred", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); - - await (Task)cleanupMethod.Invoke(service, new object[] { session, "Repeated cleanup of the old session." }); - - Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession currentSession)); - Assert.AreSame(replacement, currentSession, "Cleanup of a released session must not release its replacement."); - } - - [DataTestMethod] - [DataRow(false)] - [DataRow(true)] - public async Task WorkItemPreparationFailure_AbandonsMessagesAndReleasesSession(bool failAfterCreatingWorkItem) - { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - var instanceStore = new Mock(MockBehavior.Strict); - instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(Array.Empty()); - instanceStore.Setup(store => store.GetOrchestrationStateAsync(It.IsAny(), false)) - .ThrowsAsync(new InvalidOperationException("Simulated work item validation failure.")); - using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); - using var queue = service.AllControlQueues.Single(); - var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; - var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); - var taskMessage = new TaskMessage - { - OrchestrationInstance = instance, - Event = failAfterCreatingWorkItem - ? new ExecutionTerminatedEvent(-1, "terminate") - : new EventRaisedEvent(-1, "event"), - }; - string payload = await messageManager.SerializeMessageDataAsync(new MessageData( - taskMessage, Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); - if (!failAfterCreatingWorkItem) - { - JObject envelope = JObject.Parse(payload); - envelope.Remove(nameof(MessageData.Sender)); - payload = envelope.ToString(Formatting.None); - } - - DateTimeOffset now = DateTimeOffset.UtcNow; - QueueMessage queueMessage = QueuesModelFactory.QueueMessage( - "failed", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)); - queueClient.Setup(client => client.ReceiveMessagesAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(new[] { queueMessage }, Mock.Of())); - IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); - manager.AddMessageToPendingOrchestration(queue, messages, Guid.NewGuid(), shutdown.Token); - - if (failAfterCreatingWorkItem) - { - await Assert.ThrowsExceptionAsync( - () => service.LockNextTaskOrchestrationWorkItemAsync(TimeSpan.FromSeconds(5), shutdown.Token)); - } - else - { - // Missing sender metadata faults before a work item is constructed. - await Assert.ThrowsExceptionAsync( - () => service.LockNextTaskOrchestrationWorkItemAsync(TimeSpan.FromSeconds(5), shutdown.Token)); - } - - manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); - Assert.AreEqual(0, activeSessions, "No session may remain owned when no work item was handed off."); - Assert.AreEqual(0, bufferedMessages, "Failed messages must release their buffer slots."); - Assert.AreEqual(0, pendingInstances); - queueClient.Verify(client => client.UpdateMessageAsync( - "failed", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - } - - [TestMethod] - public async Task WorkItemPreparationFailure_PreservesPendingDuplicateReservation() - { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - var statusLookupStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var statusLookup = new TaskCompletionSource>( - TaskCreationOptions.RunContinuationsAsynchronously); - var instanceStore = new Mock(MockBehavior.Strict); - instanceStore.Setup(store => store.GetOrchestrationHistoryEventsAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(Array.Empty()); - instanceStore.Setup(store => store.GetOrchestrationStateAsync(It.IsAny(), false)) - .Callback(() => statusLookupStarted.TrySetResult(true)) - .Returns(statusLookup.Task); - using var shutdown = new CancellationTokenSource(StandardTimeout); - using var service = CreateServiceForSessionFailureTest(settings, instanceStore.Object, shutdown, out var manager); - using var queue = service.AllControlQueues.Single(); - var instance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }; - var messageManager = new MessageManager(settings, new AzureStorageClient(settings), "buffer-largemessages"); - string payload = await messageManager.SerializeMessageDataAsync(new MessageData( - new TaskMessage { OrchestrationInstance = instance, Event = new ExecutionTerminatedEvent(-1, "terminate") }, - Guid.NewGuid(), queue.Name, null, new OrchestrationInstance { InstanceId = "sender" })); - DateTimeOffset now = DateTimeOffset.UtcNow; - queueClient.SetupSequence(client => client.ReceiveMessagesAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(new[] - { - QueuesModelFactory.QueueMessage("current", "original-receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), - }, Mock.Of())) - .ReturnsAsync(Response.FromValue(new[] - { - QueuesModelFactory.QueueMessage("current", "duplicate-receipt", payload, 2, now, now.AddDays(1), now.AddMinutes(5)), - }, Mock.Of())); - IReadOnlyList messages = await queue.GetMessagesAsync(shutdown.Token); - manager.AddMessageToPendingOrchestration(queue, messages, Guid.NewGuid(), shutdown.Token); - Task preparation = service.LockNextTaskOrchestrationWorkItemAsync(StandardTimeout, shutdown.Token); - - try - { - await TestHelpers.WaitFor(() => statusLookupStarted.Task.IsCompleted, StandardTimeout); - Assert.IsFalse(preparation.IsCompleted, "Preparation must still be awaiting the storage lookup."); - Assert.IsTrue(manager.TryGetExistingSession(instance.InstanceId, out OrchestrationSession session)); - - IReadOnlyList duplicates = await queue.GetMessagesAsync(shutdown.Token); - MessageData duplicate = duplicates.Single(); - manager.AddMessageToPendingOrchestration(queue, duplicates, Guid.NewGuid(), shutdown.Token); - Assert.AreSame(duplicate, session.PendingMessages.Single()); - manager.GetStats(out _, out int bufferedBeforeFailure, out _); - Assert.AreEqual(1, bufferedBeforeFailure, "The duplicate must share the original message's reservation."); - - statusLookup.SetException(new InvalidOperationException("Simulated work item validation failure.")); - await Assert.ThrowsExceptionAsync(() => preparation); - - manager.GetStats(out int pendingInstances, out int bufferedMessages, out int activeSessions); - Assert.AreEqual(0, activeSessions); - Assert.AreEqual(1, pendingInstances); - Assert.AreEqual(1, bufferedMessages, "The retained pending duplicate must keep its buffer reservation."); - OrchestrationSession replacement = await manager.GetNextSessionAsync(entitiesOnly: false, shutdown.Token); - Assert.AreNotSame(session, replacement); - Assert.AreSame(duplicate, replacement.CurrentMessageBatch.Single()); - queueClient.Verify(client => client.UpdateMessageAsync( - "current", "original-receipt", It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - } - finally - { - shutdown.Cancel(); - statusLookup.TrySetCanceled(); - } - } - - static AzureStorageOrchestrationService CreateServiceForSessionFailureTest( - AzureStorageOrchestrationServiceSettings settings, - IOrchestrationServiceInstanceStore instanceStore, - CancellationTokenSource shutdown, - out OrchestrationSessionManager manager) - { - var service = new AzureStorageOrchestrationService(settings, instanceStore); - const BindingFlags Flags = BindingFlags.Instance | BindingFlags.NonPublic; - manager = (OrchestrationSessionManager)typeof(AzureStorageOrchestrationService) - .GetField("orchestrationSessionManager", Flags).GetValue(service); - typeof(AzureStorageOrchestrationService).GetField("shutdownSource", Flags).SetValue(service, shutdown); - - // Skip task hub creation and listener startup; all queue I/O in these tests is mocked. - object creator = typeof(AzureStorageOrchestrationService).GetField("taskHubCreator", Flags).GetValue(service); - creator.GetType().GetField("lazy", Flags).SetValue(creator, new Lazy(() => Task.CompletedTask)); - return service; - } - [TestMethod] public async Task WorkerDoesNotDiscardExternalEventBatchedWithMessageForMissingExecution() { diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index 96c0cc30c..4e2f76a2f 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -16,10 +16,10 @@ namespace DurableTask.AzureStorage.Tests using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; + using Azure.Storage.Queues; using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Storage; - using DurableTask.AzureStorage.Tracking; using DurableTask.Core; using DurableTask.Core.History; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -29,7 +29,6 @@ namespace DurableTask.AzureStorage.Tests using System; using System.Collections.Generic; using System.Linq; - using System.Reflection; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -185,11 +184,25 @@ public async Task SerializeMessageDataAsync_UsesExpectedStoragePathAtThreshold( [TestMethod] public async Task ControlQueue_FailedBatch_ReleasesOnlyNewBufferEntries() { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - settings.ControlQueueBufferThreshold = 3; + const string ConnectionString = "UseDevelopmentStorage=true"; + const string QueueName = "buffer-control-00"; + var queueClient = new Mock(MockBehavior.Strict, ConnectionString, QueueName); + queueClient.SetupGet(client => client.Name).Returns(QueueName); + var serviceClient = new Mock(MockBehavior.Strict, ConnectionString); + serviceClient.SetupGet(client => client.AccountName).Returns("devstoreaccount1"); + serviceClient.Setup(client => client.GetQueueClient(QueueName)).Returns(queueClient.Object); + var queueProvider = new Mock>(MockBehavior.Strict); + queueProvider.Setup(provider => provider.CreateOptions()).Returns(() => new QueueClientOptions()); + queueProvider.Setup(provider => provider.CreateClient(It.IsAny())).Returns(serviceClient.Object); + var provider = new StorageAccountClientProvider(ConnectionString); + var settings = new AzureStorageOrchestrationServiceSettings + { + StorageAccountClientProvider = new StorageAccountClientProvider(provider.Blob, queueProvider.Object, provider.Table), + ControlQueueBufferThreshold = 3, + }; var client = new AzureStorageClient(settings); var messageManager = new MessageManager(settings, client, "buffer-largemessages"); - using var queue = new ControlQueue(client, queueClient.Object.Name, messageManager); + using var queue = new ControlQueue(client, QueueName, messageManager); using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); string payload = await messageManager.SerializeMessageDataAsync(CreateMessageData("valid")); @@ -225,72 +238,6 @@ public async Task ControlQueue_FailedBatch_ReleasesOnlyNewBufferEntries() Assert.AreEqual(2, receiveCalls, "The failed batch must not exhaust the buffer and block the next receive."); } - [DataTestMethod] - [DataRow(false)] - [DataRow(true)] - public async Task DequeueLoop_DeduplicationFailure_ReleasesOnlyNewBufferEntries(bool activeSession) - { - var settings = TestHelpers.GetMockQueueSettings(out var queueClient); - var client = new AzureStorageClient(settings); - var messageManager = new MessageManager(settings, client, "buffer-largemessages"); - using var queue = new ControlQueue(client, queueClient.Object.Name, messageManager); - using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - MessageData message = CreateMessageData("start"); - message.TaskMessage.Event = new ExecutionStartedEvent(-1, string.Empty) - { - Name = "orchestration", - OrchestrationInstance = message.TaskMessage.OrchestrationInstance, - }; - DateTimeOffset now = DateTimeOffset.UtcNow; - string originalPayload = await messageManager.SerializeMessageDataAsync(CreateMessageData("earlier")); - QueueMessage originalQueueMessage = QueuesModelFactory.QueueMessage( - "earlier", "original-receipt", originalPayload, 1, now, now.AddDays(1), now.AddMinutes(5)); - QueueMessage duplicateQueueMessage = QueuesModelFactory.QueueMessage( - "earlier", "duplicate-receipt", originalPayload, 2, now, now.AddDays(1), now.AddMinutes(5)); - QueueMessage queueMessage = QueuesModelFactory.QueueMessage( - "start", "receipt", await messageManager.SerializeMessageDataAsync(message), 1, - now, now.AddDays(1), now.AddMinutes(5)); - queueClient.SetupSequence(client => client.ReceiveMessagesAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(Response.FromValue(new[] { originalQueueMessage }, Mock.Of())) - .ReturnsAsync(Response.FromValue(new[] { duplicateQueueMessage, queueMessage }, Mock.Of())); - - var trackingStore = new Mock(MockBehavior.Strict); - trackingStore.Setup(store => store.GetHistoryEventsAsync( - It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(new OrchestrationHistory(Array.Empty())); - trackingStore.Setup(store => store.FetchInstanceStatusAsync( - It.IsAny>(), It.IsAny())) - .Callback(() => queue.Release(null, "Stop after the simulated deduplication failure.")) - .Throws(new InvalidOperationException("Simulated instance status lookup failure.")); - using var manager = new OrchestrationSessionManager( - client.QueueAccountName, settings, client.Stats, trackingStore.Object); - IReadOnlyList originalBatch = await queue.GetMessagesAsync(timeout.Token); - MessageData originalMessage = originalBatch.Single(); - manager.AddMessageToPendingOrchestration(queue, originalBatch, Guid.NewGuid(), timeout.Token); - OrchestrationSession? originalSession = activeSession - ? await manager.GetNextSessionAsync(entitiesOnly: false, timeout.Token) - ?? throw new InvalidOperationException("The original session was not acquired.") - : null; - MethodInfo dequeueLoop = typeof(OrchestrationSessionManager).GetMethod( - "DequeueLoop", BindingFlags.Instance | BindingFlags.NonPublic) - ?? throw new MissingMethodException("DequeueLoop"); - - await (Task)(dequeueLoop.Invoke(manager, new object[] { queue.Name, queue, timeout.Token }) - ?? throw new InvalidOperationException("The dequeue loop did not return a task.")); - - CollectionAssert.AreEquivalent(new[] { "earlier" }, client.Stats.PendingOrchestratorMessages.Keys.ToArray()); - manager.GetStats(out int pendingInstances, out _, out int activeSessions); - Assert.AreEqual(activeSession ? 0 : 1, pendingInstances); - Assert.AreEqual(activeSession ? 1 : 0, activeSessions); - OrchestrationSession retainedSession = originalSession - ?? await manager.GetNextSessionAsync(entitiesOnly: false, timeout.Token) - ?? throw new InvalidOperationException("The original message's session was lost."); - Assert.AreSame(originalMessage, retainedSession.CurrentMessageBatch.Single()); - trackingStore.Verify(store => store.FetchInstanceStatusAsync( - It.IsAny>(), It.IsAny()), Times.Once); - } - [DataTestMethod] [DataRow("blob.bin", "blob.bin")] [DataRow("@#$%!", "%40%23%24%25%21")] diff --git a/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs b/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs index 6e2100279..dbe582b80 100644 --- a/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs +++ b/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs @@ -255,41 +255,6 @@ public async Task PrefetchRetry_HistoryAlreadyLoaded_DoesNotLoadHistoryAgain() Assert.AreSame(history.Events[0], session.RuntimeState.ExecutionStartedEvent); } - [DataTestMethod] - [DataRow(false)] - [DataRow(true)] - public async Task PrefetchRetry_StorageTimeout_RetriesWithoutLosingBatch(bool timeoutFetchingInstanceStatus) - { - this.settings.UseInstanceTableEtag = timeoutFetchingInstanceStatus; - OrchestrationHistory history = CreateHistory(); - var instanceETag = new ETag("instance-etag"); - var trackingStore = new FakeTrackingStore - { - OnGetHistory = attempt => !timeoutFetchingInstanceStatus && attempt == 1 - ? throw new OperationCanceledException("Simulated history request timeout.") - : Task.FromResult(history), - OnFetchInstanceStatus = attempt => attempt == 1 - ? throw new TaskCanceledException("Simulated instance status request timeout.") - : Task.FromResult(new InstanceStatus(new OrchestrationState(), instanceETag)), - }; - MessageData message = this.CreateMessage(); - using var manager = this.CreateManager(trackingStore); - using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - manager.AddMessageToPendingOrchestration(this.controlQueue, new[] { message }, Guid.NewGuid(), timeout.Token); - - OrchestrationSession session = await manager.GetNextSessionAsync(entitiesOnly: false, timeout.Token); - - Assert.IsNotNull(session, "A storage timeout must not strand the buffered message."); - Assert.AreEqual(timeoutFetchingInstanceStatus ? 1 : 2, trackingStore.HistoryCalls); - Assert.AreEqual(timeoutFetchingInstanceStatus ? 2 : 0, trackingStore.InstanceStatusCalls); - Assert.AreSame(message, session.CurrentMessageBatch.Single()); - Assert.AreSame(history.Events[0], session.RuntimeState.ExecutionStartedEvent); - if (timeoutFetchingInstanceStatus) - { - Assert.AreEqual(instanceETag, session.ETags.InstanceETag); - } - } - [MethodImpl(MethodImplOptions.NoInlining)] static bool[] GetLiveStatesAfterFullCollection(WeakReference[] references) { diff --git a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs index e31e01333..7ed63f64d 100644 --- a/test/DurableTask.AzureStorage.Tests/TestHelpers.cs +++ b/test/DurableTask.AzureStorage.Tests/TestHelpers.cs @@ -13,60 +13,17 @@ #nullable enable namespace DurableTask.AzureStorage.Tests { - using Azure; - using Azure.Storage.Queues; - using Azure.Storage.Queues.Models; using DurableTask.Core.Logging; using DurableTask.Core.Settings; using Microsoft.Extensions.Logging; - using Moq; using System; using System.Configuration; using System.Diagnostics; using System.Diagnostics.Tracing; - using System.Threading; using System.Threading.Tasks; static class TestHelpers { - public static AzureStorageOrchestrationServiceSettings GetMockQueueSettings(out Mock queueClient) - { - const string ConnectionString = "UseDevelopmentStorage=true"; - const string TaskHubName = "BufferFailureTests"; - string queueName = AzureStorageOrchestrationService.GetControlQueueName(TaskHubName, 0); - - queueClient = new Mock(MockBehavior.Strict, ConnectionString, queueName); - queueClient.SetupGet(client => client.Name).Returns(queueName); - queueClient.SetupGet(client => client.Uri).Returns(new Uri($"http://127.0.0.1:10001/devstoreaccount1/{queueName}")); - queueClient.Setup(client => client.UpdateMessageAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .ReturnsAsync(Response.FromValue( - QueuesModelFactory.UpdateReceipt("retry-receipt", DateTimeOffset.UtcNow.AddMinutes(1)), - Mock.Of())); - - var serviceClient = new Mock(MockBehavior.Strict, ConnectionString); - serviceClient.SetupGet(client => client.AccountName).Returns("devstoreaccount1"); - serviceClient.Setup(client => client.GetQueueClient(It.IsAny())).Returns(queueClient.Object); - - var queueProvider = new Mock>(); - queueProvider.Setup(provider => provider.CreateOptions()).Returns(() => new QueueClientOptions()); - queueProvider.Setup(provider => provider.CreateClient(It.IsAny())).Returns(serviceClient.Object); - - return new AzureStorageOrchestrationServiceSettings - { - TaskHubName = TaskHubName, - PartitionCount = 1, - StorageAccountClientProvider = new StorageAccountClientProvider( - StorageServiceClientProvider.ForBlob(ConnectionString), - queueProvider.Object, - StorageServiceClientProvider.ForTable(ConnectionString)), - }; - } - public static TestOrchestrationHost GetTestOrchestrationHost( bool enableExtendedSessions, int extendedSessionTimeoutInSeconds = 30,