diff --git a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs index 1c56f7fa..b4072bfb 100644 --- a/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs +++ b/src/DurableTask.AzureStorage/Messaging/ControlQueue.cs @@ -75,6 +75,7 @@ public async Task> GetMessagesAsync(CancellationToken } pendingOrchestratorMessageLimitReached = false; + var addedMessageIds = new ConcurrentBag(); try { @@ -134,7 +135,11 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } // Check to see whether we've already dequeued this message. - if (!this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1)) + if (this.stats.PendingOrchestratorMessages.TryAdd(queueMessage.MessageId, 1)) + { + addedMessageIds.Add(queueMessage.MessageId); + } + else { // This message is already loaded in memory and is therefore a duplicate. // We will continue to process it because we need the updated pop receipt. @@ -170,6 +175,12 @@ await batch.ParallelForEachAsync(async delegate (QueueMessage queueMessage) } catch (Exception e) { + // This batch was not handed off. Keep entries from earlier batches. + foreach (string messageId in addedMessageIds) + { + this.stats.PendingOrchestratorMessages.TryRemove(messageId, out _); + } + if (!linkedCts.IsCancellationRequested) { this.settings.Logger.MessageFailure( diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index 855f68d6..4e2f76a2 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -13,8 +13,12 @@ #nullable enable namespace DurableTask.AzureStorage.Tests { + using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; + using Azure.Storage.Queues; + using Azure.Storage.Queues.Models; + using DurableTask.AzureStorage.Messaging; using DurableTask.AzureStorage.Storage; using DurableTask.Core; using DurableTask.Core.History; @@ -26,6 +30,7 @@ namespace DurableTask.AzureStorage.Tests using System.Collections.Generic; using System.Linq; using System.Text; + using System.Threading; using System.Threading.Tasks; [TestClass] @@ -176,6 +181,63 @@ public async Task SerializeMessageDataAsync_UsesExpectedStoragePathAtThreshold( } } + [TestMethod] + public async Task ControlQueue_FailedBatch_ReleasesOnlyNewBufferEntries() + { + const string ConnectionString = "UseDevelopmentStorage=true"; + const string QueueName = "buffer-control-00"; + var queueClient = new Mock(MockBehavior.Strict, ConnectionString, QueueName); + queueClient.SetupGet(client => client.Name).Returns(QueueName); + var serviceClient = new Mock(MockBehavior.Strict, ConnectionString); + serviceClient.SetupGet(client => client.AccountName).Returns("devstoreaccount1"); + serviceClient.Setup(client => client.GetQueueClient(QueueName)).Returns(queueClient.Object); + var queueProvider = new Mock>(MockBehavior.Strict); + queueProvider.Setup(provider => provider.CreateOptions()).Returns(() => new QueueClientOptions()); + queueProvider.Setup(provider => provider.CreateClient(It.IsAny())).Returns(serviceClient.Object); + var provider = new StorageAccountClientProvider(ConnectionString); + var settings = new AzureStorageOrchestrationServiceSettings + { + StorageAccountClientProvider = new StorageAccountClientProvider(provider.Blob, queueProvider.Object, provider.Table), + ControlQueueBufferThreshold = 3, + }; + var client = new AzureStorageClient(settings); + var messageManager = new MessageManager(settings, client, "buffer-largemessages"); + using var queue = new ControlQueue(client, QueueName, messageManager); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + string payload = await messageManager.SerializeMessageDataAsync(CreateMessageData("valid")); + MessageData malformedMessage = CreateMessageData("malformed"); + malformedMessage.TaskMessage.Event = null; + string malformedPayload = await messageManager.SerializeMessageDataAsync(malformedMessage); + DateTimeOffset now = DateTimeOffset.UtcNow; + QueueMessage[] batch = + { + QueuesModelFactory.QueueMessage("earlier", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + QueuesModelFactory.QueueMessage("new", "receipt", payload, 1, now, now.AddDays(1), now.AddMinutes(5)), + QueuesModelFactory.QueueMessage("malformed", "receipt", malformedPayload, 1, now, now.AddDays(1), now.AddMinutes(5)), + }; + client.Stats.PendingOrchestratorMessages.TryAdd("earlier", 1); + int receiveCalls = 0; + queueClient.Setup(client => client.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + if (++receiveCalls == 1) + { + return Task.FromResult(Response.FromValue(batch, Mock.Of())); + } + + timeout.Cancel(); + return Task.FromResult(Response.FromValue(Array.Empty(), Mock.Of())); + }); + + IReadOnlyList messages = await queue.GetMessagesAsync(timeout.Token); + + Assert.AreEqual(0, messages.Count, "A failed batch must not be handed off."); + CollectionAssert.AreEquivalent(new[] { "earlier" }, client.Stats.PendingOrchestratorMessages.Keys.ToArray()); + Assert.AreEqual(2, receiveCalls, "The failed batch must not exhaust the buffer and block the next receive."); + } + [DataTestMethod] [DataRow("blob.bin", "blob.bin")] [DataRow("@#$%!", "%40%23%24%25%21")]