From 634d31f669df2227c325e16a1e1dfb18bff814a4 Mon Sep 17 00:00:00 2001 From: Benjamin Cederholm Date: Mon, 16 Nov 2026 20:26:38 +0100 Subject: [PATCH 1/2] Don't delete blobs of other instances when purging an instance whose id is their prefix --- .../MessageManager.cs | 5 ++- .../AzureStorageScenarioTests.cs | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 723e5e69..48a5984c 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -364,9 +364,12 @@ public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance { int storageOperationCount = 0; + // The trailing '/' keeps the prefix from also matching instances whose id starts with this one. + string blobPrefix = sanitizedInstanceId + "/"; + try { - await foreach (Page page in this.blobContainer.ListBlobsAsync(sanitizedInstanceId, cancellationToken).AsPages()) + await foreach (Page page in this.blobContainer.ListBlobsAsync(blobPrefix, cancellationToken).AsPages()) { storageOperationCount++; diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 4b1113f4..a0552ac0 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -788,6 +788,43 @@ public async Task PurgeSingleInstance_WithLargeBlobs_CleansUpBlobs() } } + [TestMethod] + public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId() + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); + + string instanceId = Guid.NewGuid().ToString("N"); + string otherInstanceId = instanceId + "_other"; + string largeMessage = new string('x', 70 * 1024); + + TestOrchestrationClient client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Echo), largeMessage, instanceId); + OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + TestOrchestrationClient otherClient = await host.StartOrchestrationAsync( + typeof(Orchestrations.Echo), largeMessage, otherInstanceId); + status = await otherClient.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + int otherBlobCount = await this.GetBlobCount("test-largemessages", otherInstanceId + "/"); + Assert.IsTrue(otherBlobCount > 0, "Should have large message blobs before purge"); + + await client.PurgeInstanceHistory(); + + Assert.AreEqual(0, await this.GetBlobCount("test-largemessages", instanceId + "/")); + Assert.AreEqual(otherBlobCount, await this.GetBlobCount("test-largemessages", otherInstanceId + "/")); + + // Loading the history downloads the large-property blobs, as a replay does. + List otherHistoryEvents = await otherClient.GetOrchestrationHistoryAsync(otherInstanceId); + Assert.IsTrue(otherHistoryEvents.Count > 0); + + await host.StopAsync(); + } + } + [TestMethod] public async Task PurgeInstance_WithManyHistoryRows_DeletesAll() { From 95528d99555d1a75f9f8c35ab3da1b48fb362988 Mon Sep 17 00:00:00 2001 From: Benjamin Cederholm Date: Mon, 16 Nov 2026 20:48:20 +0100 Subject: [PATCH 2/2] Match message and escaped history blob prefixes separately and skip nested ids when purging --- .../MessageManager.cs | 39 +++++++++-- .../Tracking/AzureTableTrackingStore.cs | 2 +- .../AzureStorageScenarioTests.cs | 66 +++++++++++++++++-- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 48a5984c..081a2491 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -29,6 +29,7 @@ namespace DurableTask.AzureStorage using Azure.Storage.Blobs.Models; using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; + using DurableTask.AzureStorage.Tracking; using Newtonsoft.Json; #if NETSTANDARD2_0 using Newtonsoft.Json.Serialization; @@ -43,6 +44,8 @@ class MessageManager // increasing the size by a factor of 4/3. const int MaxStorageQueuePayloadSizeInBytes = 45 * 1024; // 45KB const int DefaultBufferSize = 64 * 2014; // 64KB + internal const string MessageBlobKind = "message-"; + internal const string HistoryBlobKind = "history-"; readonly AzureStorageOrchestrationServiceSettings settings; readonly AzureStorageClient azureStorageClient; @@ -357,15 +360,29 @@ public string GetNewLargeMessageBlobName(MessageData message) string eventType = message.TaskMessage.Event.EventType.ToString(); string activityId = message.ActivityId.ToString("N"); - return $"{instanceId}/message-{activityId}-{eventType}.json.gz"; + return $"{instanceId}/{MessageBlobKind}{activityId}-{eventType}.json.gz"; } - public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, CancellationToken cancellationToken = default) + public async Task DeleteLargeMessageBlobs(string instanceId, CancellationToken cancellationToken = default) { - int storageOperationCount = 0; + // Message blobs are named "{instanceId}/message-..." (GetNewLargeMessageBlobName) and history blobs + // "{escaped instanceId}/history-..." (AzureTableTrackingStore.GetBlobName). Only blobs directly under + // each prefix are deleted, so instances whose id starts with this one (e.g. "abc_other" or "abc/other" + // when purging "abc") keep their blobs. + string escapedInstanceId = KeySanitation.EscapePartitionKey(instanceId); + if (escapedInstanceId == instanceId) + { + return await this.DeleteInstanceBlobs(instanceId + "/", new[] { MessageBlobKind, HistoryBlobKind }, cancellationToken); + } + + int messageBlobOperations = await this.DeleteInstanceBlobs(instanceId + "/", new[] { MessageBlobKind }, cancellationToken); + int historyBlobOperations = await this.DeleteInstanceBlobs(escapedInstanceId + "/", new[] { HistoryBlobKind }, cancellationToken); + return messageBlobOperations + historyBlobOperations; + } - // The trailing '/' keeps the prefix from also matching instances whose id starts with this one. - string blobPrefix = sanitizedInstanceId + "/"; + async Task DeleteInstanceBlobs(string blobPrefix, string[] blobKinds, CancellationToken cancellationToken) + { + int storageOperationCount = 0; try { @@ -373,9 +390,10 @@ public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance { storageOperationCount++; - await Task.WhenAll(page.Values.Select(b => b.DeleteIfExistsAsync(cancellationToken))); + Blob[] instanceBlobs = page.Values.Where(b => IsInstanceBlob(b.Name, blobPrefix, blobKinds)).ToArray(); + await Task.WhenAll(instanceBlobs.Select(b => b.DeleteIfExistsAsync(cancellationToken))); - storageOperationCount += page.Values.Count; + storageOperationCount += instanceBlobs.Length; } // Count the list operation even if no blobs found (the initial list request still happened) @@ -398,6 +416,13 @@ public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance return storageOperationCount; } + static bool IsInstanceBlob(string blobName, string blobPrefix, string[] blobKinds) + { + string relativeName = blobName.Substring(blobPrefix.Length); + return relativeName.IndexOf('/') < 0 + && blobKinds.Any(kind => relativeName.StartsWith(kind, StringComparison.Ordinal)); + } + /// Escapes a blob name for safe inclusion in a URI while preserving path structure. static string EscapeBlobNamePreservingSlashes(string blobName) { diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 4caffea8..daae65fb 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -1483,7 +1483,7 @@ static string GetBlobName(TableEntity entity, string property) // randomize the blob name to prevent accidental races in split-brain situations (#890) uint random = (uint)(new Random()).Next(); - return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz"; + return $"{sanitizedInstanceId}/{MessageManager.HistoryBlobKind}{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz"; } async Task GetHistoryETagAsync(string instanceId, string executionId, CancellationToken cancellationToken) diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index a0552ac0..16a0a2ed 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -788,15 +788,18 @@ public async Task PurgeSingleInstance_WithLargeBlobs_CleansUpBlobs() } } - [TestMethod] - public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId() + [DataTestMethod] + [DataRow("_other")] + [DataRow(":12")] + [DataRow("/other")] + public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId(string otherIdSuffix) { using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) { await host.StartAsync(); string instanceId = Guid.NewGuid().ToString("N"); - string otherInstanceId = instanceId + "_other"; + string otherInstanceId = instanceId + otherIdSuffix; string largeMessage = new string('x', 70 * 1024); TestOrchestrationClient client = await host.StartOrchestrationAsync( @@ -809,13 +812,14 @@ public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId status = await otherClient.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); - int otherBlobCount = await this.GetBlobCount("test-largemessages", otherInstanceId + "/"); - Assert.IsTrue(otherBlobCount > 0, "Should have large message blobs before purge"); + (int otherMessageBlobs, int otherHistoryBlobs) = await this.GetInstanceBlobCounts(otherInstanceId); + Assert.IsTrue(otherMessageBlobs > 0, "Should have large message blobs before purge"); + Assert.IsTrue(otherHistoryBlobs > 0, "Should have large history blobs before purge"); await client.PurgeInstanceHistory(); - Assert.AreEqual(0, await this.GetBlobCount("test-largemessages", instanceId + "/")); - Assert.AreEqual(otherBlobCount, await this.GetBlobCount("test-largemessages", otherInstanceId + "/")); + Assert.AreEqual((0, 0), await this.GetInstanceBlobCounts(instanceId)); + Assert.AreEqual((otherMessageBlobs, otherHistoryBlobs), await this.GetInstanceBlobCounts(otherInstanceId)); // Loading the history downloads the large-property blobs, as a replay does. List otherHistoryEvents = await otherClient.GetOrchestrationHistoryAsync(otherInstanceId); @@ -825,6 +829,54 @@ public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId } } + [DataTestMethod] + [DataRow("^")] + [DataRow("/nested")] + public async Task PurgeSingleInstance_WithEscapedInstanceId_DeletesAllBlobs(string idSuffix) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); + + string instanceId = Guid.NewGuid().ToString("N") + idSuffix; + string largeMessage = new string('x', 70 * 1024); + + TestOrchestrationClient client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Echo), largeMessage, instanceId); + OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + (int messageBlobs, int historyBlobs) = await this.GetInstanceBlobCounts(instanceId); + Assert.IsTrue(messageBlobs > 0, "Should have large message blobs before purge"); + Assert.IsTrue(historyBlobs > 0, "Should have large history blobs before purge"); + + await client.PurgeInstanceHistory(); + + Assert.AreEqual((0, 0), await this.GetInstanceBlobCounts(instanceId)); + + await host.StopAsync(); + } + } + + // Counts the blobs that belong to exactly this instance: message blobs are named after the instance id, + // history blobs after the escaped instance id, and blobs of longer ids nested below them are not counted. + async Task<(int MessageBlobs, int HistoryBlobs)> GetInstanceBlobCounts(string instanceId) + { + var containerClient = new BlobServiceClient(TestHelpers.GetTestStorageAccountConnectionString()) + .GetBlobContainerClient("test-largemessages"); + await containerClient.CreateIfNotExistsAsync(); + + async Task CountAsync(string prefix) + { + return await containerClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, CancellationToken.None) + .CountAsync(b => b.Name.IndexOf('/', prefix.Length) < 0); + } + + return ( + await CountAsync(instanceId + "/message-"), + await CountAsync(KeySanitation.EscapePartitionKey(instanceId) + "/history-")); + } + [TestMethod] public async Task PurgeInstance_WithManyHistoryRows_DeletesAll() {