diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 723e5e69..081a2491 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -29,6 +29,7 @@ namespace DurableTask.AzureStorage using Azure.Storage.Blobs.Models; using Azure.Storage.Queues.Models; using DurableTask.AzureStorage.Storage; + using DurableTask.AzureStorage.Tracking; using Newtonsoft.Json; #if NETSTANDARD2_0 using Newtonsoft.Json.Serialization; @@ -43,6 +44,8 @@ class MessageManager // increasing the size by a factor of 4/3. const int MaxStorageQueuePayloadSizeInBytes = 45 * 1024; // 45KB const int DefaultBufferSize = 64 * 2014; // 64KB + internal const string MessageBlobKind = "message-"; + internal const string HistoryBlobKind = "history-"; readonly AzureStorageOrchestrationServiceSettings settings; readonly AzureStorageClient azureStorageClient; @@ -357,22 +360,40 @@ public string GetNewLargeMessageBlobName(MessageData message) string eventType = message.TaskMessage.Event.EventType.ToString(); string activityId = message.ActivityId.ToString("N"); - return $"{instanceId}/message-{activityId}-{eventType}.json.gz"; + return $"{instanceId}/{MessageBlobKind}{activityId}-{eventType}.json.gz"; } - public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, CancellationToken cancellationToken = default) + public async Task DeleteLargeMessageBlobs(string instanceId, CancellationToken cancellationToken = default) + { + // Message blobs are named "{instanceId}/message-..." (GetNewLargeMessageBlobName) and history blobs + // "{escaped instanceId}/history-..." (AzureTableTrackingStore.GetBlobName). Only blobs directly under + // each prefix are deleted, so instances whose id starts with this one (e.g. "abc_other" or "abc/other" + // when purging "abc") keep their blobs. + string escapedInstanceId = KeySanitation.EscapePartitionKey(instanceId); + if (escapedInstanceId == instanceId) + { + return await this.DeleteInstanceBlobs(instanceId + "/", new[] { MessageBlobKind, HistoryBlobKind }, cancellationToken); + } + + int messageBlobOperations = await this.DeleteInstanceBlobs(instanceId + "/", new[] { MessageBlobKind }, cancellationToken); + int historyBlobOperations = await this.DeleteInstanceBlobs(escapedInstanceId + "/", new[] { HistoryBlobKind }, cancellationToken); + return messageBlobOperations + historyBlobOperations; + } + + async Task DeleteInstanceBlobs(string blobPrefix, string[] blobKinds, CancellationToken cancellationToken) { int storageOperationCount = 0; try { - await foreach (Page page in this.blobContainer.ListBlobsAsync(sanitizedInstanceId, cancellationToken).AsPages()) + await foreach (Page page in this.blobContainer.ListBlobsAsync(blobPrefix, cancellationToken).AsPages()) { storageOperationCount++; - await Task.WhenAll(page.Values.Select(b => b.DeleteIfExistsAsync(cancellationToken))); + Blob[] instanceBlobs = page.Values.Where(b => IsInstanceBlob(b.Name, blobPrefix, blobKinds)).ToArray(); + await Task.WhenAll(instanceBlobs.Select(b => b.DeleteIfExistsAsync(cancellationToken))); - storageOperationCount += page.Values.Count; + storageOperationCount += instanceBlobs.Length; } // Count the list operation even if no blobs found (the initial list request still happened) @@ -395,6 +416,13 @@ public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance return storageOperationCount; } + static bool IsInstanceBlob(string blobName, string blobPrefix, string[] blobKinds) + { + string relativeName = blobName.Substring(blobPrefix.Length); + return relativeName.IndexOf('/') < 0 + && blobKinds.Any(kind => relativeName.StartsWith(kind, StringComparison.Ordinal)); + } + /// Escapes a blob name for safe inclusion in a URI while preserving path structure. static string EscapeBlobNamePreservingSlashes(string blobName) { diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 4caffea8..daae65fb 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -1483,7 +1483,7 @@ static string GetBlobName(TableEntity entity, string property) // randomize the blob name to prevent accidental races in split-brain situations (#890) uint random = (uint)(new Random()).Next(); - return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz"; + return $"{sanitizedInstanceId}/{MessageManager.HistoryBlobKind}{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz"; } async Task GetHistoryETagAsync(string instanceId, string executionId, CancellationToken cancellationToken) diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 4b1113f4..16a0a2ed 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -788,6 +788,95 @@ public async Task PurgeSingleInstance_WithLargeBlobs_CleansUpBlobs() } } + [DataTestMethod] + [DataRow("_other")] + [DataRow(":12")] + [DataRow("/other")] + public async Task PurgeSingleInstance_DoesNotDeleteBlobsOfInstanceWithPrefixedId(string otherIdSuffix) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); + + string instanceId = Guid.NewGuid().ToString("N"); + string otherInstanceId = instanceId + otherIdSuffix; + string largeMessage = new string('x', 70 * 1024); + + TestOrchestrationClient client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Echo), largeMessage, instanceId); + OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + TestOrchestrationClient otherClient = await host.StartOrchestrationAsync( + typeof(Orchestrations.Echo), largeMessage, otherInstanceId); + status = await otherClient.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + (int otherMessageBlobs, int otherHistoryBlobs) = await this.GetInstanceBlobCounts(otherInstanceId); + Assert.IsTrue(otherMessageBlobs > 0, "Should have large message blobs before purge"); + Assert.IsTrue(otherHistoryBlobs > 0, "Should have large history blobs before purge"); + + await client.PurgeInstanceHistory(); + + Assert.AreEqual((0, 0), await this.GetInstanceBlobCounts(instanceId)); + Assert.AreEqual((otherMessageBlobs, otherHistoryBlobs), await this.GetInstanceBlobCounts(otherInstanceId)); + + // Loading the history downloads the large-property blobs, as a replay does. + List otherHistoryEvents = await otherClient.GetOrchestrationHistoryAsync(otherInstanceId); + Assert.IsTrue(otherHistoryEvents.Count > 0); + + await host.StopAsync(); + } + } + + [DataTestMethod] + [DataRow("^")] + [DataRow("/nested")] + public async Task PurgeSingleInstance_WithEscapedInstanceId_DeletesAllBlobs(string idSuffix) + { + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); + + string instanceId = Guid.NewGuid().ToString("N") + idSuffix; + string largeMessage = new string('x', 70 * 1024); + + TestOrchestrationClient client = await host.StartOrchestrationAsync( + typeof(Orchestrations.Echo), largeMessage, instanceId); + OrchestrationState status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + (int messageBlobs, int historyBlobs) = await this.GetInstanceBlobCounts(instanceId); + Assert.IsTrue(messageBlobs > 0, "Should have large message blobs before purge"); + Assert.IsTrue(historyBlobs > 0, "Should have large history blobs before purge"); + + await client.PurgeInstanceHistory(); + + Assert.AreEqual((0, 0), await this.GetInstanceBlobCounts(instanceId)); + + await host.StopAsync(); + } + } + + // Counts the blobs that belong to exactly this instance: message blobs are named after the instance id, + // history blobs after the escaped instance id, and blobs of longer ids nested below them are not counted. + async Task<(int MessageBlobs, int HistoryBlobs)> GetInstanceBlobCounts(string instanceId) + { + var containerClient = new BlobServiceClient(TestHelpers.GetTestStorageAccountConnectionString()) + .GetBlobContainerClient("test-largemessages"); + await containerClient.CreateIfNotExistsAsync(); + + async Task CountAsync(string prefix) + { + return await containerClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, CancellationToken.None) + .CountAsync(b => b.Name.IndexOf('/', prefix.Length) < 0); + } + + return ( + await CountAsync(instanceId + "/message-"), + await CountAsync(KeySanitation.EscapePartitionKey(instanceId) + "/history-")); + } + [TestMethod] public async Task PurgeInstance_WithManyHistoryRows_DeletesAll() {