diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs index 54228ff3..4705fc9b 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs @@ -811,6 +811,7 @@ await this.AbandonAndReleaseSessionAsync( string warningMessage = await this.IsExecutableInstanceAsync( session.RuntimeState, + session.ETags.HistoryETag, orchestrationWorkItem.NewMessages, settings.AllowReplayingTerminalInstances, cancellationToken); @@ -880,6 +881,15 @@ await this.AbandonAndReleaseSessionAsync( return orchestrationWorkItem; } + catch (DurableTaskStorageException exception) when (exception.HttpStatusCode == 409 || exception.HttpStatusCode == 412) + { + // Reload changed state without losing the message that woke this instance. + if (session != null) + { + await this.AbandonAndReleaseSessionAsync(session, "Instance or history changed during status recovery."); + } + return null; + } catch (OperationCanceledException) { if (session != null) @@ -1082,6 +1092,7 @@ internal static void TraceMessageReceived(AzureStorageOrchestrationServiceSettin async Task IsExecutableInstanceAsync( OrchestrationRuntimeState runtimeState, + ETag? historyETag, IList newMessages, bool allowReplayingTerminalInstances, CancellationToken cancellationToken) @@ -1122,7 +1133,7 @@ await this.trackingStore.UpdateStatusForTerminationAsync( runtimeState.OrchestrationStatus != OrchestrationStatus.Pending && runtimeState.OrchestrationStatus != OrchestrationStatus.Suspended) { - InstanceStatus instanceStatus = await this.trackingStore.FetchInstanceStatusAsync(runtimeState.OrchestrationInstance.InstanceId); + InstanceStatus instanceStatus = await this.trackingStore.FetchInstanceStatusAsync(runtimeState.OrchestrationInstance.InstanceId, cancellationToken); if (instanceStatus == null || (instanceStatus.State.OrchestrationInstance.ExecutionId == runtimeState.OrchestrationInstance.ExecutionId && instanceStatus.State.OrchestrationStatus != runtimeState.OrchestrationStatus)) { @@ -1130,7 +1141,7 @@ await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( runtimeState.OrchestrationInstance.InstanceId, runtimeState.OrchestrationInstance.ExecutionId, runtimeState, - instanceStatus is not null, + new OrchestrationETags { HistoryETag = historyETag, InstanceETag = instanceStatus?.ETag }, cancellationToken); } if (!allowReplayingTerminalInstances) diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs index 347b9b55..4caffea8 100644 --- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs @@ -285,14 +285,44 @@ public override async IAsyncEnumerable RewindHistoryAsync(string instanc bool hasFailedSubOrchestrations = false; string partitionFilter = AzureTableQueryFilter.PartitionKeyEquals(instanceId); - string orchestratorStartedFilter = $"{partitionFilter} and {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.OrchestratorStarted)}'"; - IReadOnlyList orchestratorStartedEntities = await this.QueryHistoryAsync(orchestratorStartedFilter, instanceId, cancellationToken); + // A child can report failure before its own checkpoint finishes. + // Wait for that checkpoint before reading or changing its history. + Stopwatch waitForFailure = Stopwatch.StartNew(); + string executionId = null; + ETag instanceETag; - // get most recent orchestratorStarted event - string recentStartRowKey = orchestratorStartedEntities.Max(x => x.RowKey); - var recentStartRow = orchestratorStartedEntities.Where(y => y.RowKey == recentStartRowKey).ToList(); - string executionId = recentStartRow[0].GetString(nameof(OrchestrationInstance.ExecutionId)); - DateTime instanceTimestamp = recentStartRow[0].Timestamp.GetValueOrDefault().DateTime; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + InstanceStatus instanceStatus = await this.FetchInstanceStatusAsync(instanceId, cancellationToken); + if (instanceStatus != null) + { + string currentExecutionId = instanceStatus.State.OrchestrationInstance.ExecutionId; + if (executionId != null && executionId != currentExecutionId) + { + throw new InvalidOperationException($"Cannot rewind instance '{instanceId}': execution '{executionId}' no longer exists."); + } + executionId = currentExecutionId; + if (instanceStatus.State.OrchestrationStatus == OrchestrationStatus.Failed) + { + instanceETag = instanceStatus.ETag; + break; + } + } + else if (executionId != null) + { + throw new InvalidOperationException($"Cannot rewind instance '{instanceId}': execution '{executionId}' no longer exists."); + } + + if (waitForFailure.Elapsed >= TimeSpan.FromMinutes(1)) + { + throw new TimeoutException($"Timed out waiting for instance '{instanceId}' to commit its failed status before rewind."); + } + + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + } + + ETag? historyETag = await this.GetHistoryETagAsync(instanceId, executionId, cancellationToken); // Use parameterized filter to prevent OData injection via crafted execution IDs string executionIdFilter = AzureTableQueryFilter.ColumnEquals(nameof(OrchestrationInstance.ExecutionId), executionId); @@ -368,8 +398,22 @@ public override async IAsyncEnumerable RewindHistoryAsync(string instanc await this.HistoryTable.ReplaceEntityAsync(entity, entity.ETag, cancellationToken); } + // Invalidate prefetched history before making the instance Pending. + var sentinel = new TableEntity(KeySanitation.EscapePartitionKey(instanceId), SentinelRowKey) + { + [nameof(OrchestrationInstance.ExecutionId)] = executionId, + }; + if (historyETag.HasValue) + { + await this.HistoryTable.MergeEntityAsync(sentinel, historyETag.Value, cancellationToken); + } + else + { + await this.HistoryTable.InsertEntityAsync(sentinel, cancellationToken); + } + // reset orchestration status in instance store table - await this.UpdateStatusForRewindAsync(instanceId, cancellationToken); + await this.UpdateStatusForRewindAsync(instanceId, instanceETag, cancellationToken); if (!hasFailedSubOrchestrations) { @@ -854,7 +898,7 @@ public override async Task SetNewExecutionAsync( } /// - public override async Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default) + public override async Task UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken = default) { string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); TableEntity entity = new TableEntity(sanitizedInstanceId, "") @@ -864,7 +908,8 @@ public override async Task UpdateStatusForRewindAsync(string instanceId, Cancell }; Stopwatch stopwatch = Stopwatch.StartNew(); - await this.InstancesTable.MergeEntityAsync(entity, ETag.All, cancellationToken); + // Do not reset an instance that changed during rewind. + await this.InstancesTable.MergeEntityAsync(entity, eTag, cancellationToken); // We don't have enough information to get the episode number. // It's also not important to have for this particular trace. @@ -1138,7 +1183,7 @@ public override async Task UpdateInstanceStatusForCompletedOrchestrationAsync( string instanceId, string executionId, OrchestrationRuntimeState runtimeState, - bool instanceEntityExists, + OrchestrationETags eTags, CancellationToken cancellationToken = default) { if (runtimeState.OrchestrationStatus != OrchestrationStatus.Completed && @@ -1149,6 +1194,21 @@ public override async Task UpdateInstanceStatusForCompletedOrchestrationAsync( return; } + // There is a particular race condition which can occur with rewind if a worker crashes after updating the instance and history state + // to "Failed", but before deleting the triggering messages. If rewind is then requested on the failed instance, its status will be + // updated to "Pending", but if we happened to fetch the history BEFORE rewind altered it, then we will see a mismatch between the failed + // history and the instance status at this point. We do not actually want to set the status to "Failed" here in that case. The easiest way + // to detect this is to check if the history's etag is still current. + // (Note that there remains a known gap for a multi-page history if the sentinel is on a different page from the ExecutionCompletedEvent. Then + // it is possible we still fetch a failed history, the rewind goes through and changes the history and the sentinel's etag, and then we read + // the sentinel with its new etag and think the history is current) + if (eTags.HistoryETag.HasValue && + eTags.HistoryETag != await this.GetHistoryETagAsync(instanceId, executionId, cancellationToken)) + { + throw new DurableTaskStorageException(new RequestFailedException( + 412, "The history changed before its terminal status could be restored.", "UpdateConditionNotSatisfied", null)); + } + string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId); ExecutionStartedEvent executionStartedEvent = runtimeState.ExecutionStartedEvent; @@ -1214,7 +1274,7 @@ static TableEntity GetSingleEntityFromHistoryTableResults(IReadOnlyList GetHistoryETagAsync(string instanceId, string executionId, CancellationToken cancellationToken) + { + string filter = $"{AzureTableQueryFilter.PartitionKeyEquals(instanceId)} and {AzureTableQueryFilter.ColumnEquals(nameof(ITableEntity.RowKey), SentinelRowKey)}"; + TableQueryResults results = await this.HistoryTable.ExecuteQueryAsync( + filter, select: new[] { nameof(OrchestrationInstance.ExecutionId) }, cancellationToken: cancellationToken) + .GetResultsAsync(cancellationToken: cancellationToken); + TableEntity sentinel = results.Entities.SingleOrDefault(); + if (sentinel?.GetString(nameof(OrchestrationInstance.ExecutionId)) is string historyExecutionId && historyExecutionId != executionId) + { + throw new DurableTaskStorageException(new RequestFailedException( + 412, "The history belongs to a different execution.", "UpdateConditionNotSatisfied", null)); + } + return sentinel?.ETag; + } + async Task UploadHistoryBatch( string instanceId, string sanitizedInstanceId, diff --git a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs index 6fc2aa8b..68d85f2a 100644 --- a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs @@ -111,9 +111,9 @@ interface ITrackingStore /// The ID of the orchestration. /// The execution ID of the orchestration. /// The runtime state of the orchestration. - /// Whether the instance entity already exists in the instance store. + /// The history and instance ETags read before recovery. A missing instance ETag means the row did not exist. /// The token to monitor for cancellation requests. The default value is . - Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, bool instanceEntityExists, CancellationToken cancellationToken = default); + Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, OrchestrationETags eTags, CancellationToken cancellationToken = default); /// /// Get The Orchestration State for querying all orchestration instances @@ -161,8 +161,9 @@ interface ITrackingStore /// Used to update a state in the tracking store to pending whenever a rewind is initiated from the client /// /// The instance being rewound + /// The instance ETag read when its failed status was confirmed. /// The token to monitor for cancellation requests. The default value is . - Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default); + Task UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken = default); /// /// Used to update the instance status to "Terminated" when a pending orchestration is terminated. diff --git a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs index a7c43988..64f51394 100644 --- a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs +++ b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs @@ -194,7 +194,7 @@ public override async Task UpdateInstanceStatusForCompletedOrchestrationAsync( string instanceId, string executionId, OrchestrationRuntimeState runtimeState, - bool instanceEntityExists, + OrchestrationETags eTags, CancellationToken cancellationToken = default) { if (runtimeState.OrchestrationStatus != OrchestrationStatus.Completed && diff --git a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs index d02a729c..3c013546 100644 --- a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs +++ b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs @@ -95,7 +95,7 @@ public virtual Task PurgeInstanceHistoryAsync(DateTime creat public abstract Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, ETag? eTag, string inputStatusOverride, CancellationToken cancellationToken = default); /// - public virtual Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default) + public virtual Task UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } @@ -110,6 +110,6 @@ public virtual Task UpdateStatusForRewindAsync(string instanceId, CancellationTo public abstract Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, OrchestrationETags eTags, object executionData, CancellationToken cancellationToken = default); /// - public abstract Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, bool instanceEntityExists, CancellationToken cancellationToken = default); + public abstract Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, OrchestrationETags eTags, CancellationToken cancellationToken = default); } } diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index 90acbe99..4b1113f4 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -1505,6 +1505,332 @@ public async Task TerminatePendingOrchestration(bool enableExtendedSessions, boo } } + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task RewindRejectsPrefetchedFailedHistory(bool allowReplayingTerminalInstances) + { + using var service = CreateRewindRaceService(); + bool started = false; + await service.CreateAsync(); + try + { + // 1. Prefetch the failed history before rewind changes it. + await SaveRewindRaceHistoryAsync(service, "instance", failed: true); + OrchestrationHistory stale = await service.TrackingStore.GetHistoryEventsAsync("instance", "execution"); + var staleRuntime = new OrchestrationRuntimeState(stale.Events); + + // 2. Rewind and enqueue its wake-up message. + await service.RewindTaskOrchestrationAsync("instance", "retry"); + OrchestrationHistory rewound = await service.TrackingStore.GetHistoryEventsAsync("instance", "execution"); + Assert.AreNotEqual(stale.ETag, rewound.ETag, "Rewind must invalidate prefetched history."); + + // 3. The worker's recovery check must reject the old history and preserve Pending. + MethodInfo check = typeof(AzureStorageOrchestrationService).GetMethod( + "IsExecutableInstanceAsync", BindingFlags.Instance | BindingFlags.NonPublic); + var recovery = (Task)check.Invoke(service, new object[] + { + staleRuntime, stale.ETag, new List(), allowReplayingTerminalInstances, CancellationToken.None, + }); + DurableTaskStorageException error = await Assert.ThrowsExceptionAsync(async () => await recovery); + Assert.AreEqual(412, error.HttpStatusCode); + Assert.AreEqual(OrchestrationStatus.Pending, + (await service.TrackingStore.FetchInstanceStatusAsync("instance")).State.OrchestrationStatus); + + // 4. Fresh history still allows the queued rewind message to run. + await service.StartAsync(); + started = true; + using var timeout = new CancellationTokenSource(StandardTimeout); + TaskOrchestrationWorkItem workItem = await service.LockNextTaskOrchestrationWorkItemAsync(StandardTimeout, timeout.Token); + Assert.IsNotNull(workItem); + Assert.AreEqual(OrchestrationStatus.Running, workItem.OrchestrationRuntimeState.OrchestrationStatus); + Assert.IsTrue(workItem.NewMessages.Any(message => message.Event is GenericEvent)); + await service.AbandonTaskOrchestrationWorkItemAsync(workItem); + await service.ReleaseTaskOrchestrationWorkItemAsync(workItem); + } + finally + { + if (started) + { + await service.StopAsync(isForced: true); + } + await service.DeleteAsync(); + } + } + + [TestMethod] + public async Task RewindRejectsHistoryFromAnotherExecution() + { + using var service = CreateRewindRaceService(); + await service.CreateAsync(); + try + { + // 1. Leave the failed instance row behind a newer history execution. + await SaveRewindRaceHistoryAsync(service, "instance", failed: true); + var store = (AzureTableTrackingStore)service.TrackingStore; + await store.HistoryTable.MergeEntityAsync(new TableEntity("instance", "sentinel") + { + ["ExecutionId"] = "new-execution", + }, Azure.ETag.All); + OrchestrationHistory before = await store.GetHistoryEventsAsync("instance", "new-execution"); + + // 2. Rewind must reject the mismatch without changing either table. + DurableTaskStorageException error = await Assert.ThrowsExceptionAsync( + async () => await store.RewindHistoryAsync("instance").ToListAsync()); + Assert.AreEqual(412, error.HttpStatusCode); + Assert.AreEqual(before.ETag, (await store.GetHistoryEventsAsync("instance", "new-execution")).ETag); + Assert.AreEqual(OrchestrationStatus.Failed, (await store.FetchInstanceStatusAsync("instance")).State.OrchestrationStatus); + } + finally + { + await service.DeleteAsync(); + } + } + + [TestMethod] + public async Task RewindDoesNotResetANewExecutionAfterChangingHistory() + { + var settings = CreateRewindRaceSettings(); + using var service = new AzureStorageOrchestrationService(settings); + await service.CreateAsync(); + try + { + // 1. Save the failed execution that rewind will read. + await SaveRewindRaceHistoryAsync(service, "instance", failed: true); + InstanceStatus failed = await service.TrackingStore.FetchInstanceStatusAsync("instance"); + OrchestrationHistory original = await service.TrackingStore.GetHistoryEventsAsync("instance", "execution"); + InstanceStatus replacement = null; + OrchestrationHistory replacementHistory = null; + var client = new AzureStorageClient(settings); + var messages = new MessageManager(settings, client, $"{settings.TaskHubName.ToLowerInvariant()}-largemessages"); + var store = new RewindStatusTrackingStore(client, messages, async (instanceId, eTag, token) => + { + // 2. Commit a new execution after rewind changes history, before it writes Pending. + Assert.AreEqual(failed.ETag, eTag); + OrchestrationHistory rewound = await service.TrackingStore.GetHistoryEventsAsync(instanceId, "execution", token); + Assert.AreNotEqual(original.ETag, rewound.ETag); + var started = new ExecutionStartedEvent(-1, "new-input") + { + Name = "NewExecution", Version = string.Empty, + OrchestrationInstance = new OrchestrationInstance { InstanceId = instanceId, ExecutionId = "new-execution" }, + }; + Assert.IsTrue(await service.TrackingStore.SetNewExecutionAsync(started, eTag, inputPayloadOverride: null, cancellationToken: token)); + InstanceStatus pending = await service.TrackingStore.FetchInstanceStatusAsync(instanceId, token); + var runtime = new OrchestrationRuntimeState(); + runtime.AddEvent(new OrchestratorStartedEvent(-1)); + runtime.AddEvent(started); + runtime.AddEvent(new ExecutionCompletedEvent(0, "new-output", OrchestrationStatus.Completed)); + runtime.AddEvent(new OrchestratorCompletedEvent(-1)); + await service.TrackingStore.UpdateStateAsync(runtime, new OrchestrationRuntimeState(), instanceId, "new-execution", + new OrchestrationETags { HistoryETag = rewound.ETag, InstanceETag = pending.ETag }, rewound.TrackingStoreContext, token); + replacement = await service.TrackingStore.FetchInstanceStatusAsync(instanceId, token); + replacementHistory = await service.TrackingStore.GetHistoryEventsAsync(instanceId, "new-execution", token); + }); + + // 3. The old rewind must fail once and preserve the new execution. + DurableTaskStorageException error = await Assert.ThrowsExceptionAsync( + async () => await store.RewindHistoryAsync("instance").ToListAsync()); + Assert.AreEqual(412, error.HttpStatusCode); + Assert.AreEqual(1, store.StatusUpdateAttempts); + InstanceStatus actual = await service.TrackingStore.FetchInstanceStatusAsync("instance"); + Assert.AreEqual("new-execution", actual.State.OrchestrationInstance.ExecutionId); + Assert.AreEqual(OrchestrationStatus.Completed, actual.State.OrchestrationStatus); + Assert.AreEqual("new-output", actual.State.Output); + Assert.AreEqual(replacement.ETag, actual.ETag); + Assert.AreEqual(replacementHistory.ETag, + (await service.TrackingStore.GetHistoryEventsAsync("instance", "new-execution")).ETag); + } + finally + { + await service.DeleteAsync(); + } + } + + [TestMethod] + public async Task RewindRecoveryDoesNotOverwriteAChangedInstanceRow() + { + using var service = CreateRewindRaceService(); + await service.CreateAsync(); + try + { + // 1. Read the history and instance ETags for recovery. + await SaveRewindRaceHistoryAsync(service, "instance", failed: true); + OrchestrationHistory history = await service.TrackingStore.GetHistoryEventsAsync("instance", "execution"); + InstanceStatus status = await service.TrackingStore.FetchInstanceStatusAsync("instance"); + + // 2. Change only the instance row, as if rewind finished after the history check. + await service.TrackingStore.UpdateStatusForRewindAsync("instance", status.ETag); + DurableTaskStorageException error = await Assert.ThrowsExceptionAsync(() => + service.TrackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( + "instance", "execution", new OrchestrationRuntimeState(history.Events), + new OrchestrationETags { HistoryETag = history.ETag, InstanceETag = status.ETag })); + + // 3. The stale conditional write must leave the rewound status intact. + Assert.AreEqual(412, error.HttpStatusCode); + Assert.AreEqual(OrchestrationStatus.Pending, + (await service.TrackingStore.FetchInstanceStatusAsync("instance")).State.OrchestrationStatus); + } + finally + { + await service.DeleteAsync(); + } + } + + [DataTestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task RewindWaitsForChildFailureCheckpoint(bool childHasEarlierCheckpoint) + { + using var service = CreateRewindRaceService(); + await service.CreateAsync(); + using var timeout = new CancellationTokenSource(StandardTimeout); + Task> rewind = null; + try + { + // 1. The parent has failed, but the child's failed checkpoint is still missing. + await SaveRewindRaceHistoryAsync(service, "parent", failed: true, childInstanceId: "child"); + if (childHasEarlierCheckpoint) + { + await SaveRewindRaceHistoryAsync(service, "child", failed: false); + } + rewind = service.TrackingStore.RewindHistoryAsync("parent", timeout.Token).ToListAsync().AsTask(); + + // 2. Wait until parent rewind has reached the child. + var store = (AzureTableTrackingStore)service.TrackingStore; + Stopwatch wait = Stopwatch.StartNew(); + while (true) + { + TableQueryResults rows = await store.HistoryTable.ExecuteQueryAsync( + "PartitionKey eq 'parent'", cancellationToken: timeout.Token).GetResultsAsync(cancellationToken: timeout.Token); + if (rows.Entities.Any(row => row.GetString("Reason") == "Rewound: SubOrchestrationInstanceCreated")) + { + break; + } + Assert.IsTrue(wait.Elapsed < StandardTimeout); + await Task.Delay(50, timeout.Token); + } + Assert.IsFalse(rewind.IsCompleted, "Rewind must wait for the child's failed checkpoint."); + + // 3. Finish that checkpoint, then verify both histories were rewound. + await SaveRewindRaceHistoryAsync(service, "child", failed: true); + CollectionAssert.AreEqual(new[] { "child" }, await rewind); + foreach (string instanceId in new[] { "parent", "child" }) + { + Assert.AreEqual(OrchestrationStatus.Pending, + (await service.TrackingStore.FetchInstanceStatusAsync(instanceId)).State.OrchestrationStatus); + OrchestrationHistory history = await service.TrackingStore.GetHistoryEventsAsync(instanceId, "execution"); + Assert.IsFalse(history.Events.Any(e => e is TaskFailedEvent || e is SubOrchestrationInstanceFailedEvent || + e is ExecutionCompletedEvent completed && completed.OrchestrationStatus == OrchestrationStatus.Failed)); + } + } + finally + { + timeout.Cancel(); + if (rewind != null) + { + try { await rewind; } catch (OperationCanceledException) { } + } + await service.DeleteAsync(); + } + } + + [TestMethod] + public async Task RewindFailureWaitCanBeCanceledWithoutChangingHistory() + { + using var service = CreateRewindRaceService(); + await service.CreateAsync(); + try + { + // 1. Leave an instance running with no failed checkpoint. + await SaveRewindRaceHistoryAsync(service, "instance", failed: false); + OrchestrationHistory before = await service.TrackingStore.GetHistoryEventsAsync("instance", "execution"); + + // 2. Cancel while rewind waits for failure. + using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); + try + { + await service.TrackingStore.RewindHistoryAsync("instance", timeout.Token).ToListAsync(); + Assert.Fail("Rewind should have been canceled."); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) { } + + // 3. Neither history nor status should have changed. + Assert.AreEqual(before.ETag, (await service.TrackingStore.GetHistoryEventsAsync("instance", "execution")).ETag); + Assert.AreEqual(OrchestrationStatus.Running, + (await service.TrackingStore.FetchInstanceStatusAsync("instance")).State.OrchestrationStatus); + } + finally + { + await service.DeleteAsync(); + } + } + + sealed class RewindStatusTrackingStore : AzureTableTrackingStore + { + readonly Func beforeStatusUpdate; + + public RewindStatusTrackingStore(AzureStorageClient client, MessageManager messages, + Func beforeStatusUpdate) + : base(client, messages) + { + this.beforeStatusUpdate = beforeStatusUpdate; + } + + public int StatusUpdateAttempts { get; private set; } + + public override async Task UpdateStatusForRewindAsync(string instanceId, Azure.ETag eTag, CancellationToken cancellationToken = default) + { + this.StatusUpdateAttempts++; + await this.beforeStatusUpdate(instanceId, eTag, cancellationToken); + await base.UpdateStatusForRewindAsync(instanceId, eTag, cancellationToken); + } + } + + static AzureStorageOrchestrationService CreateRewindRaceService() => new(CreateRewindRaceSettings()); + + static AzureStorageOrchestrationServiceSettings CreateRewindRaceSettings() => new() + { + PartitionCount = 1, + StorageAccountClientProvider = new StorageAccountClientProvider(TestHelpers.GetTestStorageAccountConnectionString()), + TaskHubName = "RewindRace" + Guid.NewGuid().ToString("N").Substring(0, 12), + ExtendedSessionsEnabled = false, + UseAppLease = false, + }; + + static async Task SaveRewindRaceHistoryAsync( + AzureStorageOrchestrationService service, string instanceId, bool failed, string childInstanceId = null) + { + OrchestrationHistory previous = await service.TrackingStore.GetHistoryEventsAsync(instanceId, "execution"); + InstanceStatus status = await service.TrackingStore.FetchInstanceStatusAsync(instanceId); + var runtime = new OrchestrationRuntimeState(); + runtime.AddEvent(new OrchestratorStartedEvent(-1)); + runtime.AddEvent(new ExecutionStartedEvent(-1, "input") + { + Name = "RewindRace", Version = string.Empty, + OrchestrationInstance = new OrchestrationInstance { InstanceId = instanceId, ExecutionId = "execution" }, + }); + if (childInstanceId != null) + { + runtime.AddEvent(new SubOrchestrationInstanceCreatedEvent(0) { Name = "Child", InstanceId = childInstanceId }); + runtime.AddEvent(new SubOrchestrationInstanceFailedEvent(1, 0, "failure", null)); + } + else + { + runtime.AddEvent(new TaskScheduledEvent(0) { Name = "Activity", Version = string.Empty }); + if (failed) + { + runtime.AddEvent(new TaskFailedEvent(1, 0, "failure", null)); + } + } + if (failed) + { + runtime.AddEvent(new ExecutionCompletedEvent(2, "failure", OrchestrationStatus.Failed)); + } + runtime.AddEvent(new OrchestratorCompletedEvent(-1)); + await service.TrackingStore.UpdateStateAsync(runtime, new OrchestrationRuntimeState(), instanceId, "execution", + new OrchestrationETags { HistoryETag = previous.ETag, InstanceETag = status?.ETag }, previous.TrackingStoreContext); + } + /// /// End-to-end test which validates the Rewind functionality on more than one orchestration. /// @@ -1757,6 +2083,7 @@ public async Task RewindNestedSubOrchestrationTest() await clientGrandparent.RewindAsync("Rewind 1: Nested child activity still fails."); + statusFail = await clientGrandparent.WaitForCompletionAsync(TimeSpan.FromSeconds(30)); Assert.AreEqual(OrchestrationStatus.Failed, statusFail?.OrchestrationStatus); Activities.HelloFailNestedSuborchestration.ShouldFail2 = false; diff --git a/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs b/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs index dbe582b8..a755aa47 100644 --- a/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs +++ b/test/DurableTask.AzureStorage.Tests/OrchestrationStatePrefetchTests.cs @@ -367,7 +367,7 @@ async Task ITrackingStore.FetchInstanceStatusAsync(string instan Task ITrackingStore.GetStateAsync(string instanceId, string executionId, bool fetchInput, CancellationToken cancellationToken) => throw new NotSupportedException(); - Task ITrackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, bool instanceEntityExists, CancellationToken cancellationToken) => throw new NotSupportedException(); + Task ITrackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, OrchestrationETags eTags, CancellationToken cancellationToken) => throw new NotSupportedException(); IAsyncEnumerable ITrackingStore.GetStateAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -379,7 +379,7 @@ async Task ITrackingStore.FetchInstanceStatusAsync(string instan Task ITrackingStore.SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, ETag? eTag, string inputPayloadOverride, CancellationToken cancellationToken) => throw new NotSupportedException(); - Task ITrackingStore.UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken) => throw new NotSupportedException(); + Task ITrackingStore.UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken) => throw new NotSupportedException(); Task ITrackingStore.UpdateStatusForTerminationAsync(string instanceId, ExecutionTerminatedEvent executionTerminatedEvent, CancellationToken cancellationToken) => throw new NotSupportedException(); diff --git a/test/DurableTask.AzureStorage.Tests/ParentInstanceIdTrackingStoreTests.cs b/test/DurableTask.AzureStorage.Tests/ParentInstanceIdTrackingStoreTests.cs index d613c4f8..250ce98a 100644 --- a/test/DurableTask.AzureStorage.Tests/ParentInstanceIdTrackingStoreTests.cs +++ b/test/DurableTask.AzureStorage.Tests/ParentInstanceIdTrackingStoreTests.cs @@ -68,24 +68,24 @@ public async Task Cleanup() /// /// Verifies that a no-parent write clears a parent ID left behind by a previous row with the same - /// instance ID. This covers the terminal-history repair write, which is unconditionally - /// InsertOrMerge, so a helper that skips the property when there is no parent would leave the + /// instance ID. This covers the terminal-history repair write, which uses a conditional + /// merge, so a helper that skips the property when there is no parent would leave the /// stale value in place. /// [TestMethod] - public async Task CompletedOrchestrationRepair_ClearsStaleParentInstanceId_WithInsertOrMerge() + public async Task CompletedOrchestrationRepair_ClearsStaleParentInstanceId_WithMerge() { string instanceId = $"stale-{Guid.NewGuid():N}"; await this.SeedInstanceRowAsync(instanceId, "stale-parent"); Assert.AreEqual("stale-parent", await this.GetRawParentInstanceIdAsync(instanceId), "Seeded row should carry the stale parent."); - // Drive a genuine no-parent write through the same production path that uses InsertOrMerge. + // Drive a genuine no-parent write through the same production path that uses a conditional merge. await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( instanceId, executionId: "execution-1", runtimeState: CreateCompletedRuntimeState(instanceId, "execution-1", parentInstanceId: null), - instanceEntityExists: true); + eTags: new OrchestrationETags { InstanceETag = (await this.trackingStore.FetchInstanceStatusAsync(instanceId)).ETag }); Assert.AreEqual( string.Empty, @@ -255,7 +255,7 @@ await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( childInstanceId, executionId: "execution-1", runtimeState: CreateCompletedRuntimeState(childInstanceId, "execution-1", parentInstanceId), - instanceEntityExists: false); + eTags: new OrchestrationETags()); Assert.AreEqual(parentInstanceId, await this.GetRawParentInstanceIdAsync(childInstanceId)); @@ -278,7 +278,7 @@ await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync( childInstanceId, executionId: "execution-1", runtimeState: CreateCompletedRuntimeState(childInstanceId, "execution-1", parentInstanceId), - instanceEntityExists: true); + eTags: new OrchestrationETags { InstanceETag = (await this.trackingStore.FetchInstanceStatusAsync(childInstanceId)).ETag }); Assert.AreEqual(parentInstanceId, await this.GetRawParentInstanceIdAsync(childInstanceId)); }