Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,7 @@ await this.AbandonAndReleaseSessionAsync(

string warningMessage = await this.IsExecutableInstanceAsync(
session.RuntimeState,
session.ETags.HistoryETag,
orchestrationWorkItem.NewMessages,
settings.AllowReplayingTerminalInstances,
cancellationToken);
Expand Down Expand Up @@ -880,6 +881,15 @@ await this.AbandonAndReleaseSessionAsync(

return orchestrationWorkItem;
}
catch (DurableTaskStorageException exception) when (exception.HttpStatusCode == 409 || exception.HttpStatusCode == 412)
{
// Reload changed state without losing the message that woke this instance.
if (session != null)
{
await this.AbandonAndReleaseSessionAsync(session, "Instance or history changed during status recovery.");
}
return null;
}
catch (OperationCanceledException)
{
if (session != null)
Expand Down Expand Up @@ -1082,6 +1092,7 @@ internal static void TraceMessageReceived(AzureStorageOrchestrationServiceSettin

async Task<string> IsExecutableInstanceAsync(
OrchestrationRuntimeState runtimeState,
ETag? historyETag,
IList<TaskMessage> newMessages,
bool allowReplayingTerminalInstances,
CancellationToken cancellationToken)
Expand Down Expand Up @@ -1122,15 +1133,15 @@ await this.trackingStore.UpdateStatusForTerminationAsync(
runtimeState.OrchestrationStatus != OrchestrationStatus.Pending &&
runtimeState.OrchestrationStatus != OrchestrationStatus.Suspended)
{
InstanceStatus instanceStatus = await this.trackingStore.FetchInstanceStatusAsync(runtimeState.OrchestrationInstance.InstanceId);
InstanceStatus instanceStatus = await this.trackingStore.FetchInstanceStatusAsync(runtimeState.OrchestrationInstance.InstanceId, cancellationToken);
if (instanceStatus == null || (instanceStatus.State.OrchestrationInstance.ExecutionId == runtimeState.OrchestrationInstance.ExecutionId
&& instanceStatus.State.OrchestrationStatus != runtimeState.OrchestrationStatus))
{
await this.trackingStore.UpdateInstanceStatusForCompletedOrchestrationAsync(
runtimeState.OrchestrationInstance.InstanceId,
runtimeState.OrchestrationInstance.ExecutionId,
runtimeState,
instanceStatus is not null,
new OrchestrationETags { HistoryETag = historyETag, InstanceETag = instanceStatus?.ETag },
cancellationToken);
}
if (!allowReplayingTerminalInstances)
Expand Down
112 changes: 99 additions & 13 deletions src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -285,14 +285,44 @@ public override async IAsyncEnumerable<string> RewindHistoryAsync(string instanc
bool hasFailedSubOrchestrations = false;
string partitionFilter = AzureTableQueryFilter.PartitionKeyEquals(instanceId);

string orchestratorStartedFilter = $"{partitionFilter} and {nameof(HistoryEvent.EventType)} eq '{nameof(EventType.OrchestratorStarted)}'";
IReadOnlyList<TableEntity> orchestratorStartedEntities = await this.QueryHistoryAsync(orchestratorStartedFilter, instanceId, cancellationToken);
// A child can report failure before its own checkpoint finishes.
// Wait for that checkpoint before reading or changing its history.
Stopwatch waitForFailure = Stopwatch.StartNew();
string executionId = null;
ETag instanceETag;

// get most recent orchestratorStarted event
string recentStartRowKey = orchestratorStartedEntities.Max(x => x.RowKey);
var recentStartRow = orchestratorStartedEntities.Where(y => y.RowKey == recentStartRowKey).ToList();
string executionId = recentStartRow[0].GetString(nameof(OrchestrationInstance.ExecutionId));
DateTime instanceTimestamp = recentStartRow[0].Timestamp.GetValueOrDefault().DateTime;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
InstanceStatus instanceStatus = await this.FetchInstanceStatusAsync(instanceId, cancellationToken);
if (instanceStatus != null)
{
string currentExecutionId = instanceStatus.State.OrchestrationInstance.ExecutionId;
if (executionId != null && executionId != currentExecutionId)
{
throw new InvalidOperationException($"Cannot rewind instance '{instanceId}': execution '{executionId}' no longer exists.");
}
Comment thread
sophiatev marked this conversation as resolved.
Resolved
executionId = currentExecutionId;
if (instanceStatus.State.OrchestrationStatus == OrchestrationStatus.Failed)
{
instanceETag = instanceStatus.ETag;
break;
Comment thread
Copilot marked this conversation as resolved.
Resolved
Comment thread
Copilot marked this conversation as resolved.
Resolved
}
}
else if (executionId != null)
{
throw new InvalidOperationException($"Cannot rewind instance '{instanceId}': execution '{executionId}' no longer exists.");
}

if (waitForFailure.Elapsed >= TimeSpan.FromMinutes(1))
{
throw new TimeoutException($"Timed out waiting for instance '{instanceId}' to commit its failed status before rewind.");
}

await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
}

ETag? historyETag = await this.GetHistoryETagAsync(instanceId, executionId, cancellationToken);

// Use parameterized filter to prevent OData injection via crafted execution IDs
string executionIdFilter = AzureTableQueryFilter.ColumnEquals(nameof(OrchestrationInstance.ExecutionId), executionId);
Expand Down Expand Up @@ -368,8 +398,22 @@ public override async IAsyncEnumerable<string> RewindHistoryAsync(string instanc
await this.HistoryTable.ReplaceEntityAsync(entity, entity.ETag, cancellationToken);
}

// Invalidate prefetched history before making the instance Pending.
var sentinel = new TableEntity(KeySanitation.EscapePartitionKey(instanceId), SentinelRowKey)
{
[nameof(OrchestrationInstance.ExecutionId)] = executionId,
};
if (historyETag.HasValue)
{
await this.HistoryTable.MergeEntityAsync(sentinel, historyETag.Value, cancellationToken);
}
else
{
await this.HistoryTable.InsertEntityAsync(sentinel, cancellationToken);
}

// reset orchestration status in instance store table
await this.UpdateStatusForRewindAsync(instanceId, cancellationToken);
await this.UpdateStatusForRewindAsync(instanceId, instanceETag, cancellationToken);

if (!hasFailedSubOrchestrations)
{
Expand Down Expand Up @@ -854,7 +898,7 @@ public override async Task<bool> SetNewExecutionAsync(
}

/// <inheritdoc />
public override async Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default)
public override async Task UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken = default)
{
string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId);
TableEntity entity = new TableEntity(sanitizedInstanceId, "")
Expand All @@ -864,7 +908,8 @@ public override async Task UpdateStatusForRewindAsync(string instanceId, Cancell
};

Stopwatch stopwatch = Stopwatch.StartNew();
await this.InstancesTable.MergeEntityAsync(entity, ETag.All, cancellationToken);
// Do not reset an instance that changed during rewind.
await this.InstancesTable.MergeEntityAsync(entity, eTag, cancellationToken);

// We don't have enough information to get the episode number.
// It's also not important to have for this particular trace.
Expand Down Expand Up @@ -1138,7 +1183,7 @@ public override async Task UpdateInstanceStatusForCompletedOrchestrationAsync(
string instanceId,
string executionId,
OrchestrationRuntimeState runtimeState,
bool instanceEntityExists,
OrchestrationETags eTags,
CancellationToken cancellationToken = default)
{
if (runtimeState.OrchestrationStatus != OrchestrationStatus.Completed &&
Expand All @@ -1149,6 +1194,21 @@ public override async Task UpdateInstanceStatusForCompletedOrchestrationAsync(
return;
}

// There is a particular race condition which can occur with rewind if a worker crashes after updating the instance and history state
// to "Failed", but before deleting the triggering messages. If rewind is then requested on the failed instance, its status will be
// updated to "Pending", but if we happened to fetch the history BEFORE rewind altered it, then we will see a mismatch between the failed
// history and the instance status at this point. We do not actually want to set the status to "Failed" here in that case. The easiest way
// to detect this is to check if the history's etag is still current.
// (Note that there remains a known gap for a multi-page history if the sentinel is on a different page from the ExecutionCompletedEvent. Then
// it is possible we still fetch a failed history, the rewind goes through and changes the history and the sentinel's etag, and then we read
// the sentinel with its new etag and think the history is current)
if (eTags.HistoryETag.HasValue &&
eTags.HistoryETag != await this.GetHistoryETagAsync(instanceId, executionId, cancellationToken))
Comment thread
Copilot marked this conversation as resolved.
Resolved
{
throw new DurableTaskStorageException(new RequestFailedException(
412, "The history changed before its terminal status could be restored.", "UpdateConditionNotSatisfied", null));
}

string sanitizedInstanceId = KeySanitation.EscapePartitionKey(instanceId);
ExecutionStartedEvent executionStartedEvent = runtimeState.ExecutionStartedEvent;

Expand Down Expand Up @@ -1214,7 +1274,7 @@ static TableEntity GetSingleEntityFromHistoryTableResults(IReadOnlyList<TableEnt
}

// If the input has not been set by a previous execution, set the input.
if (!instanceEntityExists)
if (!eTags.InstanceETag.HasValue)
{
// In the case that the input is too large and is stored in blob storage, extract the blob name from the ExecutionStarted history entity.
if (this.ExceedsMaxTablePropertySize(runtimeState.Input))
Expand All @@ -1238,7 +1298,18 @@ static TableEntity GetSingleEntityFromHistoryTableResults(IReadOnlyList<TableEnt
}

Stopwatch orchestrationInstanceUpdateStopwatch = Stopwatch.StartNew();
await this.InstancesTable.InsertOrMergeEntityAsync(instanceEntity);

// Do not overwrite an instance row that changed after we read it.
// In particular, this guards against the race condition where an instance row was not committed for a failed history, and the
// instance is then recreated with the same instance ID. In that case we do not want to override the new instance state.
if (eTags.InstanceETag.HasValue)
{
await this.InstancesTable.MergeEntityAsync(instanceEntity, eTags.InstanceETag.Value, cancellationToken);
}
else
{
await this.InstancesTable.InsertEntityAsync(instanceEntity, cancellationToken);
}

this.settings.Logger.InstanceStatusUpdate(
this.storageAccountName,
Expand Down Expand Up @@ -1415,6 +1486,21 @@ static string GetBlobName(TableEntity entity, string property)
return $"{sanitizedInstanceId}/history-{sequenceNumber}-{eventType}-{random:X8}-{property}.json.gz";
}

async Task<ETag?> GetHistoryETagAsync(string instanceId, string executionId, CancellationToken cancellationToken)
{
string filter = $"{AzureTableQueryFilter.PartitionKeyEquals(instanceId)} and {AzureTableQueryFilter.ColumnEquals(nameof(ITableEntity.RowKey), SentinelRowKey)}";
TableQueryResults<TableEntity> results = await this.HistoryTable.ExecuteQueryAsync<TableEntity>(
filter, select: new[] { nameof(OrchestrationInstance.ExecutionId) }, cancellationToken: cancellationToken)
.GetResultsAsync(cancellationToken: cancellationToken);
TableEntity sentinel = results.Entities.SingleOrDefault();
if (sentinel?.GetString(nameof(OrchestrationInstance.ExecutionId)) is string historyExecutionId && historyExecutionId != executionId)
{
throw new DurableTaskStorageException(new RequestFailedException(
412, "The history belongs to a different execution.", "UpdateConditionNotSatisfied", null));
}
return sentinel?.ETag;
}

async Task<ETag?> UploadHistoryBatch(
string instanceId,
string sanitizedInstanceId,
Expand Down
7 changes: 4 additions & 3 deletions src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,9 @@ interface ITrackingStore
/// <param name="instanceId">The ID of the orchestration.</param>
/// <param name="executionId">The execution ID of the orchestration.</param>
/// <param name="runtimeState">The runtime state of the orchestration.</param>
/// <param name="instanceEntityExists">Whether the instance entity already exists in the instance store.</param>
/// <param name="eTags">The history and instance ETags read before recovery. A missing instance ETag means the row did not exist.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests. The default value is <see cref="CancellationToken.None"/>.</param>
Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, bool instanceEntityExists, CancellationToken cancellationToken = default);
Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, OrchestrationETags eTags, CancellationToken cancellationToken = default);

/// <summary>
/// Get The Orchestration State for querying all orchestration instances
Expand Down Expand Up @@ -161,8 +161,9 @@ interface ITrackingStore
/// Used to update a state in the tracking store to pending whenever a rewind is initiated from the client
/// </summary>
/// <param name="instanceId">The instance being rewound</param>
/// <param name="eTag">The instance ETag read when its failed status was confirmed.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests. The default value is <see cref="CancellationToken.None"/>.</param>
Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default);
Task UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken = default);

/// <summary>
/// Used to update the instance status to "Terminated" when a pending orchestration is terminated.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ public override async Task UpdateInstanceStatusForCompletedOrchestrationAsync(
string instanceId,
string executionId,
OrchestrationRuntimeState runtimeState,
bool instanceEntityExists,
OrchestrationETags eTags,
CancellationToken cancellationToken = default)
{
if (runtimeState.OrchestrationStatus != OrchestrationStatus.Completed &&
Expand Down
4 changes: 2 additions & 2 deletions src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public virtual Task<PurgeHistoryResult> PurgeInstanceHistoryAsync(DateTime creat
public abstract Task<bool> SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, ETag? eTag, string inputStatusOverride, CancellationToken cancellationToken = default);

/// <inheritdoc />
public virtual Task UpdateStatusForRewindAsync(string instanceId, CancellationToken cancellationToken = default)
public virtual Task UpdateStatusForRewindAsync(string instanceId, ETag eTag, CancellationToken cancellationToken = default)
{
throw new NotSupportedException();
}
Expand All @@ -110,6 +110,6 @@ public virtual Task UpdateStatusForRewindAsync(string instanceId, CancellationTo
public abstract Task UpdateStateAsync(OrchestrationRuntimeState newRuntimeState, OrchestrationRuntimeState oldRuntimeState, string instanceId, string executionId, OrchestrationETags eTags, object executionData, CancellationToken cancellationToken = default);

/// <inheritdoc />
public abstract Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, bool instanceEntityExists, CancellationToken cancellationToken = default);
public abstract Task UpdateInstanceStatusForCompletedOrchestrationAsync(string instanceId, string executionId, OrchestrationRuntimeState runtimeState, OrchestrationETags eTags, CancellationToken cancellationToken = default);
}
}
Loading
Loading