From df5ca1832fed3e3f4135283b345bd1f670fce438 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 16 Nov 2026 14:44:28 -0800 Subject: [PATCH] feat(core): persist activity invocation start times Related to Azure/azure-functions-durable-extension#2132. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/concepts/activities.md | 12 + .../History/TaskCompletedEvent.cs | 12 + .../History/TaskFailedEvent.cs | 12 + .../TaskActivityDispatcher.cs | 7 + .../AzureTableTrackingStoreTest.cs | 66 ++++ .../MessageManagerTests.cs | 41 +++ .../ActivityStartedTimeTests.cs | 323 ++++++++++++++++++ 7 files changed, 473 insertions(+) create mode 100644 test/DurableTask.Core.Tests/ActivityStartedTimeTests.cs diff --git a/docs/concepts/activities.md b/docs/concepts/activities.md index 803e6a8f9..785a798dd 100644 --- a/docs/concepts/activities.md +++ b/docs/concepts/activities.md @@ -151,6 +151,18 @@ public override async Task RunTask( > [!IMPORTANT] > Do not include `TaskContext` or `CancellationToken` parameters in activity interface methods. Only JSON-serializable input and output types are allowed. +## Activity Start Times in History + +`TaskCompletedEvent.StartedTime` and `TaskFailedEvent.StartedTime` record the worker's UTC time immediately before the dispatcher calls `TaskActivity.RunAsync`. This excludes queueing and middleware work before that call, but includes any input conversion, host invocation, and output conversion inside `RunAsync`. The scheduled event's `Timestamp` and the response event's `Timestamp` keep their existing meanings. + +The value belongs to the invocation associated with that response's `TaskScheduledId`. Each retry scheduled by the orchestration has its own scheduled event. A redelivery of the same scheduled event captures a new start; abandoned invocations do not produce history entries. No separate start event is emitted, so this does not expose in-flight starts or a log of every execution attempt. Timestamps from different workers are subject to clock skew. + +Start time is unknown (`null`, omitted from JSON) for older history, older workers, middleware that skips the activity invocation, and default completions synthesized without an activity result. Middleware that replaces a response after invoking the activity retains the observed start only when the replacement refers to the same scheduled event. + +The Azure Storage provider preserves this metadata in activity response messages, History-table rows, and `GetOrchestrationHistoryAsync` JSON. Other providers and protocol-specific history mappings are not covered by this guarantee; they must preserve the new fields where they translate events. In particular, the Durable Functions protobuf history contract needs a separate update. + +The existing Durable Functions JSON `GetStatusAsync(showHistory: true)` projection can retain the additive fields, but requires adoption of a Core package containing this change. Consumers must not substitute `ScheduledTime` when `StartedTime` is absent. + ## Activity Best Practices ### 1. Keep Activities Focused diff --git a/src/DurableTask.Core/History/TaskCompletedEvent.cs b/src/DurableTask.Core/History/TaskCompletedEvent.cs index f06057f80..b94b8d534 100644 --- a/src/DurableTask.Core/History/TaskCompletedEvent.cs +++ b/src/DurableTask.Core/History/TaskCompletedEvent.cs @@ -13,6 +13,7 @@ namespace DurableTask.Core.History { + using System; using System.Runtime.Serialization; /// @@ -50,5 +51,16 @@ public TaskCompletedEvent(int eventId, int taskScheduledId, string result) /// [DataMember] public string Result { get; private set; } + + /// + /// Gets or sets the UTC time at which the dispatcher invoked . + /// + /// + /// This is the start of the invocation associated with this response, not the time the task was scheduled. + /// A redelivered task records only the invocation whose response is persisted. The value is null + /// for older history or when the dispatcher did not observe a matching activity invocation. + /// + [DataMember(EmitDefaultValue = false)] + public DateTime? StartedTime { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/History/TaskFailedEvent.cs b/src/DurableTask.Core/History/TaskFailedEvent.cs index 110caa050..9dee465c0 100644 --- a/src/DurableTask.Core/History/TaskFailedEvent.cs +++ b/src/DurableTask.Core/History/TaskFailedEvent.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.History { + using System; using System.Runtime.Serialization; /// @@ -77,5 +78,16 @@ private TaskFailedEvent() /// [DataMember] public FailureDetails? FailureDetails { get; private set; } + + /// + /// Gets or sets the UTC time at which the dispatcher invoked . + /// + /// + /// This is the start of the invocation associated with this response, not the time the task was scheduled. + /// A redelivered task records only the invocation whose response is persisted. The value is null + /// for older history or when the dispatcher did not observe a matching activity invocation. + /// + [DataMember(EmitDefaultValue = false)] + public DateTime? StartedTime { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/TaskActivityDispatcher.cs b/src/DurableTask.Core/TaskActivityDispatcher.cs index 6494b59ac..53eea24f2 100644 --- a/src/DurableTask.Core/TaskActivityDispatcher.cs +++ b/src/DurableTask.Core/TaskActivityDispatcher.cs @@ -188,6 +188,7 @@ async Task OnProcessWorkItemAsync(TaskActivityWorkItem workItem) }); ActivityExecutionResult? result; + DateTime? activityStartedTime = null; try { await this.dispatchPipeline.RunAsync(dispatchContext, async _ => @@ -216,6 +217,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => try { + activityStartedTime = DateTime.UtcNow; string? output = await taskActivity.RunAsync(context, scheduledEvent.Input); responseEvent = new TaskCompletedEvent(-1, scheduledEvent.EventId, output); } @@ -267,8 +269,13 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => if (eventToRespond is TaskCompletedEvent completedEvent) { + completedEvent.StartedTime = completedEvent.TaskScheduledId == scheduledEvent.EventId ? activityStartedTime : null; this.logHelper.TaskActivityCompleted(orchestrationInstance, scheduledEvent.Name, completedEvent); } + else if (eventToRespond is TaskFailedEvent failedEvent) + { + failedEvent.StartedTime = failedEvent.TaskScheduledId == scheduledEvent.EventId ? activityStartedTime : null; + } else if (eventToRespond is null) { // Default response if middleware prevents a response from being generated diff --git a/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs b/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs index 5c1b402df..c4e49bfd3 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureTableTrackingStoreTest.cs @@ -29,6 +29,7 @@ namespace DurableTask.AzureStorage.Tests using DurableTask.Core.Tracking; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; + using Newtonsoft.Json.Linq; [TestClass] public class AzureTableTrackingStoreTest @@ -164,5 +165,70 @@ public async Task InstanceStoreBackedTrackingStore_PersistsParentOnCreation() Assert.AreSame(startedEvent.ParentInstance, writtenState.State.ParentInstance); Assert.AreEqual(ParentInstanceId, writtenState.State.ParentInstance.OrchestrationInstance.InstanceId); } + + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task ActivityStartedTime_PersistsInHistoryAndHistoryJson(bool failed) + { + const string InstanceId = "instance"; + const string ExecutionId = "execution"; + DateTime scheduledTime = new DateTime(2026, 1, 2, 12, 0, 0, DateTimeKind.Utc); + DateTime startedTime = scheduledTime.AddMinutes(12); + DateTime completedTime = scheduledTime.AddMinutes(15); + var settings = TestHelpers.GetTestAzureStorageOrchestrationServiceSettings(enableExtendedSessions: false); + settings.TaskHubName = "started" + Guid.NewGuid().ToString("N"); + var storageClient = new AzureStorageClient(settings); + var messageManager = new MessageManager(settings, storageClient, settings.TaskHubName.ToLowerInvariant() + "-largemessages"); + var trackingStore = new AzureTableTrackingStore(storageClient, messageManager); + using var service = new AzureStorageOrchestrationService(settings); + var runtimeState = new OrchestrationRuntimeState(); + runtimeState.AddEvent(new ExecutionStartedEvent(0, null) + { + Name = "orchestration", + OrchestrationInstance = new OrchestrationInstance { InstanceId = InstanceId, ExecutionId = ExecutionId }, + Timestamp = scheduledTime, + }); + runtimeState.AddEvent(new TaskScheduledEvent(1, "activity", "v1") { Timestamp = scheduledTime }); + HistoryEvent response = failed + ? new TaskFailedEvent(2, 1, "failure", "details") { StartedTime = startedTime, Timestamp = completedTime } + : new TaskCompletedEvent(2, 1, "result") { StartedTime = startedTime, Timestamp = completedTime }; + runtimeState.AddEvent(response); + runtimeState.AddEvent(new TaskScheduledEvent(3, "legacy", "v1") { Timestamp = scheduledTime }); + runtimeState.AddEvent(new TaskCompletedEvent(4, 3, "legacy") { Timestamp = completedTime }); + + try + { + await trackingStore.CreateAsync(); + OrchestrationHistory empty = await trackingStore.GetHistoryEventsAsync(InstanceId, ExecutionId); + await trackingStore.UpdateStateAsync( + runtimeState, runtimeState, InstanceId, ExecutionId, new OrchestrationETags(), empty.TrackingStoreContext); + + var tableClient = new TableServiceClient(TestHelpers.GetTestStorageAccountConnectionString()) + .GetTableClient(settings.TaskHubName + "History"); + TableEntity row = (await tableClient.GetEntityAsync(InstanceId, "0000000000000002")).Value; + Assert.AreEqual(startedTime, row.GetDateTime("StartedTime")); + Assert.AreEqual(completedTime, row.GetDateTime("_Timestamp")); + TableEntity legacyRow = (await tableClient.GetEntityAsync(InstanceId, "0000000000000004")).Value; + Assert.IsFalse(legacyRow.ContainsKey("StartedTime")); + + OrchestrationHistory history = await trackingStore.GetHistoryEventsAsync(InstanceId, ExecutionId); + HistoryEvent actual = history.Events[2]; + Assert.AreEqual(response.EventType, actual.EventType); + Assert.AreEqual(startedTime, actual is TaskFailedEvent failure ? failure.StartedTime : ((TaskCompletedEvent)actual).StartedTime); + Assert.IsNull(((TaskCompletedEvent)history.Events[4]).StartedTime); + + JArray json = JArray.Parse(await service.GetOrchestrationHistoryAsync(InstanceId, ExecutionId)); + Assert.AreEqual(scheduledTime, (DateTime)json[1]["Timestamp"]); + Assert.AreEqual(startedTime, (DateTime)json[2]["StartedTime"]); + Assert.AreEqual(completedTime, (DateTime)json[2]["Timestamp"]); + Assert.AreEqual(1, (int)json[2]["TaskScheduledId"]); + Assert.IsNull(json[4]["StartedTime"]); + } + finally + { + await trackingStore.DeleteAsync(); + } + } } } diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index 4e2f76a2f..a06a56f9d 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -122,6 +122,47 @@ public async Task SerializeMessageDataAsync_InlinePayloadMatchesPreviousWirePayl ((GenericEvent)roundTrippedMessage.TaskMessage.Event).Data); } + [TestMethod] + [DataRow(false, false)] + [DataRow(false, true)] + [DataRow(true, false)] + [DataRow(true, true)] + public async Task SerializeMessageDataAsync_PreservesActivityStartedTime(bool useDataContractSerialization, bool failed) + { + var settings = new AzureStorageOrchestrationServiceSettings + { + UseDataContractSerialization = useDataContractSerialization, + }; + var manager = SetupMessageManager(settings, "$root"); + DateTime startedTime = new DateTime(2026, 1, 2, 12, 12, 0, DateTimeKind.Utc); + MessageData message = CreateMessageData("unused"); + message.TaskMessage.Event = failed + ? new TaskFailedEvent(-1, 7, "failure", "details") { StartedTime = startedTime } + : new TaskCompletedEvent(-1, 7, "result") { StartedTime = startedTime }; + + string payload = await manager.SerializeMessageDataAsync(message); + MessageData actual = manager.DeserializeMessageData(payload); + + Assert.AreEqual(MessageFormatFlags.InlineJson, manager.GetMessageFormatFlags(message)); + Assert.AreEqual(message.TaskMessage.Event.Timestamp, actual.TaskMessage.Event.Timestamp); + Assert.AreEqual(message.TaskMessage.OrchestrationInstance.InstanceId, actual.TaskMessage.OrchestrationInstance.InstanceId); + Assert.AreEqual(message.TaskMessage.OrchestrationInstance.ExecutionId, actual.TaskMessage.OrchestrationInstance.ExecutionId); + if (failed) + { + var response = (TaskFailedEvent)actual.TaskMessage.Event; + Assert.AreEqual(7, response.TaskScheduledId); + Assert.AreEqual(startedTime, response.StartedTime); + Assert.AreEqual("failure", response.Reason); + } + else + { + var response = (TaskCompletedEvent)actual.TaskMessage.Event; + Assert.AreEqual(7, response.TaskScheduledId); + Assert.AreEqual(startedTime, response.StartedTime); + Assert.AreEqual("result", response.Result); + } + } + [DataTestMethod] [DataRow(false, 0)] [DataRow(false, 1)] diff --git a/test/DurableTask.Core.Tests/ActivityStartedTimeTests.cs b/test/DurableTask.Core.Tests/ActivityStartedTimeTests.cs new file mode 100644 index 000000000..bc93c44cb --- /dev/null +++ b/test/DurableTask.Core.Tests/ActivityStartedTimeTests.cs @@ -0,0 +1,323 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- +#nullable enable +namespace DurableTask.Core.Tests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Reflection; + using System.Runtime.Serialization; + using System.Threading.Tasks; + using System.Xml; + using DurableTask.Core.Exceptions; + using DurableTask.Core.History; + using DurableTask.Core.Logging; + using DurableTask.Core.Middleware; + using DurableTask.Core.Serializing; + using DurableTask.Emulator; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json.Linq; + + [TestClass] + public sealed class ActivityStartedTimeTests + { + static readonly DateTime ScheduledTime = new DateTime(2000, 1, 1, 12, 0, 0, DateTimeKind.Utc); + + [TestMethod] + [DataRow(EventType.TaskCompleted, false)] + [DataRow(EventType.TaskFailed, false)] + [DataRow(EventType.TaskFailed, true)] + public async Task Dispatcher_RecordsStartForCompletedAndFailedInvocations(EventType expectedType, bool unexpectedFailure) + { + using var service = new RecordingService(); + var activity = new RecordingActivity((_, input) => + { + if (expectedType == EventType.TaskFailed) + { + throw unexpectedFailure ? new InvalidOperationException("unexpected") : new TaskFailureException("normal"); + } + + return Task.FromResult(input); + }); + TaskActivityDispatcher dispatcher = CreateDispatcher(service, activity, new DispatchMiddlewarePipeline()); + TaskActivityWorkItem workItem = CreateWorkItem(7, 42); + + await DispatchAsync(dispatcher, workItem); + + TaskMessage response = service.Responses.Single(); + Assert.AreEqual(expectedType, response.Event.EventType); + Assert.AreSame(workItem.TaskMessage.OrchestrationInstance, response.OrchestrationInstance); + Assert.AreEqual(7, GetScheduledId(response.Event)); + AssertKnownStart(response.Event); + Assert.AreEqual(ScheduledTime, workItem.TaskMessage.Event.Timestamp); + Assert.AreEqual(42L, activity.Invocations.Single().DeliveryAttempt); + Assert.AreEqual(7, activity.Invocations.Single().TaskId); + if (response.Event is TaskCompletedEvent completed) + { + Assert.AreEqual("input", completed.Result); + } + else + { + Assert.AreEqual(unexpectedFailure ? "unexpected" : "normal", ((TaskFailedEvent)response.Event).Reason); + } + } + + [TestMethod] + [DataRow("default")] + [DataRow("completed")] + [DataRow("failed")] + [DataRow("exception")] + public async Task Dispatcher_LeavesStartUnknownWhenMiddlewareSkipsInvocation(string outcome) + { + using var service = new RecordingService(); + var activity = new RecordingActivity((_, input) => Task.FromResult(input)); + var pipeline = new DispatchMiddlewarePipeline(); + pipeline.Add((context, _) => + { + if (outcome == "exception") + { + throw new TaskFailureException("middleware"); + } + + if (outcome != "default") + { + context.SetProperty(new ActivityExecutionResult + { + ResponseEvent = CreateResponse(outcome == "failed", 7, ScheduledTime), + }); + } + + return Task.CompletedTask; + }); + TaskActivityDispatcher dispatcher = CreateDispatcher(service, activity, pipeline); + + await DispatchAsync(dispatcher, CreateWorkItem(7, 1)); + + HistoryEvent response = service.Responses.Single().Event; + Assert.AreEqual(outcome == "failed" || outcome == "exception" ? EventType.TaskFailed : EventType.TaskCompleted, response.EventType); + Assert.IsNull(GetStartedTime(response)); + Assert.AreEqual(0, activity.Invocations.Count); + } + + [TestMethod] + [DataRow(false, true)] + [DataRow(true, true)] + [DataRow(false, false)] + [DataRow(true, false)] + public async Task Dispatcher_CorrelatesStartWithMiddlewareReplacement(bool failed, bool matchingTask) + { + using var service = new RecordingService(); + var activity = new RecordingActivity((_, input) => + failed ? throw new TaskFailureException("activity") : Task.FromResult(input)); + var pipeline = new DispatchMiddlewarePipeline(); + pipeline.Add(async (context, next) => + { + try + { + await next(); + } + catch (TaskFailureException) + { + Assert.IsTrue(failed); + } + + context.SetProperty(new ActivityExecutionResult + { + ResponseEvent = CreateResponse(failed, matchingTask ? 7 : 99, ScheduledTime), + }); + }); + TaskActivityDispatcher dispatcher = CreateDispatcher(service, activity, pipeline); + + await DispatchAsync(dispatcher, CreateWorkItem(7, 1)); + + HistoryEvent response = service.Responses.Single().Event; + Assert.AreEqual(failed ? EventType.TaskFailed : EventType.TaskCompleted, response.EventType); + Assert.AreEqual(matchingTask ? 7 : 99, GetScheduledId(response)); + Assert.AreEqual(1, activity.Invocations.Count); + if (matchingTask) + { + AssertKnownStart(response); + } + else + { + Assert.IsNull(GetStartedTime(response)); + } + } + + [TestMethod] + public async Task Dispatcher_DoesNotCarryStartAcrossAbandonmentRedeliveryOrNewScheduledTasks() + { + using var service = new RecordingService(); + bool abort = true; + bool skipInvocation = false; + var activity = new RecordingActivity((_, input) => + { + if (abort) + { + abort = false; + throw new SessionAbortedException("redeliver"); + } + + return Task.FromResult(input); + }); + var pipeline = new DispatchMiddlewarePipeline(); + pipeline.Add((_, next) => skipInvocation ? throw new TaskFailureException("middleware") : next()); + TaskActivityDispatcher dispatcher = CreateDispatcher(service, activity, pipeline); + TaskActivityWorkItem workItem = CreateWorkItem(7, 1); + + await DispatchAsync(dispatcher, workItem); + Assert.AreEqual(1, service.AbandonCount); + Assert.AreEqual(0, service.Responses.Count); + + skipInvocation = true; + workItem.DeliveryAttempt = 2; + await DispatchAsync(dispatcher, workItem); + Assert.IsNull(GetStartedTime(service.Responses.Single().Event)); + + skipInvocation = false; + workItem.DeliveryAttempt = 3; + await DispatchAsync(dispatcher, workItem); + AssertKnownStart(service.Responses[1].Event); + Assert.AreEqual(7, GetScheduledId(service.Responses[1].Event)); + + await DispatchAsync(dispatcher, CreateWorkItem(9, 1)); + AssertKnownStart(service.Responses[2].Event); + Assert.AreEqual(9, GetScheduledId(service.Responses[2].Event)); + CollectionAssert.AreEqual(new long?[] { 1, 3, 1 }, activity.Invocations.Select(c => c.DeliveryAttempt).ToArray()); + CollectionAssert.AreEqual(new[] { 7, 7, 9 }, activity.Invocations.Select(c => c.TaskId).ToArray()); + } + + [TestMethod] + [DataRow(false, false)] + [DataRow(false, true)] + [DataRow(true, false)] + [DataRow(true, true)] + public void HistorySerialization_PreservesStartAndLeavesLegacyHistoryUnknown(bool failed, bool hasStart) + { + DateTime? startedTime = hasStart ? ScheduledTime.AddMinutes(12) : null; + HistoryEvent original = CreateResponse(failed, 7, startedTime); + original.Timestamp = ScheduledTime.AddMinutes(15); + + string json = JsonDataConverter.Default.Serialize(original); + Assert.AreEqual(hasStart, JObject.Parse(json).Property("StartedTime") != null); + AssertRoundTrip(JsonDataConverter.Default.Deserialize(json)); + + var serializer = new DataContractSerializer(typeof(HistoryEvent)); + using var stream = new MemoryStream(); + serializer.WriteObject(stream, original); + stream.Position = 0; + using (var reader = XmlReader.Create(stream, new XmlReaderSettings { CloseInput = false })) + { + AssertRoundTrip((HistoryEvent)serializer.ReadObject(reader)!); + } + + void AssertRoundTrip(HistoryEvent actual) + { + Assert.AreEqual(original.EventType, actual.EventType); + Assert.AreEqual(original.EventId, actual.EventId); + Assert.AreEqual(original.Timestamp, actual.Timestamp); + Assert.AreEqual(7, GetScheduledId(actual)); + Assert.AreEqual(startedTime, GetStartedTime(actual)); + if (hasStart) + { + Assert.AreEqual(DateTimeKind.Utc, GetStartedTime(actual)!.Value.Kind); + } + } + } + + static TaskActivityDispatcher CreateDispatcher(RecordingService service, TaskActivity activity, DispatchMiddlewarePipeline pipeline) + { + var manager = new NameVersionObjectManager(); + manager.Add(new TestObjectCreator("activity", "v1", () => activity)); + return new TaskActivityDispatcher(service, manager, pipeline, new LogHelper(null), ErrorPropagationMode.UseFailureDetails, null); + } + + static TaskActivityWorkItem CreateWorkItem(int scheduledId, long deliveryAttempt) => new TaskActivityWorkItem + { + Id = "work-item", + LockedUntilUtc = DateTime.MaxValue, + DeliveryAttempt = deliveryAttempt, + TaskMessage = new TaskMessage + { + OrchestrationInstance = new OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }, + Event = new TaskScheduledEvent(scheduledId, "activity", "v1") { Input = "input", Timestamp = ScheduledTime }, + }, + }; + + static Task DispatchAsync(TaskActivityDispatcher dispatcher, TaskActivityWorkItem workItem) + { + // Exercise the real dispatcher without background polling or changing production visibility. + MethodInfo? process = typeof(TaskActivityDispatcher).GetMethod("OnProcessWorkItemAsync", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.IsNotNull(process); + return (Task)process.Invoke(dispatcher, new object[] { workItem })!; + } + + static HistoryEvent CreateResponse(bool failed, int scheduledId, DateTime? startedTime) => failed + ? new TaskFailedEvent(-1, scheduledId, "failure", "details") { StartedTime = startedTime } + : new TaskCompletedEvent(-1, scheduledId, "result") { StartedTime = startedTime }; + + static DateTime? GetStartedTime(HistoryEvent e) => e is TaskCompletedEvent completed + ? completed.StartedTime + : ((TaskFailedEvent)e).StartedTime; + + static int GetScheduledId(HistoryEvent e) => e is TaskCompletedEvent completed + ? completed.TaskScheduledId + : ((TaskFailedEvent)e).TaskScheduledId; + + static void AssertKnownStart(HistoryEvent response) + { + DateTime? startedTime = GetStartedTime(response); + Assert.IsNotNull(startedTime); + Assert.AreEqual(DateTimeKind.Utc, startedTime.Value.Kind); + Assert.AreNotEqual(ScheduledTime, startedTime.Value); + } + + sealed class RecordingActivity : TaskActivity + { + readonly Func> run; + + public RecordingActivity(Func> run) => this.run = run; + + public List Invocations { get; } = new List(); + + public override string Run(TaskContext context, string input) => throw new NotSupportedException(); + + public override Task RunAsync(TaskContext context, string input) + { + this.Invocations.Add(context); + return this.run(context, input); + } + } + + sealed class RecordingService : LocalOrchestrationService, IOrchestrationService + { + public List Responses { get; } = new List(); + + public int AbandonCount { get; private set; } + + public new Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workItem, TaskMessage responseMessage) + { + this.Responses.Add(responseMessage); + return Task.CompletedTask; + } + + public new Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) + { + this.AbandonCount++; + return Task.CompletedTask; + } + } + } +}