diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index 1a71fc0d4c..a54199a122 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -72,7 +72,17 @@ static async Task> ExecuteMergeRequest( return TypedResults.Problem($"Project is blocked from syncing. Reason: {blockInfo.Reason}", statusCode: StatusCodes.Status423Locked); } - syncHostedService.QueueJob(projectId); + switch (syncHostedService.QueueJob(projectId)) + { + case QueueJobResult.SyncAlreadyRunning: + activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running"); + return TypedResults.Problem("A sync is already running for this project, try again once it has finished", + statusCode: StatusCodes.Status429TooManyRequests); + case QueueJobResult.QueueFull: + activity?.SetStatus(ActivityStatusCode.Error, "Sync queue is full"); + return TypedResults.Problem("Unable to queue sync job", statusCode: StatusCodes.Status503ServiceUnavailable); + // Queued and AlreadyQueued both mean a sync that hasn't started yet will pick up the caller's changes + } activity?.SetStatus(ActivityStatusCode.Ok, "Sync job queued"); return TypedResults.Ok(); } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index ec01da8db0..33bc51334a 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -16,15 +16,40 @@ namespace FwHeadless.Services; +public enum QueueJobResult +{ + Queued, + AlreadyQueued, + // The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes + SyncAlreadyRunning, + QueueFull, +} + public class SyncHostedService(IServiceProvider services, ILogger logger, IMemoryCache memoryCache) : BackgroundService { private readonly Channel _projectsToSync = Channel.CreateUnbounded(); - private readonly ConcurrentDictionary> _projectsQueuedOrRunning = new(); + + // Projects with a sync queued or running. A single atomic GetOrAdd decides whether a request queues a new sync, + // so no lock is needed (this requires that SyncHostedService be a singleton). + private readonly ConcurrentDictionary _syncJobs = new(); + + /// + /// The TaskCompletionSource carries the outcome to anyone long-polling AwaitSyncFinished. + /// distinguishes a queued sync (which will pick up any changes made before it starts) + /// from a running one (which might not). + /// + private record SyncJob(TaskCompletionSource Result, bool Started = false) + { + // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on + // the thread that finished the sync. + public static SyncJob New() => new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) { + MarkSyncStarted(projectId); using var activity = FwHeadlessActivitySource.Value.StartActivity("SyncHostedService.ExecuteAsync"); await using var scope = services.CreateAsyncScope(); var syncWorker = ActivatorUtilities.CreateInstance(scope.ServiceProvider, projectId); @@ -48,44 +73,55 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Give clients a bit more time to poll the status CacheRecentSyncResult(projectId, result); - _projectsQueuedOrRunning.TryRemove(projectId, out var tcs); - tcs?.TrySetResult(result); + if (_syncJobs.TryRemove(projectId, out var job)) + job.Result.TrySetResult(result); } } + // QueueJob never modifies an existing entry, and only ExecuteAsync removes one, so nothing can race this update + internal void MarkSyncStarted(Guid projectId) + { + if (_syncJobs.TryGetValue(projectId, out var job)) + _syncJobs.TryUpdate(projectId, job with { Started = true }, job); + } + public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _projectsQueuedOrRunning.ContainsKey(projectId); + return _syncJobs.ContainsKey(projectId); } public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) { - if (_projectsQueuedOrRunning.TryGetValue(projectId, out var tcs)) - return await tcs.Task.WaitAsync(cancellationToken); + if (_syncJobs.TryGetValue(projectId, out var job)) + return await job.Result.Task.WaitAsync(cancellationToken); return TryGetRecentSyncResult(projectId); } - public bool QueueJob(Guid projectId) + public QueueJobResult QueueJob(Guid projectId) { + var newJob = SyncJob.New(); + var job = _syncJobs.GetOrAdd(projectId, newJob); //will only queue job if it's not already queued - var addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); - if (addedToQueue) + if (!ReferenceEquals(job, newJob)) { - if (!_projectsToSync.Writer.TryWrite(projectId)) + if (job.Started) { - logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); - _projectsQueuedOrRunning.TryRemove(projectId, out _); - return false; + logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId); + return QueueJobResult.SyncAlreadyRunning; } - - logger.LogInformation("Queued sync job for project {ProjectId}", projectId); + logger.LogInformation("Project {ProjectId} is already queued", projectId); + return QueueJobResult.AlreadyQueued; } - else + + if (!_projectsToSync.Writer.TryWrite(projectId)) { - logger.LogInformation("Project {ProjectId} is already queued", projectId); + logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); + _syncJobs.TryRemove(KeyValuePair.Create(projectId, newJob)); + return QueueJobResult.QueueFull; } - return addedToQueue; + logger.LogInformation("Queued sync job for project {ProjectId}", projectId); + return QueueJobResult.Queued; } private void CacheRecentSyncResult(Guid projectId, SyncJobResult result) diff --git a/backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs new file mode 100644 index 0000000000..67cd97a00b --- /dev/null +++ b/backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs @@ -0,0 +1,53 @@ +using FwHeadless.Services; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.FwHeadless; + +/// +/// Unit tests for how SyncHostedService answers a sync request, depending on whether a sync for the +/// project is already queued or already running. +/// +public class SyncHostedServiceQueueTests +{ + private static SyncHostedService NewService() => + new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + + [Fact] + public void QueueJob_reports_a_second_request_as_already_queued_while_the_sync_has_not_started() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued, + "the queued sync hasn't read the repo yet, so it will pick up the caller's changes"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + } + + [Fact] + public void QueueJob_is_refused_while_a_sync_is_running() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.MarkSyncStarted(projectId); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning, + "the running sync may have missed the caller's changes, so it must not be reported as covering them"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + } + + [Fact] + public void MarkSyncStarted_does_nothing_for_a_project_with_no_sync() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.MarkSyncStarted(projectId); + + svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + } +} diff --git a/docs/site/technical/sync/fwheadless-merge.md b/docs/site/technical/sync/fwheadless-merge.md index f4846e23ba..a80b212b04 100644 --- a/docs/site/technical/sync/fwheadless-merge.md +++ b/docs/site/technical/sync/fwheadless-merge.md @@ -15,8 +15,10 @@ There is no scheduler, no cron and no Mercurial hook. A merge happens because so Both go `POST /api/fw-lite/sync/trigger/{projectId}` (Lexbox) → `POST /api/merge/execute` (FwHeadless), which only *queues* the job. [One global worker](https://github.com/sillsdev/languageforge-lexbox/blob/develop/backend/FwHeadless/Services/SyncHostedService.cs) processes projects one -at a time, so a queued project waits. Queuing is deduplicated: triggering a project that's already queued or running -is a no-op. +at a time, so a queued project waits. Queuing is deduplicated: triggering a project that's already queued is a no-op, +because the queued job hasn't read the repo yet and will pick up the caller's changes. Triggering a project whose job is +already *running* is refused with `429`, since that job may have read the repo before the caller's changes arrived; the +caller sees "A sync is already running for this project" and tries again once it finishes. The client then polls for the result, but the job doesn't depend on it: closing FieldWorks Lite or the web page doesn't stop a queued job. FieldWorks Lite stops waiting after **15 minutes**; the Lexbox web button keeps waiting. The **first-ever**