From 94f3d19655dfa0543fb994aa623405ce5a3bb64d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 9 Nov 2026 15:54:46 +0700 Subject: [PATCH] Refuse a sync request while that project is already syncing A sync request that arrived while the project's sync was already running got a 200, as if a sync covering the caller's changes had been queued. But the running sync may have read the repo before those changes were pushed, so the caller could wrongly conclude its changes were synced. QueueJob now returns a QueueJobResult and tracks whether a queued sync has started. A request that finds the sync still waiting in the queue is covered by it and still gets a 200; one that finds it already running gets 429, and a full queue gets 503 instead of a false 200. Split out of #2439. Co-authored-by: Robin Munn Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Routes/MergeRoutes.cs | 12 +++- .../FwHeadless/Services/SyncHostedService.cs | 72 ++++++++++++++----- .../FwHeadless/SyncHostedServiceQueueTests.cs | 53 ++++++++++++++ docs/site/technical/sync/fwheadless-merge.md | 6 +- 4 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index 1a71fc0d4c..a54199a122 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -72,7 +72,17 @@ static async Task> ExecuteMergeRequest( return TypedResults.Problem($"Project is blocked from syncing. Reason: {blockInfo.Reason}", statusCode: StatusCodes.Status423Locked); } - syncHostedService.QueueJob(projectId); + switch (syncHostedService.QueueJob(projectId)) + { + case QueueJobResult.SyncAlreadyRunning: + activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running"); + return TypedResults.Problem("A sync is already running for this project, try again once it has finished", + statusCode: StatusCodes.Status429TooManyRequests); + case QueueJobResult.QueueFull: + activity?.SetStatus(ActivityStatusCode.Error, "Sync queue is full"); + return TypedResults.Problem("Unable to queue sync job", statusCode: StatusCodes.Status503ServiceUnavailable); + // Queued and AlreadyQueued both mean a sync that hasn't started yet will pick up the caller's changes + } activity?.SetStatus(ActivityStatusCode.Ok, "Sync job queued"); return TypedResults.Ok(); } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index ec01da8db0..33bc51334a 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -16,15 +16,40 @@ namespace FwHeadless.Services; +public enum QueueJobResult +{ + Queued, + AlreadyQueued, + // The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes + SyncAlreadyRunning, + QueueFull, +} + public class SyncHostedService(IServiceProvider services, ILogger logger, IMemoryCache memoryCache) : BackgroundService { private readonly Channel _projectsToSync = Channel.CreateUnbounded(); - private readonly ConcurrentDictionary> _projectsQueuedOrRunning = new(); + + // Projects with a sync queued or running. A single atomic GetOrAdd decides whether a request queues a new sync, + // so no lock is needed (this requires that SyncHostedService be a singleton). + private readonly ConcurrentDictionary _syncJobs = new(); + + /// + /// The TaskCompletionSource carries the outcome to anyone long-polling AwaitSyncFinished. + /// distinguishes a queued sync (which will pick up any changes made before it starts) + /// from a running one (which might not). + /// + private record SyncJob(TaskCompletionSource Result, bool Started = false) + { + // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on + // the thread that finished the sync. + public static SyncJob New() => new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) { + MarkSyncStarted(projectId); using var activity = FwHeadlessActivitySource.Value.StartActivity("SyncHostedService.ExecuteAsync"); await using var scope = services.CreateAsyncScope(); var syncWorker = ActivatorUtilities.CreateInstance(scope.ServiceProvider, projectId); @@ -48,44 +73,55 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Give clients a bit more time to poll the status CacheRecentSyncResult(projectId, result); - _projectsQueuedOrRunning.TryRemove(projectId, out var tcs); - tcs?.TrySetResult(result); + if (_syncJobs.TryRemove(projectId, out var job)) + job.Result.TrySetResult(result); } } + // QueueJob never modifies an existing entry, and only ExecuteAsync removes one, so nothing can race this update + internal void MarkSyncStarted(Guid projectId) + { + if (_syncJobs.TryGetValue(projectId, out var job)) + _syncJobs.TryUpdate(projectId, job with { Started = true }, job); + } + public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _projectsQueuedOrRunning.ContainsKey(projectId); + return _syncJobs.ContainsKey(projectId); } public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) { - if (_projectsQueuedOrRunning.TryGetValue(projectId, out var tcs)) - return await tcs.Task.WaitAsync(cancellationToken); + if (_syncJobs.TryGetValue(projectId, out var job)) + return await job.Result.Task.WaitAsync(cancellationToken); return TryGetRecentSyncResult(projectId); } - public bool QueueJob(Guid projectId) + public QueueJobResult QueueJob(Guid projectId) { + var newJob = SyncJob.New(); + var job = _syncJobs.GetOrAdd(projectId, newJob); //will only queue job if it's not already queued - var addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); - if (addedToQueue) + if (!ReferenceEquals(job, newJob)) { - if (!_projectsToSync.Writer.TryWrite(projectId)) + if (job.Started) { - logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); - _projectsQueuedOrRunning.TryRemove(projectId, out _); - return false; + logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId); + return QueueJobResult.SyncAlreadyRunning; } - - logger.LogInformation("Queued sync job for project {ProjectId}", projectId); + logger.LogInformation("Project {ProjectId} is already queued", projectId); + return QueueJobResult.AlreadyQueued; } - else + + if (!_projectsToSync.Writer.TryWrite(projectId)) { - logger.LogInformation("Project {ProjectId} is already queued", projectId); + logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); + _syncJobs.TryRemove(KeyValuePair.Create(projectId, newJob)); + return QueueJobResult.QueueFull; } - return addedToQueue; + logger.LogInformation("Queued sync job for project {ProjectId}", projectId); + return QueueJobResult.Queued; } private void CacheRecentSyncResult(Guid projectId, SyncJobResult result) diff --git a/backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs new file mode 100644 index 0000000000..67cd97a00b --- /dev/null +++ b/backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs @@ -0,0 +1,53 @@ +using FwHeadless.Services; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.FwHeadless; + +/// +/// Unit tests for how SyncHostedService answers a sync request, depending on whether a sync for the +/// project is already queued or already running. +/// +public class SyncHostedServiceQueueTests +{ + private static SyncHostedService NewService() => + new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + + [Fact] + public void QueueJob_reports_a_second_request_as_already_queued_while_the_sync_has_not_started() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued, + "the queued sync hasn't read the repo yet, so it will pick up the caller's changes"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + } + + [Fact] + public void QueueJob_is_refused_while_a_sync_is_running() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.MarkSyncStarted(projectId); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning, + "the running sync may have missed the caller's changes, so it must not be reported as covering them"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + } + + [Fact] + public void MarkSyncStarted_does_nothing_for_a_project_with_no_sync() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.MarkSyncStarted(projectId); + + svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + } +} diff --git a/docs/site/technical/sync/fwheadless-merge.md b/docs/site/technical/sync/fwheadless-merge.md index f4846e23ba..a80b212b04 100644 --- a/docs/site/technical/sync/fwheadless-merge.md +++ b/docs/site/technical/sync/fwheadless-merge.md @@ -15,8 +15,10 @@ There is no scheduler, no cron and no Mercurial hook. A merge happens because so Both go `POST /api/fw-lite/sync/trigger/{projectId}` (Lexbox) → `POST /api/merge/execute` (FwHeadless), which only *queues* the job. [One global worker](https://github.com/sillsdev/languageforge-lexbox/blob/develop/backend/FwHeadless/Services/SyncHostedService.cs) processes projects one -at a time, so a queued project waits. Queuing is deduplicated: triggering a project that's already queued or running -is a no-op. +at a time, so a queued project waits. Queuing is deduplicated: triggering a project that's already queued is a no-op, +because the queued job hasn't read the repo yet and will pick up the caller's changes. Triggering a project whose job is +already *running* is refused with `429`, since that job may have read the repo before the caller's changes arrived; the +caller sees "A sync is already running for this project" and tries again once it finishes. The client then polls for the result, but the job doesn't depend on it: closing FieldWorks Lite or the web page doesn't stop a queued job. FieldWorks Lite stops waiting after **15 minutes**; the Lexbox web button keeps waiting. The **first-ever**