Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion backend/FwHeadless/Routes/MergeRoutes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,17 @@ static async Task<Results<Ok, ProblemHttpResult>> ExecuteMergeRequest(
return TypedResults.Problem($"Project is blocked from syncing. Reason: {blockInfo.Reason}", statusCode: StatusCodes.Status423Locked);
}

syncHostedService.QueueJob(projectId);
switch (syncHostedService.QueueJob(projectId))
{
case QueueJobResult.SyncAlreadyRunning:
activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running");
return TypedResults.Problem("A sync is already running for this project, try again once it has finished",
statusCode: StatusCodes.Status429TooManyRequests);
case QueueJobResult.QueueFull:
activity?.SetStatus(ActivityStatusCode.Error, "Sync queue is full");
return TypedResults.Problem("Unable to queue sync job", statusCode: StatusCodes.Status503ServiceUnavailable);
// Queued and AlreadyQueued both mean a sync that hasn't started yet will pick up the caller's changes
}
activity?.SetStatus(ActivityStatusCode.Ok, "Sync job queued");
return TypedResults.Ok();
}
Expand Down
72 changes: 54 additions & 18 deletions backend/FwHeadless/Services/SyncHostedService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,40 @@

namespace FwHeadless.Services;

public enum QueueJobResult
{
Queued,
AlreadyQueued,
// The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes
SyncAlreadyRunning,
QueueFull,
}

public class SyncHostedService(IServiceProvider services, ILogger<SyncHostedService> logger, IMemoryCache memoryCache) : BackgroundService
{
private readonly Channel<Guid> _projectsToSync = Channel.CreateUnbounded<Guid>();
private readonly ConcurrentDictionary<Guid, TaskCompletionSource<SyncJobResult>> _projectsQueuedOrRunning = new();

// Projects with a sync queued or running. A single atomic GetOrAdd decides whether a request queues a new sync,
// so no lock is needed (this requires that SyncHostedService be a singleton).
private readonly ConcurrentDictionary<Guid, SyncJob> _syncJobs = new();

/// <summary>
/// The TaskCompletionSource carries the outcome to anyone long-polling AwaitSyncFinished.
/// <see cref="Started"/> distinguishes a queued sync (which will pick up any changes made before it starts)
/// from a running one (which might not).
/// </summary>
private record SyncJob(TaskCompletionSource<SyncJobResult> Result, bool Started = false)
{
// RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on
// the thread that finished the sync.
public static SyncJob New() => new(new TaskCompletionSource<SyncJobResult>(TaskCreationOptions.RunContinuationsAsynchronously));
}

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken))
{
MarkSyncStarted(projectId);
using var activity = FwHeadlessActivitySource.Value.StartActivity("SyncHostedService.ExecuteAsync");
await using var scope = services.CreateAsyncScope();
var syncWorker = ActivatorUtilities.CreateInstance<SyncWorker>(scope.ServiceProvider, projectId);
Expand All @@ -48,44 +73,55 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)

// Give clients a bit more time to poll the status
CacheRecentSyncResult(projectId, result);
_projectsQueuedOrRunning.TryRemove(projectId, out var tcs);
tcs?.TrySetResult(result);
if (_syncJobs.TryRemove(projectId, out var job))
job.Result.TrySetResult(result);
}
}

// QueueJob never modifies an existing entry, and only ExecuteAsync removes one, so nothing can race this update
internal void MarkSyncStarted(Guid projectId)
{
if (_syncJobs.TryGetValue(projectId, out var job))
_syncJobs.TryUpdate(projectId, job with { Started = true }, job);
}

public virtual bool IsJobQueuedOrRunning(Guid projectId)
{
return _projectsQueuedOrRunning.ContainsKey(projectId);
return _syncJobs.ContainsKey(projectId);
}

public async Task<SyncJobResult?> AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken)
{
if (_projectsQueuedOrRunning.TryGetValue(projectId, out var tcs))
return await tcs.Task.WaitAsync(cancellationToken);
if (_syncJobs.TryGetValue(projectId, out var job))
return await job.Result.Task.WaitAsync(cancellationToken);
return TryGetRecentSyncResult(projectId);
}

public bool QueueJob(Guid projectId)
public QueueJobResult QueueJob(Guid projectId)
{
var newJob = SyncJob.New();
var job = _syncJobs.GetOrAdd(projectId, newJob);
//will only queue job if it's not already queued
var addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new());
if (addedToQueue)
if (!ReferenceEquals(job, newJob))
{
if (!_projectsToSync.Writer.TryWrite(projectId))
if (job.Started)
{
logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId);
_projectsQueuedOrRunning.TryRemove(projectId, out _);
return false;
logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId);
return QueueJobResult.SyncAlreadyRunning;
}

logger.LogInformation("Queued sync job for project {ProjectId}", projectId);
logger.LogInformation("Project {ProjectId} is already queued", projectId);
return QueueJobResult.AlreadyQueued;
}
else

if (!_projectsToSync.Writer.TryWrite(projectId))
{
logger.LogInformation("Project {ProjectId} is already queued", projectId);
logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId);
_syncJobs.TryRemove(KeyValuePair.Create(projectId, newJob));
return QueueJobResult.QueueFull;
}

return addedToQueue;
logger.LogInformation("Queued sync job for project {ProjectId}", projectId);
return QueueJobResult.Queued;
}

private void CacheRecentSyncResult(Guid projectId, SyncJobResult result)
Expand Down
53 changes: 53 additions & 0 deletions backend/Testing/FwHeadless/SyncHostedServiceQueueTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
using FwHeadless.Services;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;

namespace Testing.FwHeadless;

/// <summary>
/// Unit tests for how SyncHostedService answers a sync request, depending on whether a sync for the
/// project is already queued or already running.
/// </summary>
public class SyncHostedServiceQueueTests
{
private static SyncHostedService NewService() =>
new(services: null!, NullLogger<SyncHostedService>.Instance, new MemoryCache(new MemoryCacheOptions()));

[Fact]
public void QueueJob_reports_a_second_request_as_already_queued_while_the_sync_has_not_started()
{
var svc = NewService();
var projectId = Guid.NewGuid();

svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued);
svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued,
"the queued sync hasn't read the repo yet, so it will pick up the caller's changes");
svc.IsJobQueuedOrRunning(projectId).Should().BeTrue();
}

[Fact]
public void QueueJob_is_refused_while_a_sync_is_running()
{
var svc = NewService();
var projectId = Guid.NewGuid();

svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued);
svc.MarkSyncStarted(projectId);

svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning,
"the running sync may have missed the caller's changes, so it must not be reported as covering them");
svc.IsJobQueuedOrRunning(projectId).Should().BeTrue();
}

[Fact]
public void MarkSyncStarted_does_nothing_for_a_project_with_no_sync()
{
var svc = NewService();
var projectId = Guid.NewGuid();

svc.MarkSyncStarted(projectId);

svc.IsJobQueuedOrRunning(projectId).Should().BeFalse();
svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued);
}
}
6 changes: 4 additions & 2 deletions docs/site/technical/sync/fwheadless-merge.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@ There is no scheduler, no cron and no Mercurial hook. A merge happens because so

Both go `POST /api/fw-lite/sync/trigger/{projectId}` (Lexbox) → `POST /api/merge/execute` (FwHeadless), which
only *queues* the job. [One global worker](https://github.com/sillsdev/languageforge-lexbox/blob/develop/backend/FwHeadless/Services/SyncHostedService.cs) processes projects one
at a time, so a queued project waits. Queuing is deduplicated: triggering a project that's already queued or running
is a no-op.
at a time, so a queued project waits. Queuing is deduplicated: triggering a project that's already queued is a no-op,
because the queued job hasn't read the repo yet and will pick up the caller's changes. Triggering a project whose job is
already *running* is refused with `429`, since that job may have read the repo before the caller's changes arrived; the
caller sees "A sync is already running for this project" and tries again once it finishes.

The client then polls for the result, but the job doesn't depend on it: closing FieldWorks Lite or the web page
doesn't stop a queued job. FieldWorks Lite stops waiting after **15 minutes**; the Lexbox web button keeps waiting. The **first-ever**
Expand Down
Loading