From 71c590b943f9b6309cf8d23f5484f362f983f8c4 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:06:12 +0700 Subject: [PATCH 01/60] Add AnthropologyCategories enum Shared by LexBoxApi (query param) and FwHeadless (creation handler); None is the default. Values: None, Standard, Enhanced. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/LexCore/Entities/AnthropologyCategories.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 backend/LexCore/Entities/AnthropologyCategories.cs diff --git a/backend/LexCore/Entities/AnthropologyCategories.cs b/backend/LexCore/Entities/AnthropologyCategories.cs new file mode 100644 index 0000000000..bfddf2d1dd --- /dev/null +++ b/backend/LexCore/Entities/AnthropologyCategories.cs @@ -0,0 +1,11 @@ +namespace LexCore.Entities; + +/// +/// Which set of anthropology categories to populate a newly-created project with. +/// +public enum AnthropologyCategories +{ + None = 0, + Standard = 1, + Enhanced = 2, +} From cbbb91447dcd9f4598bf220a82f3bec9bd50da92 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:14:56 +0700 Subject: [PATCH 02/60] Add FwHeadless ProjectCreationService MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds a new FieldWorks project from the SIL.LCModel template and pushes it into the empty hg repo LexBox already created: clone the empty remote → build fw.fwdata (first vernacular + first analysis WS) → add the remaining writing systems via the existing CreateWritingSystem surface (dedup + save-on-dispose, then release LCM file locks) → hg add/commit → push. Guards the fdoDataModelVersion, cleans up the local folder on failure, and stubs anthropologyCategories (// TODO: Implement). Adds a per-project creation reservation to SyncHostedService so a sync can't be queued for a project mid-creation (and two creations can't race). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/ProjectCreationService.cs | 174 ++++++++++++++++++ .../FwHeadless/Services/SyncHostedService.cs | 28 ++- 2 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 backend/FwHeadless/Services/ProjectCreationService.cs diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs new file mode 100644 index 0000000000..e8e1a4ef25 --- /dev/null +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -0,0 +1,174 @@ +using System.Xml; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Api; +using FwDataMiniLcmBridge.LcmUtils; +using LexCore.Entities; +using LexCore.Exceptions; +using Microsoft.Extensions.Options; +using MiniLcm.Models; + +namespace FwHeadless.Services; + +/// +/// Creates a brand-new FieldWorks project from the SIL.LCModel template and pushes it into the +/// (empty) Mercurial repo that LexBox has already initialised for the project. +/// +public class ProjectCreationService( + IOptions config, + ISendReceiveService srService, + FwDataFactory fwDataFactory, + IProjectLoader projectLoader, + SyncHostedService syncHostedService, + ILogger logger) +{ + private const string InitialCommitMessage = "Initial project creation"; + + public async Task CreateFromTemplate( + Guid projectId, + string projectCode, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems) + { + if (vernacularWritingSystems.Count == 0) + throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); + if (analysisWritingSystems.Count == 0) + throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); + + // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. + if (!syncHostedService.TryStartProjectCreation(projectId)) + throw new ProjectSyncInProgressException(projectId); + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + try + { + // Populate the empty repo using only already-exercised FwHeadless primitives: + // 1. Clone the empty remote -> local fw/ folder (.hg, no fwdata yet). + // 2. Build fw.fwdata into that folder from the SIL.LCModel template. + // 3. Add the remaining writing systems, saving and releasing the LCM file locks. + // 4. hg add + commit the (untracked) fwdata, then push. + var cloneResult = await srService.Clone(fwDataProject, projectCode); + if (!cloneResult.Success) + throw new SendReceiveException("Clone of the new project's repo failed", cloneResult); + + BuildFromTemplate(fwDataProject, vernacularWritingSystems[0], analysisWritingSystems[0]); + await AddWritingSystems(fwDataProject, vernacularWritingSystems, analysisWritingSystems); + ApplyAnthropologyCategories(fwDataProject); + AssertModelVersionMatches(fwDataProject); + + await srService.CommitFile(fwDataProject.FilePath, InitialCommitMessage); + var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage); + if (!pushResult.Success) + throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); + + logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId); + } + catch + { + CleanupLocalProject(fwDataProject); + throw; + } + finally + { + syncHostedService.EndProjectCreation(projectId); + } + } + + private void BuildFromTemplate(FwDataProject fwDataProject, string vernacularWs, string analysisWs) + { + // NewProject copies the SIL.LCModel template (with the first vernacular + analysis WS) and + // returns a loaded cache; dispose it right away so its file locks are released before we + // re-open the project through FwDataFactory to add the remaining writing systems. + using var cache = projectLoader.NewProject(fwDataProject, analysisWs, vernacularWs); + } + + private async Task AddWritingSystems( + FwDataProject fwDataProject, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems) + { + // The template already has the first vernacular + first analysis WS; add the rest, deduping + // against what's present. saveOnDispose flushes the additions to the .fwdata XML when the api + // is disposed; CloseProjectAsync then disposes the underlying LcmCache so the following hg + // add/commit doesn't hit a file lock. + try + { + using (var api = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, saveOnDispose: true)) + { + var existing = await api.GetWritingSystems(); + var vernacularPresent = existing.Vernacular.Select(ws => ws.WsId).ToHashSet(); + var analysisPresent = existing.Analysis.Select(ws => ws.WsId).ToHashSet(); + foreach (var code in vernacularWritingSystems) + await AddWritingSystem(api, code, WritingSystemType.Vernacular, vernacularPresent); + foreach (var code in analysisWritingSystems) + await AddWritingSystem(api, code, WritingSystemType.Analysis, analysisPresent); + } + } + finally + { + await fwDataFactory.CloseProjectAsync(fwDataProject); + } + } + + private static async Task AddWritingSystem(FwDataMiniLcmApi api, string code, WritingSystemType type, HashSet present) + { + var wsId = new WritingSystemId(code); + if (!present.Add(wsId)) return; // already present (the template's first WS, or a repeated request) + await api.CreateWritingSystem(new WritingSystem + { + Id = Guid.Empty, + WsId = wsId, + Name = code, + Abbreviation = code, + Font = "Charis SIL", + Type = type, + }); + } + + private void ApplyAnthropologyCategories(FwDataProject fwDataProject) + { + // TODO: Implement — populate the project with the requested anthropology categories + // (enhanced/standard). Currently a no-op; 'none' is the only fully-handled option. + } + + /// + /// The freshly-created project is stamped with the SIL.LCModel package's data-model version. + /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data + /// (see FwHeadless AGENTS.md), so fail before the push rather than after. + /// + private void AssertModelVersionMatches(FwDataProject fwDataProject) + { + var expected = config.Value.FdoDataModelVersion; + var actual = ReadModelVersion(fwDataProject.FilePath); + if (actual is not null && actual != expected) + { + throw new InvalidOperationException( + $"New project's data-model version '{actual}' does not match FwHeadless FdoDataModelVersion " + + $"'{expected}'; refusing to push to avoid data corruption."); + } + } + + private static string? ReadModelVersion(string fwDataFilePath) + { + using var reader = XmlReader.Create(fwDataFilePath, new XmlReaderSettings { IgnoreComments = true, IgnoreWhitespace = true }); + // The .fwdata root element carries the FDO model version, e.g. . + return reader.MoveToContent() == XmlNodeType.Element ? reader.GetAttribute("version") : null; + } + + private void CleanupLocalProject(FwDataProject fwDataProject) + { + // Mirror SyncWorker.SetupFwData's failure cleanup so a retry starts clean. The remote repo + // (created by LexBox) is torn down by the caller on the LexBox side. + try + { + if (Directory.Exists(fwDataProject.ProjectFolder)) + Directory.Delete(fwDataProject.ProjectFolder, true); + if (Directory.Exists(fwDataProject.ProjectsPath) && + !Directory.EnumerateFileSystemEntries(fwDataProject.ProjectsPath).Any()) + Directory.Delete(fwDataProject.ProjectsPath); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to clean up local project folder after a failed creation: {ProjectFolder}", + fwDataProject.ProjectFolder); + } + } +} diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 91b77edead..3144b9d5db 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -20,6 +20,10 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsToSync = Channel.CreateUnbounded(); private readonly ConcurrentDictionary> _projectsQueuedOrRunning = new(); + // Projects currently being created from a template. Tracked here (not just in ProjectCreationService) + // so a sync job can't be queued for a project mid-creation and race on the same fw/ folder and repo. + private readonly ConcurrentDictionary _projectsBeingCreated = new(); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) @@ -54,7 +58,23 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _projectsQueuedOrRunning.ContainsKey(projectId); + return _projectsQueuedOrRunning.ContainsKey(projectId) || _projectsBeingCreated.ContainsKey(projectId); + } + + /// + /// Reserves a project for creation-from-template, blocking a concurrent creation or sync of the + /// same project. Returns false if a sync is already queued/running or another creation is in flight. + /// Pair with in a finally block. + /// + public bool TryStartProjectCreation(Guid projectId) + { + if (_projectsQueuedOrRunning.ContainsKey(projectId)) return false; + return _projectsBeingCreated.TryAdd(projectId, 0); + } + + public void EndProjectCreation(Guid projectId) + { + _projectsBeingCreated.TryRemove(projectId, out _); } public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) @@ -66,6 +86,12 @@ public virtual bool IsJobQueuedOrRunning(Guid projectId) public bool QueueJob(Guid projectId) { + //don't sync a project while it's still being created from a template (would race on the repo) + if (_projectsBeingCreated.ContainsKey(projectId)) + { + logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); + return false; + } //will only queue job if it's not already queued var addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); if (addedToQueue) From 466212a8d096b32e9637c23e0ed1807e227bd7f9 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:18:01 +0700 Subject: [PATCH 03/60] Add FwHeadless create-from-template route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/project/create-from-template (projectId query param + JSON body of vernacular/analysis writing systems and anthropologyCategories). Validates the WS tags with IetfLanguageTag at the boundary, resolves the code via IProjectLookupService, and delegates to ProjectCreationService. Registers the service and maps the route. Threads AnthropologyCategories through to the stub. Internal endpoint (cluster-only, no auth) — LexBoxApi does the admin check and the project/repo creation before calling this. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/FwHeadlessKernel.cs | 1 + backend/FwHeadless/Program.cs | 1 + backend/FwHeadless/Routes/ProjectRoutes.cs | 46 +++++++++++++++++++ .../Services/ProjectCreationService.cs | 7 +-- .../CreateProjectFromTemplateInput.cs | 11 +++++ 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 backend/FwHeadless/Routes/ProjectRoutes.cs create mode 100644 backend/LexCore/Entities/CreateProjectFromTemplateInput.cs diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index 51700d8f53..0d62de9a48 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -25,6 +25,7 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index 79e1cb3ae1..ed832e53e9 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -83,6 +83,7 @@ app.MapDefaultEndpoints(); app.MapMediaFileRoutes(); app.MapMergeRoutes(); +app.MapProjectRoutes(); // DELETE endpoint to delete the FieldWorks repo/project (and nothing else) app.MapDelete("/api/manage/repo/{projectId}", async (Guid projectId, ProjectDeletionService deletionService) => diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs new file mode 100644 index 0000000000..90960430e1 --- /dev/null +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -0,0 +1,46 @@ +using FwHeadless.Services; +using LexCore.Entities; +using Microsoft.AspNetCore.Http.HttpResults; +using SIL.WritingSystems; + +namespace FwHeadless.Routes; + +public static class ProjectRoutes +{ + public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app) + { + var group = app.MapGroup("/api/project"); + group.MapPost("/create-from-template", CreateProjectFromTemplate); + return group; + } + + // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the + // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the + // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates. + private static async Task> CreateProjectFromTemplate( + Guid projectId, + CreateProjectFromTemplateInput input, + IProjectLookupService projectLookupService, + ProjectCreationService projectCreationService, + ILogger logger) + { + if (input.WsVernacular.Count == 0) + return TypedResults.Problem("At least one vernacular writing system is required", + statusCode: StatusCodes.Status400BadRequest); + var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); + if (invalidWs is not null) + return TypedResults.Problem($"Invalid writing system code: {invalidWs}", + statusCode: StatusCodes.Status400BadRequest); + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) + { + logger.LogError("Create-from-template request for non-existent project {ProjectId}", projectId); + return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound); + } + + await projectCreationService.CreateFromTemplate( + projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.AnthropologyCategories); + return TypedResults.Ok(); + } +} diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index e8e1a4ef25..c6434fac28 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -27,7 +27,8 @@ public async Task CreateFromTemplate( Guid projectId, string projectCode, IReadOnlyList vernacularWritingSystems, - IReadOnlyList analysisWritingSystems) + IReadOnlyList analysisWritingSystems, + AnthropologyCategories anthropologyCategories) { if (vernacularWritingSystems.Count == 0) throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); @@ -51,7 +52,7 @@ public async Task CreateFromTemplate( BuildFromTemplate(fwDataProject, vernacularWritingSystems[0], analysisWritingSystems[0]); await AddWritingSystems(fwDataProject, vernacularWritingSystems, analysisWritingSystems); - ApplyAnthropologyCategories(fwDataProject); + ApplyAnthropologyCategories(anthropologyCategories); AssertModelVersionMatches(fwDataProject); await srService.CommitFile(fwDataProject.FilePath, InitialCommitMessage); @@ -123,7 +124,7 @@ await api.CreateWritingSystem(new WritingSystem }); } - private void ApplyAnthropologyCategories(FwDataProject fwDataProject) + private void ApplyAnthropologyCategories(AnthropologyCategories anthropologyCategories) { // TODO: Implement — populate the project with the requested anthropology categories // (enhanced/standard). Currently a no-op; 'none' is the only fully-handled option. diff --git a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs b/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs new file mode 100644 index 0000000000..adc9b10069 --- /dev/null +++ b/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs @@ -0,0 +1,11 @@ +namespace LexCore.Entities; + +/// +/// Body of the internal FwHeadless "create project from template" call. The vernacular/analysis +/// lists are already validated and defaulted by LexBoxApi (at least one vernacular; analysis +/// defaults to "en"). +/// +public record CreateProjectFromTemplateInput( + IReadOnlyList WsVernacular, + IReadOnlyList WsAnalysis, + AnthropologyCategories AnthropologyCategories); From 0638ee94ed3f1ef7c63a8632052a3e16806a9e9d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:19:31 +0700 Subject: [PATCH 04/60] Add FwHeadlessClient.CreateProjectFromTemplate Posts the vernacular/analysis writing systems and anthropologyCategories to the FwHeadless create-from-template endpoint for a given project id. Runs inline; returns null on success or the error body on failure (for the controller's saga rollback). Threads a CancellationToken like the sibling methods. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../LexBoxApi/Services/FwHeadlessClient.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 0ffc587077..45ab698894 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -1,5 +1,7 @@ using System.Net; +using System.Net.Http.Json; using System.Text.Json; +using LexCore.Entities; using LexCore.Exceptions; using LexCore.Sync; using Microsoft.AspNetCore.Mvc; @@ -197,4 +199,29 @@ public async Task UnblockProject(Guid projectId) return null; } + /// + /// Populates a newly-created (empty) project's repo with a template .fwdata configured for the + /// given writing systems. Runs inline (a new empty project is small). Returns null on success, + /// or the error body on failure. + /// + public async Task CreateProjectFromTemplate(Guid projectId, + IReadOnlyList wsVernacular, + IReadOnlyList wsAnalysis, + AnthropologyCategories anthropologyCategories, + CancellationToken cancellationToken = default) + { + var input = new CreateProjectFromTemplateInput(wsVernacular, wsAnalysis, anthropologyCategories); + var response = await httpClient.PostAsJsonAsync($"/api/project/create-from-template?projectId={projectId}", + input, + cancellationToken); + if (response.IsSuccessStatusCode) return null; + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + logger.LogError("Failed to create project from template: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + responseBody); + return responseBody; + } + } From d43b45aa74d3e418ec39654498a8225ede0fffae Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:22:26 +0700 Subject: [PATCH 05/60] Add admin API to create a project from a template POST /api/project/createFromTemplate (admin only): validates the code and that at least one wsVernacular is given (400), rejects a taken code (409), defaults wsAnalysis to ["en"], then creates the FLEx project (Postgres row + empty repo) and asks FwHeadless to populate the repo with the template .fwdata. On FwHeadless failure it compensates by deleting the repo + project row; on success it refreshes LastCommit and returns the new project id. wsVernacular/wsAnalysis are repeatable query params; anthropologyCategories (enhanced|standard|none) defaults to none. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Controllers/ProjectController.cs | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index b05e8b058c..a7b3a196f1 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,6 +1,8 @@ +using System.Text.RegularExpressions; using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; using LexBoxApi.Jobs; +using LexBoxApi.Models.Project; using LexBoxApi.Services; using LexCore.Entities; using LexCore.Exceptions; @@ -20,9 +22,96 @@ public class ProjectController( IHgService hgService, LexBoxDbContext lexBoxDbContext, IPermissionService permissionService, - ISchedulerFactory scheduler) + ISchedulerFactory scheduler, + FwHeadlessClient fwHeadlessClient, + ILogger logger) : ControllerBase { + /// + /// Admin-only: create a new FLEx project whose repo is populated with a template .fwdata + /// (from the SIL.LCModel package) configured for the requested writing systems. + /// + /// Project code for the new project. + /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. + /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. + /// Optional display name; defaults to the code. + /// Which anthropology categories to populate (default none). + [HttpPost("createFromTemplate")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status409Conflict)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public async Task> CreateFromTemplate( + string code, + [FromQuery] string[] wsVernacular, + [FromQuery] string[]? wsAnalysis = null, + string? name = null, + AnthropologyCategories anthropologyCategories = AnthropologyCategories.None, + CancellationToken cancellationToken = default) + { + if (wsVernacular is null || wsVernacular.Length == 0) + return Problem("At least one vernacular writing system is required", statusCode: StatusCodes.Status400BadRequest); + if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) + return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); + if (await projectService.ProjectExists(code)) + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + + // Default the analysis writing systems to English when none were supplied. + var wsAnalysisOrDefault = wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; + + var projectId = await projectService.CreateProject(new CreateProjectInput( + Id: null, + Name: string.IsNullOrWhiteSpace(name) ? code : name, + Description: string.Empty, + Code: code, + Type: ProjectType.FLEx, + RetentionPolicy: RetentionPolicy.Verified, + IsConfidential: false, + ProjectManagerId: null, + OrgId: null)); + + // Saga: the project row + empty repo now exist. Have FwHeadless populate the repo with the + // template .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs + // on a fresh token so a client disconnect can't skip it. + string? error; + try + { + error = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, anthropologyCategories, cancellationToken); + } + catch + { + await CleanupFailedCreation(projectId, code); + throw; + } + if (error is not null) + { + await CleanupFailedCreation(projectId, code); + return Problem($"Failed to create the project from the template: {error}", statusCode: StatusCodes.Status500InternalServerError); + } + + await projectService.UpdateLastCommit(code); + return projectId; + } + + private async Task CleanupFailedCreation(Guid projectId, string code) + { + try + { + await hgService.DeleteRepoIfExists(code); + var project = await lexBoxDbContext.Projects.FindAsync(projectId); + if (project is not null) + { + lexBoxDbContext.Projects.Remove(project); + await lexBoxDbContext.SaveChangesAsync(); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}) after a failed template creation", projectId, code); + } + } + [HttpPost("refreshProjectLastChanged")] public async Task RefreshProjectLastChanged(string projectCode) { From f15650592cf1fd3e0b883dd1482c30f0f7eb43a2 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:26:34 +0700 Subject: [PATCH 06/60] Test create-from-template validation and creation reservation Unit tests (no infrastructure): ProjectController.CreateFromTemplate rejects a missing vernacular WS and an invalid code with 400; SyncHostedService's creation reservation blocks a second concurrent creation and refuses to queue a sync while a project is being created, and is reusable after release. The end-to-end (template build + empty-repo first push) is integration-level and runs in CI. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ncHostedServiceCreationReservationTests.cs | 43 +++++++++++++++++++ .../CreateFromTemplateValidationTests.cs | 43 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs create mode 100644 backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs new file mode 100644 index 0000000000..c395e1331b --- /dev/null +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -0,0 +1,43 @@ +using FwHeadless.Services; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.FwHeadless; + +/// +/// Unit tests for the per-project creation reservation that stops a sync from racing a project that +/// is still being created from a template (and stops two concurrent creations of the same project). +/// +public class SyncHostedServiceCreationReservationTests +{ + private static SyncHostedService NewService() => + new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + + [Fact] + public void TryStartProjectCreation_blocks_a_second_concurrent_creation_and_is_reusable_after_release() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a creation is already in flight for this project"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + + svc.EndProjectCreation(projectId); + svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); + svc.TryStartProjectCreation(projectId).Should().BeTrue("the reservation is released and reusable"); + } + + [Fact] + public void QueueJob_is_refused_while_a_project_is_being_created() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.QueueJob(projectId).Should().BeFalse("a sync must not race a project that's still being created"); + + svc.EndProjectCreation(projectId); + svc.QueueJob(projectId).Should().BeTrue("syncing is allowed once creation has finished"); + } +} diff --git a/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs b/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs new file mode 100644 index 0000000000..fc5df4bb76 --- /dev/null +++ b/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs @@ -0,0 +1,43 @@ +using LexBoxApi.Controllers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.LexBoxApi; + +/// +/// Unit tests for ProjectController.CreateFromTemplate input validation. The 400 branches run before +/// any injected service is touched, so the controller can be built with null dependencies; only a +/// ProblemDetailsFactory (from AddControllers) is needed for Problem() to render. +/// +public class CreateFromTemplateValidationTests +{ + private static ProjectController NewController() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves + return new ProjectController(null!, null!, null!, null!, null!, null!, NullLogger.Instance) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() } + } + }; + } + + [Fact] + public async Task Rejects_when_no_vernacular_writing_system_is_supplied() + { + var result = await NewController().CreateFromTemplate("myproj", wsVernacular: []); + result.Result.Should().BeOfType().Which.StatusCode.Should().Be(StatusCodes.Status400BadRequest); + } + + [Fact] + public async Task Rejects_an_invalid_project_code() + { + var result = await NewController().CreateFromTemplate("Bad Code!", wsVernacular: ["fr"]); + result.Result.Should().BeOfType().Which.StatusCode.Should().Be(StatusCodes.Status400BadRequest); + } +} From db9dd95f77e75df812fd0a2adc39adb3a09329b6 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:38:27 +0700 Subject: [PATCH 07/60] Harden FwHeadless project creation after review - Close a TOCTOU race between TryStartProjectCreation and QueueJob (they checked each other's dictionary without a shared lock, so a sync could be queued for a project mid-creation and race on the repo). Both check-then-add paths now run under one lock. - AssertModelVersionMatches now fails closed: an unreadable version is treated as a mismatch rather than silently allowing the push. - Log a warning when a not-yet-implemented anthropologyCategories value is requested so the no-op stub isn't silent. - Route: null-safe writing-system checks and reject an empty analysis list with 400 (rather than a 500 from the service guard). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/Routes/ProjectRoutes.cs | 6 ++++- .../Services/ProjectCreationService.cs | 11 +++++--- .../FwHeadless/Services/SyncHostedService.cs | 27 +++++++++++++------ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index 90960430e1..541fcf2076 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -24,9 +24,13 @@ private static async Task> CreateProjectFromTempl ProjectCreationService projectCreationService, ILogger logger) { - if (input.WsVernacular.Count == 0) + // Null-safe against a malformed body (System.Text.Json leaves an absent list null). + if (input.WsVernacular is not { Count: > 0 }) return TypedResults.Problem("At least one vernacular writing system is required", statusCode: StatusCodes.Status400BadRequest); + if (input.WsAnalysis is not { Count: > 0 }) + return TypedResults.Problem("At least one analysis writing system is required", + statusCode: StatusCodes.Status400BadRequest); var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); if (invalidWs is not null) return TypedResults.Problem($"Invalid writing system code: {invalidWs}", diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index c6434fac28..57e78616c9 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -128,6 +128,9 @@ private void ApplyAnthropologyCategories(AnthropologyCategories anthropologyCate { // TODO: Implement — populate the project with the requested anthropology categories // (enhanced/standard). Currently a no-op; 'none' is the only fully-handled option. + if (anthropologyCategories != AnthropologyCategories.None) + logger.LogWarning("anthropologyCategories '{Categories}' was requested but is not implemented yet; " + + "the new project will have no anthropology categories", anthropologyCategories); } /// @@ -139,11 +142,13 @@ private void AssertModelVersionMatches(FwDataProject fwDataProject) { var expected = config.Value.FdoDataModelVersion; var actual = ReadModelVersion(fwDataProject.FilePath); - if (actual is not null && actual != expected) + // Fail closed: an unreadable version ('actual' == null) is treated as a mismatch, since this is + // the last safety net before an irreversible push in a data-loss-risk area. + if (actual != expected) { throw new InvalidOperationException( - $"New project's data-model version '{actual}' does not match FwHeadless FdoDataModelVersion " + - $"'{expected}'; refusing to push to avoid data corruption."); + $"New project's data-model version '{actual ?? "(unreadable)"}' does not match FwHeadless " + + $"FdoDataModelVersion '{expected}'; refusing to push to avoid data corruption."); } } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 3144b9d5db..d01b878593 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -24,6 +24,10 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsBeingCreated = new(); + // Serialises the check-then-add across the two dictionaries above, so a creation and a sync can't + // both slip through by interleaving their checks. + private readonly Lock _reservationLock = new(); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) @@ -68,8 +72,11 @@ public virtual bool IsJobQueuedOrRunning(Guid projectId) /// public bool TryStartProjectCreation(Guid projectId) { - if (_projectsQueuedOrRunning.ContainsKey(projectId)) return false; - return _projectsBeingCreated.TryAdd(projectId, 0); + lock (_reservationLock) + { + if (_projectsQueuedOrRunning.ContainsKey(projectId)) return false; + return _projectsBeingCreated.TryAdd(projectId, 0); + } } public void EndProjectCreation(Guid projectId) @@ -86,14 +93,18 @@ public void EndProjectCreation(Guid projectId) public bool QueueJob(Guid projectId) { - //don't sync a project while it's still being created from a template (would race on the repo) - if (_projectsBeingCreated.ContainsKey(projectId)) + bool addedToQueue; + lock (_reservationLock) { - logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); - return false; + //don't sync a project while it's still being created from a template (would race on the repo) + if (_projectsBeingCreated.ContainsKey(projectId)) + { + logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); + return false; + } + //will only queue job if it's not already queued + addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); } - //will only queue job if it's not already queued - var addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); if (addedToQueue) { if (!_projectsToSync.Writer.TryWrite(projectId)) From 543004fac0a6b6e031d2ef732e14c69933f17ff9 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:43:03 +0700 Subject: [PATCH 08/60] Address LexBox review feedback on create-from-template - Propagate FwHeadless's status code: an invalid writing-system tag now surfaces as 400 instead of being flattened to 500 (FwHeadlessClient returns the status + body). - Give the FwHeadless HTTP client a 5-minute timeout so inline creation (cold LCM load + push) doesn't hit the 100s default. - Route the failure compensation through a new ProjectService.CleanupFailedProjectCreation so repo+row deletion and cache invalidation stay in one place (the controller no longer deletes the row directly and skip cache invalidation). - Extract the analysis-default (["en"]) into a testable helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Controllers/ProjectController.cs | 28 ++++++++++--------- backend/LexBoxApi/LexBoxKernel.cs | 8 +++++- .../LexBoxApi/Services/FwHeadlessClient.cs | 11 ++++---- backend/LexBoxApi/Services/ProjectService.cs | 21 ++++++++++++++ 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index a7b3a196f1..731646843e 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Text.RegularExpressions; using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; @@ -57,8 +58,7 @@ public async Task> CreateFromTemplate( if (await projectService.ProjectExists(code)) return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); - // Default the analysis writing systems to English when none were supplied. - var wsAnalysisOrDefault = wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; + var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); var projectId = await projectService.CreateProject(new CreateProjectInput( Id: null, @@ -74,37 +74,39 @@ public async Task> CreateFromTemplate( // Saga: the project row + empty repo now exist. Have FwHeadless populate the repo with the // template .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs // on a fresh token so a client disconnect can't skip it. - string? error; + (HttpStatusCode statusCode, string? error) result; try { - error = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, anthropologyCategories, cancellationToken); + result = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, anthropologyCategories, cancellationToken); } catch { await CleanupFailedCreation(projectId, code); throw; } - if (error is not null) + if (result.error is not null) { await CleanupFailedCreation(projectId, code); - return Problem($"Failed to create the project from the template: {error}", statusCode: StatusCodes.Status500InternalServerError); + // Surface a bad request from FwHeadless (e.g. an invalid writing-system tag) as 400, not 500. + var statusCode = result.statusCode == HttpStatusCode.BadRequest + ? StatusCodes.Status400BadRequest + : StatusCodes.Status500InternalServerError; + return Problem($"Failed to create the project from the template: {result.error}", statusCode: statusCode); } await projectService.UpdateLastCommit(code); return projectId; } + /// Analysis writing systems to use, defaulting to English when none are supplied. + public static string[] AnalysisWritingSystemsOrDefault(string[]? wsAnalysis) => + wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; + private async Task CleanupFailedCreation(Guid projectId, string code) { try { - await hgService.DeleteRepoIfExists(code); - var project = await lexBoxDbContext.Projects.FindAsync(projectId); - if (project is not null) - { - lexBoxDbContext.Projects.Remove(project); - await lexBoxDbContext.SaveChangesAsync(); - } + await projectService.CleanupFailedProjectCreation(projectId, code); } catch (Exception ex) { diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 8171ca09c6..b8768932ab 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -52,7 +52,13 @@ public static void AddLexBoxApi(this IServiceCollection services, .ValidateOnStart(); services.AddHttpClient(); services.AddServiceDiscovery(); - services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless")) + services.AddHttpClient(client => + { + client.BaseAddress = new ("http://fwHeadless"); + // create-from-template runs inline (clone empty repo + build the template project + push), + // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom. + client.Timeout = TimeSpan.FromMinutes(5); + }) .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config services.AddHttpContextAccessor(); services.AddMemoryCache(); diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 45ab698894..9f263449c5 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -201,10 +201,11 @@ public async Task UnblockProject(Guid projectId) /// /// Populates a newly-created (empty) project's repo with a template .fwdata configured for the - /// given writing systems. Runs inline (a new empty project is small). Returns null on success, - /// or the error body on failure. + /// given writing systems. Runs inline (a new empty project is small). Returns a null Error on + /// success; on failure returns the FwHeadless status code and error body so the caller can + /// propagate the right status (e.g. surface a 400 rather than flattening it to 500). /// - public async Task CreateProjectFromTemplate(Guid projectId, + public async Task<(HttpStatusCode StatusCode, string? Error)> CreateProjectFromTemplate(Guid projectId, IReadOnlyList wsVernacular, IReadOnlyList wsAnalysis, AnthropologyCategories anthropologyCategories, @@ -214,14 +215,14 @@ public async Task UnblockProject(Guid projectId) var response = await httpClient.PostAsJsonAsync($"/api/project/create-from-template?projectId={projectId}", input, cancellationToken); - if (response.IsSuccessStatusCode) return null; + if (response.IsSuccessStatusCode) return (response.StatusCode, null); var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); logger.LogError("Failed to create project from template: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", response.StatusCode, response.ReasonPhrase, projectId, responseBody); - return responseBody; + return (response.StatusCode, responseBody); } } diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index dcab6b7b72..f563e8efa9 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -232,6 +232,27 @@ public async Task FinishReset(string code, Stream? zipFile = null) return project; } + /// + /// Compensating cleanup for a project whose template population failed after CreateProject: + /// removes the repo and the project row and invalidates the same caches as a permanent delete. + /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a + /// project this same request just created. FwHeadless cleans up its own local state on failure. + /// + public async Task CleanupFailedProjectCreation(Guid projectId, string code) + { + await hgService.DeleteRepoIfExists(code); + var project = await dbContext.Projects.FindAsync(projectId); + if (project is not null) + { + dbContext.Projects.Remove(project); + await dbContext.SaveChangesAsync(); + } + // Don't forget to add more Invalidate calls here if we add new caches + InvalidateProjectCodeCache(code); + InvalidateProjectConfidentialityCache(projectId); + InvalidateProjectOrgIdsCache(projectId); + } + public async ValueTask LookupProjectOrgIds(Guid projectId) { var cacheKey = $"ProjectOrgsForId:{projectId}"; From 24e1bc691accb50d27c1910bb1a4ab60bc7700ce Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 09:43:03 +0700 Subject: [PATCH 09/60] Strengthen create-from-template tests after audit - Assert the ProblemDetails.Detail so each validation test proves its own branch (the two 400s were previously indistinguishable). - Add the mirror race case: a queued sync blocks a creation. - Cover the analysis-default helper (null/empty -> ["en"], supplied kept). Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ncHostedServiceCreationReservationTests.cs | 10 +++++++ .../CreateFromTemplateValidationTests.cs | 28 +++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs index c395e1331b..1846cebb08 100644 --- a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -40,4 +40,14 @@ public void QueueJob_is_refused_while_a_project_is_being_created() svc.EndProjectCreation(projectId); svc.QueueJob(projectId).Should().BeTrue("syncing is allowed once creation has finished"); } + + [Fact] + public void A_queued_sync_blocks_a_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); + } } diff --git a/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs b/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs index fc5df4bb76..7d1424d6a8 100644 --- a/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs +++ b/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs @@ -31,13 +31,37 @@ private static ProjectController NewController() public async Task Rejects_when_no_vernacular_writing_system_is_supplied() { var result = await NewController().CreateFromTemplate("myproj", wsVernacular: []); - result.Result.Should().BeOfType().Which.StatusCode.Should().Be(StatusCodes.Status400BadRequest); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("vernacular"); } [Fact] public async Task Rejects_an_invalid_project_code() { var result = await NewController().CreateFromTemplate("Bad Code!", wsVernacular: ["fr"]); - result.Result.Should().BeOfType().Which.StatusCode.Should().Be(StatusCodes.Status400BadRequest); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project code"); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_null() + { + ProjectController.AnalysisWritingSystemsOrDefault(null).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_empty() + { + ProjectController.AnalysisWritingSystemsOrDefault([]).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_are_kept_when_supplied() + { + ProjectController.AnalysisWritingSystemsOrDefault(["fr", "es"]).Should().Equal("fr", "es"); } } From 5415444edc3c50c8d1954be6ec210a6f25116d7f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 14:07:11 +0700 Subject: [PATCH 10/60] Integration-test create-from-template first push MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end test (Category=Integration, CI-only — needs the lexbox stack): calls the admin createFromTemplate endpoint, then asserts the empty repo received the first commit (GetProjectLastCommit non-null; hgweb tip is no longer the all-zero hash) and that the pushed fw.fwdata carries the requested vernacular + analysis writing systems. Also asserts a missing vernacular is rejected with 400. Cleans up via softDeleteProject. This exercises the one path that can't be unit-tested: LfMergeBridge/Chorus pushing a locally-committed brand-new project into an empty remote repo. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CreateProjectFromTemplateTests.cs | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs diff --git a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs new file mode 100644 index 0000000000..42cfcdbcb1 --- /dev/null +++ b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs @@ -0,0 +1,104 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using FluentAssertions; +using Testing.ApiTests; +using Testing.Fixtures; +using Xunit.Abstractions; + +namespace Testing.SyncReverseProxy; + +/// +/// End-to-end coverage for the admin "create project from a template" API. This is the one path that +/// can't be unit-tested: LexBox creates an empty hg repo and FwHeadless does the first push of a +/// template-built .fwdata into it (via LfMergeBridge/Chorus against the real hgweb). Requires the +/// lexbox stack, so it only runs in CI. +/// +[Trait("Category", "Integration")] +public class CreateProjectFromTemplateTests : IClassFixture +{ + private readonly ITestOutputHelper _output; + private readonly ApiTestBase _adminApiTester; + + public CreateProjectFromTemplateTests(ITestOutputHelper output, IntegrationFixture fixture) + { + _output = output; + _adminApiTester = fixture.AdminApiTester; + } + + [Fact] + public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritingSystems() + { + // Valid code (lowercase/digits/hyphen, doesn't start with a hyphen), unique per run. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var vernacular = new[] { "fr", "es" }; + var analysis = new[] { "de", "pt" }; + var query = $"?code={code}" + + string.Concat(vernacular.Select(ws => $"&wsVernacular={ws}")) + + string.Concat(analysis.Select(ws => $"&wsAnalysis={ws}")); + + Guid projectId = default; + try + { + // 1. Create the project via the admin endpoint (creates the DB row + empty repo, then has + // FwHeadless build the template .fwdata and push it into the empty repo). + var response = await _adminApiTester.HttpClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/createFromTemplate{query}", null); + response.StatusCode.Should().Be(HttpStatusCode.OK, + "creation should succeed; body: {0}", await response.Content.ReadAsStringAsync()); + projectId = await response.Content.ReadFromJsonAsync(); + projectId.Should().NotBe(Guid.Empty); + + // 2. The first push landed a commit server-side (the empty repo is no longer empty). + var lastCommit = await _adminApiTester.GetProjectLastCommit(code); + lastCommit.Should().NotBeNull("the template project should have been pushed to the repo"); + + var tagsResponse = await _adminApiTester.HttpClient.GetAsync($"/hg/{code}/tags?style=json"); + tagsResponse.EnsureSuccessStatusCode(); + var tip = (await tagsResponse.Content.ReadFromJsonAsync())?["node"]?.ToString(); + tip.Should().NotBeNullOrEmpty(); + tip!.Replace("0", "").Should().NotBeEmpty("the repo tip should not be the all-zero empty-repo hash"); + + // 3. The pushed .fwdata carries the requested writing systems. FwHeadless always names the + // file fw.fwdata (its fixed project name), so fetch that from hgweb and check the codes. + var fwDataResponse = await _adminApiTester.HttpClient.GetAsync($"/hg/{code}/raw-file/tip/fw.fwdata"); + fwDataResponse.EnsureSuccessStatusCode(); + var fwData = await fwDataResponse.Content.ReadAsStringAsync(); + fwData.Should().NotBeEmpty(); + foreach (var ws in vernacular.Concat(analysis)) + fwData.Should().Contain($"\"{ws}\"", "writing system {0} should be present in the pushed project", ws); + } + finally + { + if (projectId != default) await SoftDeleteProject(projectId); + } + } + + [Fact] + public async Task CreateFromTemplate_RejectsWhenNoVernacularWritingSystem() + { + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await _adminApiTester.HttpClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/createFromTemplate?code={code}&wsAnalysis=en", null); + response.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } + + private async Task SoftDeleteProject(Guid projectId) + { + try + { + await _adminApiTester.ExecuteGql($$""" + mutation { + softDeleteProject(input: { projectId: "{{projectId}}" }) { + project { id } + errors { __typename } + } + } + """); + } + catch (Exception ex) + { + _output.WriteLine($"[CreateProjectFromTemplateTests] Ignored cleanup exception: {ex}"); + } + } +} From b3f723f6b31d69c3ac9c000408008eae4bd997c8 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 14:36:40 +0700 Subject: [PATCH 11/60] Ship LCM project templates with FwHeadless MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProjectCreationService needs NewLangProj.fwdata (and its sibling template files) to build a new project, but FwDataBridgeConfig.TemplatesFolder defaults to the FieldWorks user-data dir (~/.local/share/fieldworks/Templates), which doesn't exist in the container — creation failed with DirectoryNotFoundException on NewLangProj.fwdata. Copy the SIL.LCModel package's contentFiles/Templates/* into FwHeadless's output and point TemplatesFolder at that shipped folder (AppContext.BaseDirectory/Templates) unless overridden in config. Mirrors how FwLiteWeb and the bridge tests ship the templates. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/FwHeadless.csproj | 8 ++++++++ backend/FwHeadless/FwHeadlessKernel.cs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/backend/FwHeadless/FwHeadless.csproj b/backend/FwHeadless/FwHeadless.csproj index 9cc1bebd7e..fdd1a3cd6e 100644 --- a/backend/FwHeadless/FwHeadless.csproj +++ b/backend/FwHeadless/FwHeadless.csproj @@ -19,6 +19,9 @@ ~108MB of duplicate Mercurial. (build assets still import, so CopyFiles still runs.) --> + + @@ -43,6 +46,11 @@ + + diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index 0d62de9a48..769d00c92d 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -33,6 +33,14 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) .AddLcmCrdtClientCore() .AddFwDataBridge(ServiceLifetime.Scoped) .AddFwLiteProjectSync(); + // The LCM project templates (NewLangProj.fwdata etc.) ship in our output (see FwHeadless.csproj). + // FwDataBridgeConfig's default TemplatesFolder points at the FieldWorks user-data dir, which + // doesn't exist in the container; point it at the shipped templates unless overridden in config. + services.Configure(config => + { + if (config.TemplatesFolder == new FwDataBridgeConfig().TemplatesFolder) + config.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + }); services.RemoveAll(typeof(IMediaAdapter)); services.AddScoped(); services.AddScoped(); From 6baffacf46189fbb2a14ae00c76c6426d5b67890 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 15:20:21 +0700 Subject: [PATCH 12/60] Build first commit from scratch on the model-version branch Creating a project from a template pushes into a repo LexBox has only `hg init`'d server-side, so it has zero changesets. Cloning that empty remote is a no-op that returns Success=true but leaves no local `.hg`, so the subsequent `hg add` failed with "no repository found ... (.hg not found)". Replace the clone with LfMerge's from-scratch genesis recipe (verified against the lfmerge reference implementation): `hg init` the local fw/ folder, set the branch to the FDO model version BEFORE the first commit, build the .fwdata, then commit + push. Chorus's Language_Forge_Send_Receive refuses to make the first-ever commit itself, so the manual CommitFile stays. The branch step is the data-correctness fix: FLEx clients clone looking for a branch named after the model version, so a first commit landing on `default` would be invisible to them. Numeric branch names work via the fixutf8 hg extension already wired into Mercurial/mercurial.ini. Adds InitRepo and SetBranch primitives to SendReceiveHelpers / ISendReceiveService. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/ISendReceiveService.cs | 2 ++ .../Services/ProjectCreationService.cs | 19 ++++++++------- .../FwHeadless/Services/SendReceiveHelpers.cs | 23 +++++++++++++++++++ .../FwHeadless/Services/SendReceiveService.cs | 10 ++++++++ 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs index 4755ff1ac8..ffb975413c 100644 --- a/backend/FwHeadless/Services/ISendReceiveService.cs +++ b/backend/FwHeadless/Services/ISendReceiveService.cs @@ -10,4 +10,6 @@ public interface ISendReceiveService Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode); Task PendingCommitCountBothWays(FwDataProject project, string? projectCode); Task CommitFile(string filePath, string commitMessage); + Task InitRepo(string folder); + Task SetBranch(string folder, string branchName); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 57e78616c9..dc8e90d0f8 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -41,14 +41,17 @@ public async Task CreateFromTemplate( var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); try { - // Populate the empty repo using only already-exercised FwHeadless primitives: - // 1. Clone the empty remote -> local fw/ folder (.hg, no fwdata yet). - // 2. Build fw.fwdata into that folder from the SIL.LCModel template. - // 3. Add the remaining writing systems, saving and releasing the LCM file locks. - // 4. hg add + commit the (untracked) fwdata, then push. - var cloneResult = await srService.Clone(fwDataProject, projectCode); - if (!cloneResult.Success) - throw new SendReceiveException("Clone of the new project's repo failed", cloneResult); + // Build the first commit of a brand-new FLEx repo from scratch, mirroring LfMerge's genesis + // recipe (hg init -> branch -> write files -> commit) since Clone can't establish an empty + // remote and Language_Forge_Send_Receive refuses to make the *first* commit itself: + // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). + // 2. Set the branch to the FDO model version BEFORE any commit; FLEx clients look for the + // data on that branch, so a first commit on 'default' would be invisible to them. + // 3. Build fw.fwdata into that folder from the SIL.LCModel template. + // 4. Add the remaining writing systems, saving and releasing the LCM file locks. + // 5. hg add + commit the (untracked) fwdata onto the model-version branch, then push. + await srService.InitRepo(fwDataProject.ProjectFolder); + await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.FdoDataModelVersion); BuildFromTemplate(fwDataProject, vernacularWritingSystems[0], analysisWritingSystems[0]); await AddWritingSystems(fwDataProject, vernacularWritingSystems, analysisWritingSystems); diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 4caebce379..6701744eb8 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -160,6 +160,29 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", fileDir, progress); } + public static async Task InitRepo(string folder, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + progress ??= new NullProgress(); + Directory.CreateDirectory(folder); + // Use Chorus rather than a bare `hg init`: CreateOrUseExisting also wires up the custom hg + // extensions FwHeadless relies on (e.g. fixutf8) so later send/receive works correctly. + await Task.Run(() => HgRepository.CreateOrUseExisting(folder, progress)); + } + + public static async Task SetBranch(string folder, string branchName, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.branch", branchName); + progress ??= new NullProgress(); + // FLEx repos keep their data on a branch named after the FDO model version; a clone looking for + // that branch reports "no such branch" if the initial commit landed on 'default' instead. hg + // records the branch for the *next* commit, so this must run before the first commit. Numeric + // branch names are accepted thanks to the fixutf8 extension wired into Mercurial/mercurial.ini. + await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress); + } + private static string EscapeShellArg(string arg) { var quote = """ diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs index 735e90b8ae..9ab17bd8f9 100644 --- a/backend/FwHeadless/Services/SendReceiveService.cs +++ b/backend/FwHeadless/Services/SendReceiveService.cs @@ -70,4 +70,14 @@ public async Task CommitFile(string filePath, string commitMessage) { await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress); } + + public async Task InitRepo(string folder) + { + await SendReceiveHelpers.InitRepo(folder, progress); + } + + public async Task SetBranch(string folder, string branchName) + { + await SendReceiveHelpers.SetBranch(folder, branchName, progress); + } } From 2374ea0cdf04797badf3be35d297fb073447c968 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 14 Jul 2026 16:41:32 +0700 Subject: [PATCH 13/60] Fix integration test URL --- .../SyncReverseProxy/CreateProjectFromTemplateTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs index 42cfcdbcb1..0bac5b784d 100644 --- a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs +++ b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs @@ -53,7 +53,7 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin var lastCommit = await _adminApiTester.GetProjectLastCommit(code); lastCommit.Should().NotBeNull("the template project should have been pushed to the repo"); - var tagsResponse = await _adminApiTester.HttpClient.GetAsync($"/hg/{code}/tags?style=json"); + var tagsResponse = await _adminApiTester.HttpClient.GetAsync($"{_adminApiTester.BaseUrl}/hg/{code}/tags?style=json"); tagsResponse.EnsureSuccessStatusCode(); var tip = (await tagsResponse.Content.ReadFromJsonAsync())?["node"]?.ToString(); tip.Should().NotBeNullOrEmpty(); @@ -61,7 +61,7 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin // 3. The pushed .fwdata carries the requested writing systems. FwHeadless always names the // file fw.fwdata (its fixed project name), so fetch that from hgweb and check the codes. - var fwDataResponse = await _adminApiTester.HttpClient.GetAsync($"/hg/{code}/raw-file/tip/fw.fwdata"); + var fwDataResponse = await _adminApiTester.HttpClient.GetAsync($"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/fw.fwdata"); fwDataResponse.EnsureSuccessStatusCode(); var fwData = await fwDataResponse.Content.ReadAsStringAsync(); fwData.Should().NotBeEmpty(); From 14fc50ab32b69b285936835baadadee74e11519c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 15 Jul 2026 14:17:38 +0700 Subject: [PATCH 14/60] Add wsUi parameter to create-from-template The template project can be built with an explicit UI writing system. Thread a new `wsUi` query parameter (default "en") from the admin endpoint through FwHeadless to ProjectLoader.NewProject, which passes it to LcmCache.CreateNewLangProj as the trailing user-interface writing-system string (after the analysis/vernacular definitions). Like wsVernacular/wsAnalysis it must be a valid writing-system tag; that check lives in ProjectRoutes.CreateProjectFromTemplate alongside the existing ones. NewProject defaults uiWs to "en" so existing callers are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/Routes/ProjectRoutes.cs | 5 ++++- .../FwHeadless/Services/ProjectCreationService.cs | 13 ++++++++----- .../Fixtures/MockFwProjectLoader.cs | 4 ++-- .../FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs | 14 +++++++++----- backend/LexBoxApi/Controllers/ProjectController.cs | 3 ++- backend/LexBoxApi/Services/FwHeadlessClient.cs | 3 ++- .../Entities/CreateProjectFromTemplateInput.cs | 5 +++-- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index 541fcf2076..073d71082f 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -35,6 +35,9 @@ private static async Task> CreateProjectFromTempl if (invalidWs is not null) return TypedResults.Problem($"Invalid writing system code: {invalidWs}", statusCode: StatusCodes.Status400BadRequest); + if (string.IsNullOrEmpty(input.WsUi) || !IetfLanguageTag.IsValid(input.WsUi)) + return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", + statusCode: StatusCodes.Status400BadRequest); var projectCode = await projectLookupService.GetProjectCode(projectId); if (projectCode is null) @@ -44,7 +47,7 @@ private static async Task> CreateProjectFromTempl } await projectCreationService.CreateFromTemplate( - projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.AnthropologyCategories); + projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi, input.AnthropologyCategories); return TypedResults.Ok(); } } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index dc8e90d0f8..bd671b4ba0 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -28,12 +28,15 @@ public async Task CreateFromTemplate( string projectCode, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, + string uiWritingSystem, AnthropologyCategories anthropologyCategories) { if (vernacularWritingSystems.Count == 0) throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); if (analysisWritingSystems.Count == 0) throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); + if (string.IsNullOrEmpty(uiWritingSystem)) + throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem)); // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. if (!syncHostedService.TryStartProjectCreation(projectId)) @@ -53,7 +56,7 @@ public async Task CreateFromTemplate( await srService.InitRepo(fwDataProject.ProjectFolder); await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.FdoDataModelVersion); - BuildFromTemplate(fwDataProject, vernacularWritingSystems[0], analysisWritingSystems[0]); + BuildFromTemplate(fwDataProject, vernacularWritingSystems[0], analysisWritingSystems[0], uiWritingSystem); await AddWritingSystems(fwDataProject, vernacularWritingSystems, analysisWritingSystems); ApplyAnthropologyCategories(anthropologyCategories); AssertModelVersionMatches(fwDataProject); @@ -76,12 +79,12 @@ public async Task CreateFromTemplate( } } - private void BuildFromTemplate(FwDataProject fwDataProject, string vernacularWs, string analysisWs) + private void BuildFromTemplate(FwDataProject fwDataProject, string vernacularWs, string analysisWs, string uiWs) { - // NewProject copies the SIL.LCModel template (with the first vernacular + analysis WS) and - // returns a loaded cache; dispose it right away so its file locks are released before we + // NewProject copies the SIL.LCModel template (with the first vernacular + analysis WS and the UI + // WS) and returns a loaded cache; dispose it right away so its file locks are released before we // re-open the project through FwDataFactory to add the remaining writing systems. - using var cache = projectLoader.NewProject(fwDataProject, analysisWs, vernacularWs); + using var cache = projectLoader.NewProject(fwDataProject, analysisWs, vernacularWs, uiWs); } private async Task AddWritingSystems( diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs index 890bca1624..4123cdd39f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs @@ -19,7 +19,7 @@ public override LcmCache LoadCache(FwDataProject project) return cache; } - public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs) + public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -28,7 +28,7 @@ public override LcmCache NewProject(FwDataProject project, string analysisWs, st new SimpleProjectId(BackendProviderType.kMemoryOnly, Path.GetFullPath(project.FilePath)), analysisWs, vernacularWs, - null, + uiWs, new LfLcmUi(progress.SynchronizeInvoke), lcmDirectories, new LcmSettings()); diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index 3d64f8016e..4dc6d1a4e5 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -19,7 +19,7 @@ public interface IProjectLoader /// LcmCache LoadCache(FwDataProject project); - LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs); + LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en"); } public class ProjectLoader(IOptions config) : IProjectLoader @@ -75,7 +75,7 @@ public virtual LcmCache LoadCache(FwDataProject project) return cache; } - public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs) + public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -85,7 +85,8 @@ public virtual LcmCache NewProject(FwDataProject project, string analysisWs, str lcmDirectories, progress.SynchronizeInvoke, new CoreWritingSystemDefinition(analysisWs) { Id = analysisWs }, - new CoreWritingSystemDefinition(vernacularWs) { Id = vernacularWs }); + new CoreWritingSystemDefinition(vernacularWs) { Id = vernacularWs }, + uiWs); return LoadCache(project); } @@ -94,8 +95,11 @@ private static void NewProject(IThreadedProgress progress, ILcmDirectories lcmDirectories, ISynchronizeInvoke syncInvoke, CoreWritingSystemDefinition analysisWs, - CoreWritingSystemDefinition vernacularWs) + CoreWritingSystemDefinition vernacularWs, + string uiWs) { - LcmCache.CreateNewLangProj(progress, [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs]); + // uiWs is the user-interface writing system; CreateNewLangProj takes it as a plain string ICU + // locale after the analysis/vernacular definitions. + LcmCache.CreateNewLangProj(progress, [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs, uiWs]); } } diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 731646843e..7ba4f699dc 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -47,6 +47,7 @@ public async Task> CreateFromTemplate( string code, [FromQuery] string[] wsVernacular, [FromQuery] string[]? wsAnalysis = null, + string wsUi = "en", string? name = null, AnthropologyCategories anthropologyCategories = AnthropologyCategories.None, CancellationToken cancellationToken = default) @@ -77,7 +78,7 @@ public async Task> CreateFromTemplate( (HttpStatusCode statusCode, string? error) result; try { - result = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, anthropologyCategories, cancellationToken); + result = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, anthropologyCategories, cancellationToken); } catch { diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 9f263449c5..0a3f01f8f1 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -208,10 +208,11 @@ public async Task UnblockProject(Guid projectId) public async Task<(HttpStatusCode StatusCode, string? Error)> CreateProjectFromTemplate(Guid projectId, IReadOnlyList wsVernacular, IReadOnlyList wsAnalysis, + string wsUi, AnthropologyCategories anthropologyCategories, CancellationToken cancellationToken = default) { - var input = new CreateProjectFromTemplateInput(wsVernacular, wsAnalysis, anthropologyCategories); + var input = new CreateProjectFromTemplateInput(wsVernacular, wsAnalysis, wsUi, anthropologyCategories); var response = await httpClient.PostAsJsonAsync($"/api/project/create-from-template?projectId={projectId}", input, cancellationToken); diff --git a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs b/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs index adc9b10069..24fcbccf79 100644 --- a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs +++ b/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs @@ -2,10 +2,11 @@ namespace LexCore.Entities; /// /// Body of the internal FwHeadless "create project from template" call. The vernacular/analysis -/// lists are already validated and defaulted by LexBoxApi (at least one vernacular; analysis -/// defaults to "en"). +/// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one +/// vernacular; analysis and UI default to "en"). /// public record CreateProjectFromTemplateInput( IReadOnlyList WsVernacular, IReadOnlyList WsAnalysis, + string WsUi, AnthropologyCategories AnthropologyCategories); From 7d00f06ed995974685e7917d5408eb348e7c25e5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 15 Jul 2026 14:57:53 +0700 Subject: [PATCH 15/60] Add all writing systems via CreateNewLangProj MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the template was built with only the first vernacular + analysis writing system, then FwDataFactory was reopened to add the rest one by one. LcmCache.CreateNewLangProj accepts two more parameters after the uiWs string — a HashSet of additional analysis writing systems and one of additional vernacular writing systems — so let it create the whole set in one pass instead. ProjectLoader gains a list-based NewProject overload that builds those sets by turning each list into a hash set and removing the default (the first item, which is passed separately). ProjectCreationService.BuildFromTemplate now takes the two lists and no longer needs FwDataFactory or the AddWritingSystems post-step. Adds a test creating a project with two analysis and two vernacular writing systems and asserting each lands on the correct list (also guards the analysis-then-vernacular argument order into CreateNewLangProj). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/ProjectCreationService.cs | 65 ++++--------------- .../LcmUtils/ProjectLoader.cs | 46 +++++++++++-- .../ProjectTemplateTests.cs | 20 ++++++ 3 files changed, 70 insertions(+), 61 deletions(-) diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index bd671b4ba0..5fab68b897 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -1,11 +1,9 @@ using System.Xml; using FwDataMiniLcmBridge; -using FwDataMiniLcmBridge.Api; using FwDataMiniLcmBridge.LcmUtils; using LexCore.Entities; using LexCore.Exceptions; using Microsoft.Extensions.Options; -using MiniLcm.Models; namespace FwHeadless.Services; @@ -16,7 +14,6 @@ namespace FwHeadless.Services; public class ProjectCreationService( IOptions config, ISendReceiveService srService, - FwDataFactory fwDataFactory, IProjectLoader projectLoader, SyncHostedService syncHostedService, ILogger logger) @@ -50,14 +47,13 @@ public async Task CreateFromTemplate( // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). // 2. Set the branch to the FDO model version BEFORE any commit; FLEx clients look for the // data on that branch, so a first commit on 'default' would be invisible to them. - // 3. Build fw.fwdata into that folder from the SIL.LCModel template. - // 4. Add the remaining writing systems, saving and releasing the LCM file locks. - // 5. hg add + commit the (untracked) fwdata onto the model-version branch, then push. + // 3. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all + // of the requested writing systems. + // 4. hg add + commit the (untracked) fwdata onto the model-version branch, then push. await srService.InitRepo(fwDataProject.ProjectFolder); await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.FdoDataModelVersion); - BuildFromTemplate(fwDataProject, vernacularWritingSystems[0], analysisWritingSystems[0], uiWritingSystem); - await AddWritingSystems(fwDataProject, vernacularWritingSystems, analysisWritingSystems); + BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); ApplyAnthropologyCategories(anthropologyCategories); AssertModelVersionMatches(fwDataProject); @@ -79,55 +75,16 @@ public async Task CreateFromTemplate( } } - private void BuildFromTemplate(FwDataProject fwDataProject, string vernacularWs, string analysisWs, string uiWs) - { - // NewProject copies the SIL.LCModel template (with the first vernacular + analysis WS and the UI - // WS) and returns a loaded cache; dispose it right away so its file locks are released before we - // re-open the project through FwDataFactory to add the remaining writing systems. - using var cache = projectLoader.NewProject(fwDataProject, analysisWs, vernacularWs, uiWs); - } - - private async Task AddWritingSystems( + private void BuildFromTemplate( FwDataProject fwDataProject, IReadOnlyList vernacularWritingSystems, - IReadOnlyList analysisWritingSystems) - { - // The template already has the first vernacular + first analysis WS; add the rest, deduping - // against what's present. saveOnDispose flushes the additions to the .fwdata XML when the api - // is disposed; CloseProjectAsync then disposes the underlying LcmCache so the following hg - // add/commit doesn't hit a file lock. - try - { - using (var api = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, saveOnDispose: true)) - { - var existing = await api.GetWritingSystems(); - var vernacularPresent = existing.Vernacular.Select(ws => ws.WsId).ToHashSet(); - var analysisPresent = existing.Analysis.Select(ws => ws.WsId).ToHashSet(); - foreach (var code in vernacularWritingSystems) - await AddWritingSystem(api, code, WritingSystemType.Vernacular, vernacularPresent); - foreach (var code in analysisWritingSystems) - await AddWritingSystem(api, code, WritingSystemType.Analysis, analysisPresent); - } - } - finally - { - await fwDataFactory.CloseProjectAsync(fwDataProject); - } - } - - private static async Task AddWritingSystem(FwDataMiniLcmApi api, string code, WritingSystemType type, HashSet present) + IReadOnlyList analysisWritingSystems, + string uiWs) { - var wsId = new WritingSystemId(code); - if (!present.Add(wsId)) return; // already present (the template's first WS, or a repeated request) - await api.CreateWritingSystem(new WritingSystem - { - Id = Guid.Empty, - WsId = wsId, - Name = code, - Abbreviation = code, - Font = "Charis SIL", - Type = type, - }); + // NewProject copies the SIL.LCModel template, configured with all of the requested writing + // systems (the first of each list is the default of that type), and returns a loaded cache; + // dispose it right away so its file locks are released before the following hg add/commit. + using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs); } private void ApplyAnthropologyCategories(AnthropologyCategories anthropologyCategories) diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index 4dc6d1a4e5..e9a2f6f52a 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -20,6 +20,12 @@ public interface IProjectLoader LcmCache LoadCache(FwDataProject project); LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en"); + + /// + /// Like the single-WS overload, but the new project also gets every analysis/vernacular writing + /// system in the given lists. The first item of each list is the default of that type. + /// + LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, string uiWs = "en"); } public class ProjectLoader(IOptions config) : IProjectLoader @@ -76,30 +82,56 @@ public virtual LcmCache LoadCache(FwDataProject project) } public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") + { + return NewProject(project, [analysisWs], [vernacularWs], uiWs); + } + + public virtual LcmCache NewProject(FwDataProject project, + IReadOnlyList analysisWss, + IReadOnlyList vernacularWss, + string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); var progress = new LcmThreadedProgress(); + var analysisDefinitions = analysisWss.Select(CreateWritingSystemDefinition).ToList(); + var vernacularDefinitions = vernacularWss.Select(CreateWritingSystemDefinition).ToList(); NewProject(progress, project.Name, lcmDirectories, progress.SynchronizeInvoke, - new CoreWritingSystemDefinition(analysisWs) { Id = analysisWs }, - new CoreWritingSystemDefinition(vernacularWs) { Id = vernacularWs }, - uiWs); + analysisDefinitions[0], + vernacularDefinitions[0], + uiWs, + AdditionalWritingSystems(analysisDefinitions), + AdditionalWritingSystems(vernacularDefinitions)); return LoadCache(project); } + private static CoreWritingSystemDefinition CreateWritingSystemDefinition(string ws) => new(ws) { Id = ws }; + + // The default writing system (the first of the list) is passed to CreateNewLangProj on its own, so + // the "additional" set is the whole list as a hash set with that default removed. + private static HashSet AdditionalWritingSystems(List definitions) + { + var additional = definitions.ToHashSet(); + additional.Remove(definitions[0]); + return additional; + } + private static void NewProject(IThreadedProgress progress, string projectName, ILcmDirectories lcmDirectories, ISynchronizeInvoke syncInvoke, CoreWritingSystemDefinition analysisWs, CoreWritingSystemDefinition vernacularWs, - string uiWs) + string uiWs, + HashSet additionalAnalysisWss, + HashSet additionalVernacularWss) { - // uiWs is the user-interface writing system; CreateNewLangProj takes it as a plain string ICU - // locale after the analysis/vernacular definitions. - LcmCache.CreateNewLangProj(progress, [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs, uiWs]); + // After the default analysis/vernacular definitions and the string UI writing system, + // CreateNewLangProj takes the additional analysis then additional vernacular writing systems. + LcmCache.CreateNewLangProj(progress, + [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs, uiWs, additionalAnalysisWss, additionalVernacularWss]); } } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs index b3e298cefd..6469b12e33 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs @@ -110,6 +110,26 @@ public async Task CreateFromTemplateMatchesTemplateFilePlusRequestedVernacular() actual.Should().BeEquivalentTo(expected); } + [Fact] + public async Task NewProjectAddsEveryRequestedWritingSystemToTheCorrectList() + { + var fwDataBridgeConfig = Services.GetRequiredService>().Value; + // The SIL.LCModel template ships next to the test assembly; point liblcm at it (this project, + // unlike FwDataMiniLcmBridge.Tests, doesn't otherwise configure TemplatesFolder). + fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + var fwDataProject = new FwDataProject("multi-ws-source", fwDataBridgeConfig.ProjectsFolder); + // First of each list is the default; the rest go through CreateNewLangProj's additional-WS sets. + // Distinct codes per type so a swapped analysis/vernacular argument would fail this test. + string[] analysis = ["en", "de"]; + string[] vernacular = ["fr", "es"]; + Services.GetRequiredService().NewProject(fwDataProject, analysis, vernacular, uiWs: "en").Dispose(); + + using var api = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); + var writingSystems = await api.GetWritingSystems(); + writingSystems.Analysis.Select(ws => ws.WsId).Should().Contain(["en", "de"]); + writingSystems.Vernacular.Select(ws => ws.WsId).Should().Contain(["fr", "es"]); + } + private static ProjectSnapshot DeserializeTemplate() { using var stream = File.OpenRead(TemplatePath); From a31d137ddfdd7cebc259d8117fc40a3ff0c5a2a2 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 11:54:43 +0700 Subject: [PATCH 16/60] Pin FwHeadless CWD so Chorus finds its FieldWorks plugin Chorus discovers file-type handlers via DirectoryCatalog(".", "*-ChorusPlugin.dll"), which scans the current working directory. WebApplication.CreateBuilder does not set CWD, so when FwHeadless launches from a dir other than the one holding the co-published LibFLExBridge-ChorusPlugin.dll (e.g. `dotnet run`), that plugin is never loaded. Without it, FieldWorks extensions like `.list` fall back to Chorus's 1 MB LargeFileFilter default, so the >1 MB SemanticDomainList.list is silently filtered out of a new project's first push, leaving projects with no semantic domains. Set Environment.CurrentDirectory to AppContext.BaseDirectory before building the host so the plugin is always discovered (which gives FieldWorks files their intended unlimited size and restores the full merge/diff handler set). Safe here: hg runs with explicit working dirs and TemplatesFolder already resolves against AppContext.BaseDirectory. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/Program.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index ed832e53e9..cdc459031b 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -12,6 +12,17 @@ using WebServiceDefaults; using AppVersion = LexCore.AppVersion; +// Chorus discovers its file-type handler plugins with `new DirectoryCatalog(".", "*-ChorusPlugin.dll")`, +// which scans the *current working directory* -- not the app base dir -- and WebApplication.CreateBuilder +// does not change CWD. When launched from anywhere other than the folder holding the co-published +// LibFLExBridge-ChorusPlugin.dll (e.g. `dotnet run`, whose CWD is the project dir), that plugin isn't +// found. Without it, FieldWorks extensions like `.list` fall back to Chorus's 1 MB LargeFileFilter +// default, so the >1 MB SemanticDomainList.list is silently filtered out of a new project's first push +// (and later merges lose their FieldWorks handlers too). Pin CWD to the base dir so the plugin is always +// discovered. Safe here: hg runs with explicit working dirs and TemplatesFolder resolves against +// AppContext.BaseDirectory already. +Environment.CurrentDirectory = AppContext.BaseDirectory; + var builder = WebApplication.CreateBuilder(args); // Add services to the container. From 1977e22fd826ce7d9bde09c7303afef40e55a25c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 12:27:30 +0700 Subject: [PATCH 17/60] Create the S/R branch the modern way (FlexBridgeDataVersion.modelVersion) New project creation set the genesis Mercurial branch to the bare FDO model version (7000072), which LfMergeBridge only accepts via a backwards-compatibility fallback. The modern convention is to name the branch by joining the FlexBridgeDataVersion, a dot, and the model version (7500002.7000072) -- exactly what LfMergeBridge derives from the model version via FlexUpdateBranchHelperStrategy.GetBranchNameFromModelVersion. Add FlexBridgeDataVersion (default 7500002, mirroring FLExBridge's internal constant we can't reference) and a SendReceiveBranchName config property, and set the genesis branch from it so the first commit lands on the branch send/receive expects with no fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/FwHeadlessConfig.cs | 16 ++++++++++++++++ .../Services/ProjectCreationService.cs | 9 +++++---- .../FwHeadless/Services/SendReceiveHelpers.cs | 9 +++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/FwHeadless/FwHeadlessConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs index 95eb49f3ae..836d732b1f 100644 --- a/backend/FwHeadless/FwHeadlessConfig.cs +++ b/backend/FwHeadless/FwHeadlessConfig.cs @@ -22,6 +22,22 @@ public class FwHeadlessConfig public long MaxUploadFileSizeBytes => MaxUploadFileSizeKb * 1024; public string FdoDataModelVersion { get; init; } = "7000072"; + /// + /// The FLExBridge data version. FieldWorks/LfMergeBridge form the Mercurial branch a project's + /// data lives on by joining this and the FDO model version with a dot (see ). + /// Mirrors FlexBridgeConstants.FlexBridgeDataVersion in FLExBridge, which is internal so we can't + /// reference it directly; bump this if FLExBridge bumps its data version. + /// + public string FlexBridgeDataVersion { get; init; } = "7500002"; + + /// + /// The Mercurial branch FieldWorks/LfMergeBridge keep a project's data on: FlexBridgeDataVersion, + /// a dot, then the FDO model version (e.g. "7500002.7000072"). This is exactly the branch name + /// LfMergeBridge derives from , so a new project's genesis commit + /// must land on this branch for send/receive to find the data. + /// + public string SendReceiveBranchName => $"{FlexBridgeDataVersion}.{FdoDataModelVersion}"; + /// /// Project directory structure in FwHeadless: (Note that FwDataProject.ProjectsPath is the root of a SINGLE project) /// {ProjectStorageRoot}/ diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 5fab68b897..e1245c3086 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -45,13 +45,14 @@ public async Task CreateFromTemplate( // recipe (hg init -> branch -> write files -> commit) since Clone can't establish an empty // remote and Language_Forge_Send_Receive refuses to make the *first* commit itself: // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). - // 2. Set the branch to the FDO model version BEFORE any commit; FLEx clients look for the - // data on that branch, so a first commit on 'default' would be invisible to them. + // 2. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, + // e.g. 7500002.7000072) BEFORE any commit; FLEx/LfMergeBridge look for the data on that + // branch, so a first commit on 'default' would be invisible to them. // 3. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all // of the requested writing systems. - // 4. hg add + commit the (untracked) fwdata onto the model-version branch, then push. + // 4. hg add + commit the (untracked) fwdata onto that branch, then push. await srService.InitRepo(fwDataProject.ProjectFolder); - await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.FdoDataModelVersion); + await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); ApplyAnthropologyCategories(anthropologyCategories); diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 6701744eb8..902de71411 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -176,10 +176,11 @@ public static async Task SetBranch(string folder, string branchName, IProgress? using var activity = FwHeadlessActivitySource.Value.StartActivity(); activity?.SetTag("app.branch", branchName); progress ??= new NullProgress(); - // FLEx repos keep their data on a branch named after the FDO model version; a clone looking for - // that branch reports "no such branch" if the initial commit landed on 'default' instead. hg - // records the branch for the *next* commit, so this must run before the first commit. Numeric - // branch names are accepted thanks to the fixutf8 extension wired into Mercurial/mercurial.ini. + // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version + // (e.g. 7500002.7000072); a clone looking for that branch reports "no such branch" if the initial + // commit landed on 'default' instead. hg records the branch for the *next* commit, so this must + // run before the first commit. Numeric/dotted branch names are accepted thanks to the fixutf8 + // extension wired into Mercurial/mercurial.ini. await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress); } From 51b46b85c53360d8addb3027fb397b10828c1e09 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 13:42:59 +0700 Subject: [PATCH 18/60] Commit FLExProject.CustomProperties as project genesis, not the .fwdata Match how FieldWorks bootstraps a new repo: the first commit is a minimal FLExProject.CustomProperties file (an empty list) on hg's default branch, and the .fwdata is no longer committed directly. Send/Receive then splits the .fwdata into the nested files Mercurial actually tracks (the .fwdata is excluded from tracking), landing them on the model-version branch. The genesis file is written with SIL's CanonicalXmlSettings so its bytes match exactly what the splitter regenerates, leaving no spurious diff on the first Send/Receive. Reorder so the genesis commit lands on 'default', then set the model-version branch, then Send/Receive. Add a CommitEmpty helper used as a temporary workaround for a bug in LanguageForgeSendReceiveActionHandler, which refuses to commit when doing so could create a new branch and so won't establish the model-version branch itself; the empty commit records the branch change and creates the head Send/Receive needs. Remove once that bug is fixed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/ISendReceiveService.cs | 1 + .../Services/ProjectCreationService.cs | 54 +++++++++++++++---- .../FwHeadless/Services/SendReceiveHelpers.cs | 11 ++++ .../FwHeadless/Services/SendReceiveService.cs | 5 ++ 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs index ffb975413c..98c9a573e9 100644 --- a/backend/FwHeadless/Services/ISendReceiveService.cs +++ b/backend/FwHeadless/Services/ISendReceiveService.cs @@ -10,6 +10,7 @@ public interface ISendReceiveService Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode); Task PendingCommitCountBothWays(FwDataProject project, string? projectCode); Task CommitFile(string filePath, string commitMessage); + Task CommitEmpty(string folder, string commitMessage); Task InitRepo(string folder); Task SetBranch(string folder, string branchName); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index e1245c3086..bad21ad212 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -1,9 +1,11 @@ using System.Xml; +using System.Xml.Linq; using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.LcmUtils; using LexCore.Entities; using LexCore.Exceptions; using Microsoft.Extensions.Options; +using SIL.Xml; namespace FwHeadless.Services; @@ -20,6 +22,10 @@ public class ProjectCreationService( { private const string InitialCommitMessage = "Initial project creation"; + // The one file FieldWorks/FLExBridge put in a brand-new FLEx repo's first commit: a minimal custom + // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge). + private const string CustomPropertiesFilename = "FLExProject.CustomProperties"; + public async Task CreateFromTemplate( Guid projectId, string projectCode, @@ -41,24 +47,36 @@ public async Task CreateFromTemplate( var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); try { - // Build the first commit of a brand-new FLEx repo from scratch, mirroring LfMerge's genesis - // recipe (hg init -> branch -> write files -> commit) since Clone can't establish an empty - // remote and Language_Forge_Send_Receive refuses to make the *first* commit itself: + // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/ + // FLExBridge create a repo, since Clone can't establish an empty remote and + // Language_Forge_Send_Receive refuses to make the *first* commit itself: // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). - // 2. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, - // e.g. 7500002.7000072) BEFORE any commit; FLEx/LfMergeBridge look for the data on that - // branch, so a first commit on 'default' would be invisible to them. - // 3. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all + // 2. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all // of the requested writing systems. - // 4. hg add + commit the (untracked) fwdata onto that branch, then push. + // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch -- the same + // genesis file, on the same branch, FieldWorks commits first. The fwdata itself is NOT + // committed; Send/Receive splits it into the nested files Mercurial actually tracks + // (fwdata is excluded from tracking). + // 4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, + // e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on + // that branch, so the split data must land there while the genesis stays on 'default'. + // 5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push. await srService.InitRepo(fwDataProject.ProjectFolder); - await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); ApplyAnthropologyCategories(anthropologyCategories); AssertModelVersionMatches(fwDataProject); - await srService.CommitFile(fwDataProject.FilePath, InitialCommitMessage); + var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject); + await srService.CommitFile(customPropertiesFile, InitialCommitMessage); + + await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); + // BUG WORKAROUND (remove once the LanguageForgeSendReceiveActionHandler bug is fixed): that + // handler refuses to commit when doing so "could possibly create a new branch", so it won't + // establish the model-version branch itself. This empty commit (no file changes -- it only + // records the branch change from SetBranch above) creates a head on the branch so Send/Receive + // will proceed. + await srService.CommitEmpty(fwDataProject.ProjectFolder, InitialCommitMessage); var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage); if (!pushResult.Success) throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); @@ -88,6 +106,22 @@ private void BuildFromTemplate( using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs); } + /// + /// Writes the genesis FLExProject.CustomProperties file into the repo root: an empty custom-property + /// list (just an <AdditionalFields /> root). Uses SIL's canonical XML settings so the bytes match + /// exactly what the first Send/Receive's project splitter regenerates, leaving no spurious diff. + /// + private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) + { + var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename); + using (var writer = XmlWriter.Create(path, CanonicalXmlSettings.CreateXmlWriterSettings())) + { + writer.WriteStartDocument(); + new XElement("AdditionalFields").WriteTo(writer); + } + return path; + } + private void ApplyAnthropologyCategories(AnthropologyCategories anthropologyCategories) { // TODO: Implement — populate the project with the requested anthropology categories diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 902de71411..04b7c6f0e3 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -160,6 +160,17 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", fileDir, progress); } + public static async Task CommitEmpty(string folder, string commitMessage, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + progress ??= new NullProgress(); + // No `hg add` and no file changes: this succeeds only because a pending branch change (from a + // preceding SetBranch) is itself a committable change in Mercurial, so it records a new head on + // that branch. A plain commit with truly nothing changed would exit 1 and throw. + await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress); + } + public static async Task InitRepo(string folder, IProgress? progress = null) { using var activity = FwHeadlessActivitySource.Value.StartActivity(); diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs index 9ab17bd8f9..3ef8955a36 100644 --- a/backend/FwHeadless/Services/SendReceiveService.cs +++ b/backend/FwHeadless/Services/SendReceiveService.cs @@ -71,6 +71,11 @@ public async Task CommitFile(string filePath, string commitMessage) await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress); } + public async Task CommitEmpty(string folder, string commitMessage) + { + await SendReceiveHelpers.CommitEmpty(folder, commitMessage, progress); + } + public async Task InitRepo(string folder) { await SendReceiveHelpers.InitRepo(folder, progress); From e01f766333462ab563f0f1a9c2a2492cebea6e09 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 13:45:38 +0700 Subject: [PATCH 19/60] Verify created project's writing systems via LanguageProject.langproj The template project no longer commits fw.fwdata to the repo, so fetching raw-file/tip/fw.fwdata always 404s. Instead fetch General/LanguageProject.langproj (one of the nested files Send/Receive produces), parse it as XML, and assert the requested writing systems are current: each analysis code appears in CurAnalysisWss/Uni and each vernacular code in CurVernWss/Uni. Check presence rather than exact equality because FieldWorks adds its own defaults (e.g. "en" as an analysis WS), so the stored list is a superset of what was requested. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../CreateProjectFromTemplateTests.cs | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs index 0bac5b784d..3cb445f986 100644 --- a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs +++ b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs @@ -1,6 +1,7 @@ using System.Net; using System.Net.Http.Json; using System.Text.Json.Nodes; +using System.Xml.Linq; using FluentAssertions; using Testing.ApiTests; using Testing.Fixtures; @@ -59,14 +60,28 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin tip.Should().NotBeNullOrEmpty(); tip!.Replace("0", "").Should().NotBeEmpty("the repo tip should not be the all-zero empty-repo hash"); - // 3. The pushed .fwdata carries the requested writing systems. FwHeadless always names the - // file fw.fwdata (its fixed project name), so fetch that from hgweb and check the codes. - var fwDataResponse = await _adminApiTester.HttpClient.GetAsync($"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/fw.fwdata"); - fwDataResponse.EnsureSuccessStatusCode(); - var fwData = await fwDataResponse.Content.ReadAsStringAsync(); - fwData.Should().NotBeEmpty(); - foreach (var ws in vernacular.Concat(analysis)) - fwData.Should().Contain($"\"{ws}\"", "writing system {0} should be present in the pushed project", ws); + // 3. The pushed project carries the requested writing systems. Send/Receive split the + // template .fwdata into the nested files hg actually tracks; the LangProject's current + // analysis/vernacular writing systems live in General/LanguageProject.langproj (XML). + var langprojResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/General/LanguageProject.langproj"); + langprojResponse.EnsureSuccessStatusCode(); + var langprojXml = await langprojResponse.Content.ReadAsStringAsync(); + langprojXml.Should().NotBeEmpty(); + + // de en pt + // fr es ... + var langProject = XDocument.Parse(langprojXml).Root?.Element("LangProject"); + langProject.Should().NotBeNull("LanguageProject.langproj should contain a LangProject element"); + var curAnalysisWss = SpaceSeparatedUni(langProject!, "CurAnalysisWss"); + var curVernWss = SpaceSeparatedUni(langProject!, "CurVernWss"); + + // The requested writing systems should be current. FieldWorks may add its own defaults (e.g. + // "en" as an analysis WS), so assert each requested code is present rather than exact equality. + foreach (var ws in analysis) + curAnalysisWss.Should().Contain(ws, "analysis writing system {0} should be current in the project", ws); + foreach (var ws in vernacular) + curVernWss.Should().Contain(ws, "vernacular writing system {0} should be current in the project", ws); } finally { @@ -83,6 +98,13 @@ public async Task CreateFromTemplate_RejectsWhenNoVernacularWritingSystem() response.StatusCode.Should().Be(HttpStatusCode.BadRequest); } + // Reads a LangProject child element's text (a space-separated writing-system list) and splits it. + private static string[] SpaceSeparatedUni(XElement langProject, string elementName) + { + var uni = langProject.Element(elementName)?.Element("Uni")?.Value ?? ""; + return uni.Split(' ', StringSplitOptions.RemoveEmptyEntries); + } + private async Task SoftDeleteProject(Guid projectId) { try From 94a74b3059ebc11ccc5bab6c7d7af9f74625375b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 14:46:54 +0700 Subject: [PATCH 20/60] Rewrite LinkedFilesRootDir to the Windows separator in new projects liblcm's CreateNewLangProj builds LinkedFilesRootDir with Path.Combine, which on this Linux host yields '%proj%/LinkedFiles' (forward slash). FieldWorks on Windows compares this value with an exact string match against '%proj%\LinkedFiles' (backslash), so it prints a warning on its first Send/Receive of a project we created. After building the template .fwdata, rewrite just that one value to the backslash form. Rather than parse the potentially multi-MB fwdata as XML, stream it once: find the line beginning with and replace the forward slash on the following line, copying every other line through unchanged. The edit is length-preserving (exactly one byte changes) and anchored to the LinkedFilesRootDir element so no other path is touched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/ProjectCreationService.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index bad21ad212..0445b435ce 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -64,6 +64,7 @@ public async Task CreateFromTemplate( await srService.InitRepo(fwDataProject.ProjectFolder); BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); + FixLinkedFilesRootDirSeparator(fwDataProject); ApplyAnthropologyCategories(anthropologyCategories); AssertModelVersionMatches(fwDataProject); @@ -106,6 +107,49 @@ private void BuildFromTemplate( using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs); } + /// + /// FieldWorks (Windows) compares LangProject.LinkedFilesRootDir with an exact string, expecting the + /// Windows-style default %proj%\LinkedFiles. liblcm builds that path with Path.Combine, which on + /// this Linux host yields %proj%/LinkedFiles (forward slash), so FieldWorks warns on its first + /// Send/Receive. Rewrite just that one value to the backslash form. + /// + /// We deliberately do NOT parse the (potentially multi-MB) fwdata as XML -- that would be a needless + /// cost. The value always sits on the line immediately after the <LinkedFilesRootDir> open tag, + /// so stream the file once, fix the <Uni> on the line following that tag, and copy every other + /// line through untouched. + /// + private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) + { + const string markerLinePrefix = "%proj%\LinkedFiles"; + + var sourcePath = fwDataProject.FilePath; + var tempPath = sourcePath + ".tmp"; + using (var reader = new StreamReader(sourcePath)) + using (var writer = new StreamWriter(tempPath)) + { + var nextLineIsValue = false; + var done = false; + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (nextLineIsValue) + { + line = line.Replace(forwardSlashValue, backslashValue); + nextLineIsValue = false; + done = true; + } + else if (!done && line.StartsWith(markerLinePrefix, StringComparison.Ordinal)) + { + nextLineIsValue = true; + } + writer.WriteLine(line); + } + } + File.Move(tempPath, sourcePath, overwrite: true); + } + /// /// Writes the genesis FLExProject.CustomProperties file into the repo root: an empty custom-property /// list (just an <AdditionalFields /> root). Uses SIL's canonical XML settings so the bytes match From 666789e0432b8f05ea7bec6817947b03f93f1d10 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 14:56:05 +0700 Subject: [PATCH 21/60] Remove the unimplemented AnthropologyCategories feature Populating a new project's anthropology categories requires template files that live only in FieldWorks, not in liblcm, so the feature can't be implemented on the FwHeadless side. Remove the AnthropologyCategories enum and every parameter, DTO field, and no-op handler that referenced it: the createFromTemplate REST param, CreateProjectFromTemplateInput, FwHeadlessClient, the FwHeadless route, and ProjectCreationService (including the ApplyAnthropologyCategories stub). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/FwHeadless/Routes/ProjectRoutes.cs | 2 +- .../FwHeadless/Services/ProjectCreationService.cs | 14 +------------- backend/LexBoxApi/Controllers/ProjectController.cs | 4 +--- backend/LexBoxApi/Services/FwHeadlessClient.cs | 3 +-- backend/LexCore/Entities/AnthropologyCategories.cs | 11 ----------- .../Entities/CreateProjectFromTemplateInput.cs | 3 +-- 6 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 backend/LexCore/Entities/AnthropologyCategories.cs diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index 073d71082f..d90bafde95 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -47,7 +47,7 @@ private static async Task> CreateProjectFromTempl } await projectCreationService.CreateFromTemplate( - projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi, input.AnthropologyCategories); + projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi); return TypedResults.Ok(); } } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 0445b435ce..ac20ca9606 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -2,7 +2,6 @@ using System.Xml.Linq; using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.LcmUtils; -using LexCore.Entities; using LexCore.Exceptions; using Microsoft.Extensions.Options; using SIL.Xml; @@ -31,8 +30,7 @@ public async Task CreateFromTemplate( string projectCode, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, - string uiWritingSystem, - AnthropologyCategories anthropologyCategories) + string uiWritingSystem) { if (vernacularWritingSystems.Count == 0) throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); @@ -65,7 +63,6 @@ public async Task CreateFromTemplate( BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); FixLinkedFilesRootDirSeparator(fwDataProject); - ApplyAnthropologyCategories(anthropologyCategories); AssertModelVersionMatches(fwDataProject); var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject); @@ -166,15 +163,6 @@ private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) return path; } - private void ApplyAnthropologyCategories(AnthropologyCategories anthropologyCategories) - { - // TODO: Implement — populate the project with the requested anthropology categories - // (enhanced/standard). Currently a no-op; 'none' is the only fully-handled option. - if (anthropologyCategories != AnthropologyCategories.None) - logger.LogWarning("anthropologyCategories '{Categories}' was requested but is not implemented yet; " + - "the new project will have no anthropology categories", anthropologyCategories); - } - /// /// The freshly-created project is stamped with the SIL.LCModel package's data-model version. /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 7ba4f699dc..c6b0b248e3 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -36,7 +36,6 @@ public class ProjectController( /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. /// Optional display name; defaults to the code. - /// Which anthropology categories to populate (default none). [HttpPost("createFromTemplate")] [AdminRequired] [ProducesResponseType(StatusCodes.Status200OK)] @@ -49,7 +48,6 @@ public async Task> CreateFromTemplate( [FromQuery] string[]? wsAnalysis = null, string wsUi = "en", string? name = null, - AnthropologyCategories anthropologyCategories = AnthropologyCategories.None, CancellationToken cancellationToken = default) { if (wsVernacular is null || wsVernacular.Length == 0) @@ -78,7 +76,7 @@ public async Task> CreateFromTemplate( (HttpStatusCode statusCode, string? error) result; try { - result = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, anthropologyCategories, cancellationToken); + result = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, cancellationToken); } catch { diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 0a3f01f8f1..3c1b35c343 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -209,10 +209,9 @@ public async Task UnblockProject(Guid projectId) IReadOnlyList wsVernacular, IReadOnlyList wsAnalysis, string wsUi, - AnthropologyCategories anthropologyCategories, CancellationToken cancellationToken = default) { - var input = new CreateProjectFromTemplateInput(wsVernacular, wsAnalysis, wsUi, anthropologyCategories); + var input = new CreateProjectFromTemplateInput(wsVernacular, wsAnalysis, wsUi); var response = await httpClient.PostAsJsonAsync($"/api/project/create-from-template?projectId={projectId}", input, cancellationToken); diff --git a/backend/LexCore/Entities/AnthropologyCategories.cs b/backend/LexCore/Entities/AnthropologyCategories.cs deleted file mode 100644 index bfddf2d1dd..0000000000 --- a/backend/LexCore/Entities/AnthropologyCategories.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace LexCore.Entities; - -/// -/// Which set of anthropology categories to populate a newly-created project with. -/// -public enum AnthropologyCategories -{ - None = 0, - Standard = 1, - Enhanced = 2, -} diff --git a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs b/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs index 24fcbccf79..c71a0c75fc 100644 --- a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs +++ b/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs @@ -8,5 +8,4 @@ namespace LexCore.Entities; public record CreateProjectFromTemplateInput( IReadOnlyList WsVernacular, IReadOnlyList WsAnalysis, - string WsUi, - AnthropologyCategories AnthropologyCategories); + string WsUi); From 2cf3b2da5f3f6452004986b29398c49c5788b58c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 23 Jul 2026 15:54:51 +0700 Subject: [PATCH 22/60] Guard failed-creation cleanup with FwHeadless project deletion CleanupFailedProjectCreation deleted the hg repo and project row unconditionally, without asking FwHeadless to delete its local project state. Call fwHeadless.DeleteProject first, mirroring DeleteProjectPermanently: it cleans FwHeadless's local state and throws ProjectSyncInProgressException if a creation or sync is still in flight, so we don't tear down an active or already-completed creation. The repo, database, and cache cleanup are preserved after the guard succeeds, with the repo deletion moved after the database removal. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/LexBoxApi/Services/ProjectService.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index f563e8efa9..0568df2de8 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -234,19 +234,23 @@ public async Task FinishReset(string code, Stream? zipFile = null) /// /// Compensating cleanup for a project whose template population failed after CreateProject: - /// removes the repo and the project row and invalidates the same caches as a permanent delete. + /// deletes FwHeadless's local project state, then removes the repo and the project row and + /// invalidates the same caches as a permanent delete. /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a - /// project this same request just created. FwHeadless cleans up its own local state on failure. + /// project this same request just created. /// public async Task CleanupFailedProjectCreation(Guid projectId, string code) { - await hgService.DeleteRepoIfExists(code); + // do this first, because it throws if a creation or sync is in progress, which stops us from + // tearing down an active or already-completed creation (mirrors DeleteProjectPermanently) + await fwHeadless.DeleteProject(projectId); var project = await dbContext.Projects.FindAsync(projectId); if (project is not null) { dbContext.Projects.Remove(project); await dbContext.SaveChangesAsync(); } + await hgService.DeleteRepoIfExists(code); // Don't forget to add more Invalidate calls here if we add new caches InvalidateProjectCodeCache(code); InvalidateProjectConfidentialityCache(projectId); From abc9eee7fb1f00b247859ba2fd454b62a11430b9 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 24 Jul 2026 09:14:19 +0700 Subject: [PATCH 23/60] Stop the create-from-template test retrying its non-idempotent POST CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritingSystems failed in CI with 409 "already exists". The shared ApiTestBase HttpClient retries transient failures and timeouts, but createFromTemplate is a long-running, non-idempotent operation: the first attempt creates the project row, then (now that the semantic domains actually get committed and pushed) runs long enough to time out or transiently fail, and the retried POST hits the just-created project and returns 409. Give NewHttpClient a retryTransientFailures flag and have the test issue the creation POST on a non-retrying client with a longer timeout, authenticated as the same admin via Bearer token. A single attempt now surfaces the real result instead of a misleading conflict. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/Testing/ApiTests/ApiTestBase.cs | 22 +++++++++++++------ .../CreateProjectFromTemplateTests.cs | 10 ++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/backend/Testing/ApiTests/ApiTestBase.cs b/backend/Testing/ApiTests/ApiTestBase.cs index 74d9492b0c..e0ffec9e8e 100644 --- a/backend/Testing/ApiTests/ApiTestBase.cs +++ b/backend/Testing/ApiTests/ApiTestBase.cs @@ -29,17 +29,25 @@ public ApiTestBase() /// /// bas url for the client /// enable or disable cookies for the client - public static (SocketsHttpHandler Handler, HttpClient Client) NewHttpClient(string? baseUrl = null, bool useCookies = true) + /// + /// retry transient failures (5xx/timeouts). Leave on for idempotent calls; turn off for + /// non-idempotent operations (e.g. project creation) where a retried request would collide with + /// the resource the first attempt already created. + /// + public static (SocketsHttpHandler Handler, HttpClient Client) NewHttpClient(string? baseUrl = null, bool useCookies = true, bool retryTransientFailures = true) { - var retryPipeline = new ResiliencePipelineBuilder() - .AddRetry(new HttpRetryStrategyOptions { BackoffType = DelayBackoffType.Linear, MaxRetryAttempts = 3 }) - .Build(); - var socketsHttpHandler = new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(15), UseCookies = useCookies }; + HttpMessageHandler handler = socketsHttpHandler; + if (retryTransientFailures) + { + var retryPipeline = new ResiliencePipelineBuilder() + .AddRetry(new HttpRetryStrategyOptions { BackoffType = DelayBackoffType.Linear, MaxRetryAttempts = 3 }) + .Build(); #pragma warning disable EXTEXP0001 - var resilienceHandler = new ResilienceHandler(retryPipeline) { InnerHandler = socketsHttpHandler }; + handler = new ResilienceHandler(retryPipeline) { InnerHandler = socketsHttpHandler }; #pragma warning restore EXTEXP0001 - var httpClient = new HttpClient(resilienceHandler); + } + var httpClient = new HttpClient(handler); if (!string.IsNullOrEmpty(baseUrl)) { httpClient.BaseAddress = new Uri(baseUrl); diff --git a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs index 3cb445f986..d78aa86037 100644 --- a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs +++ b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json.Nodes; using System.Xml.Linq; @@ -43,7 +44,14 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin { // 1. Create the project via the admin endpoint (creates the DB row + empty repo, then has // FwHeadless build the template .fwdata and push it into the empty repo). - var response = await _adminApiTester.HttpClient.PostAsync( + // This is a long-running, non-idempotent operation. The shared tester's HttpClient retries + // transient failures/timeouts, so a slow first attempt (which already created the project + // row) would be retried and the retry would return 409 "already exists". Use a client that + // doesn't retry and allows more time, authenticated as the same admin. + using var createClient = ApiTestBase.NewHttpClient(_adminApiTester.BaseUrl, retryTransientFailures: false).Client; + createClient.Timeout = TimeSpan.FromMinutes(5); + createClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _adminApiTester.CurrJwt); + var response = await createClient.PostAsync( $"{_adminApiTester.BaseUrl}/api/project/createFromTemplate{query}", null); response.StatusCode.Should().Be(HttpStatusCode.OK, "creation should succeed; body: {0}", await response.Content.ReadAsStringAsync()); From 9d1f53646d5055f1eba5180335613242feee57db Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 24 Jul 2026 09:49:50 +0700 Subject: [PATCH 24/60] Authenticate the create-from-template client via its own login The previous fix gave the creation POST a dedicated non-retrying client but authenticated it with _adminApiTester.CurrJwt as a Bearer token. That JWT is captured once at login and goes stale (the shared client stays authenticated via its refreshed cookie, not that value), so the request failed with 401. Have the dedicated client log in itself via JwtHelper.ExecuteLogin, which sets a fresh auth cookie on it - matching how the shared tester authenticates - while keeping the no-retry behavior and longer timeout. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../SyncReverseProxy/CreateProjectFromTemplateTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs index d78aa86037..7e17ff5402 100644 --- a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs +++ b/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs @@ -1,12 +1,13 @@ using System.Net; -using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json.Nodes; using System.Xml.Linq; using FluentAssertions; using Testing.ApiTests; using Testing.Fixtures; +using Testing.Services; using Xunit.Abstractions; +using static Testing.Services.Constants; namespace Testing.SyncReverseProxy; @@ -47,10 +48,11 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin // This is a long-running, non-idempotent operation. The shared tester's HttpClient retries // transient failures/timeouts, so a slow first attempt (which already created the project // row) would be retried and the retry would return 409 "already exists". Use a client that - // doesn't retry and allows more time, authenticated as the same admin. + // doesn't retry and allows more time. Log it in itself (cookie auth) rather than reusing the + // shared tester's captured JWT, which may be stale. using var createClient = ApiTestBase.NewHttpClient(_adminApiTester.BaseUrl, retryTransientFailures: false).Client; createClient.Timeout = TimeSpan.FromMinutes(5); - createClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _adminApiTester.CurrJwt); + await JwtHelper.ExecuteLogin(AdminAuth, includeDefaultScope: true, createClient); var response = await createClient.PostAsync( $"{_adminApiTester.BaseUrl}/api/project/createFromTemplate{query}", null); response.StatusCode.Should().Be(HttpStatusCode.OK, From 1d1454306ad0b4b0d498579adfe714d18e763947 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 12 Aug 2026 11:07:19 +0700 Subject: [PATCH 25/60] Rename create-from-template to init-fwdata-project Both endpoints initialize a new project's repo with a template .fwdata, so name them for what they produce: - FwHeadless: POST /api/project/create-from-template -> /init-fwdata-project - LexBoxApi: POST /api/project/createFromTemplate -> /initFwDataProject Renames the handlers, ProjectCreationService.CreateFromTemplate, the FwHeadlessClient method, the CreateProjectFromTemplateInput record, and the two test classes covering them. CrdtProjectsService.CreateProjectFromTemplate is an unrelated FwLite/CRDT API and keeps its name. Co-Authored-By: Claude Opus 5 (1M context) --- backend/FwHeadless/Routes/ProjectRoutes.cs | 10 +++++----- .../Services/ProjectCreationService.cs | 2 +- .../LexBoxApi/Controllers/ProjectController.cs | 6 +++--- backend/LexBoxApi/LexBoxKernel.cs | 2 +- backend/LexBoxApi/Services/FwHeadlessClient.cs | 8 ++++---- ...emplateInput.cs => InitFwDataProjectInput.cs} | 4 ++-- ...ts.cs => InitFwDataProjectValidationTests.cs} | 8 ++++---- ...emplateTests.cs => InitFwDataProjectTests.cs} | 16 ++++++++-------- 8 files changed, 28 insertions(+), 28 deletions(-) rename backend/LexCore/Entities/{CreateProjectFromTemplateInput.cs => InitFwDataProjectInput.cs} (67%) rename backend/Testing/LexBoxApi/{CreateFromTemplateValidationTests.cs => InitFwDataProjectValidationTests.cs} (87%) rename backend/Testing/SyncReverseProxy/{CreateProjectFromTemplateTests.cs => InitFwDataProjectTests.cs} (89%) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index d90bafde95..3e8e42b52f 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -10,16 +10,16 @@ public static class ProjectRoutes public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app) { var group = app.MapGroup("/api/project"); - group.MapPost("/create-from-template", CreateProjectFromTemplate); + group.MapPost("/init-fwdata-project", InitFwDataProject); return group; } // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates. - private static async Task> CreateProjectFromTemplate( + private static async Task> InitFwDataProject( Guid projectId, - CreateProjectFromTemplateInput input, + InitFwDataProjectInput input, IProjectLookupService projectLookupService, ProjectCreationService projectCreationService, ILogger logger) @@ -42,11 +42,11 @@ private static async Task> CreateProjectFromTempl var projectCode = await projectLookupService.GetProjectCode(projectId); if (projectCode is null) { - logger.LogError("Create-from-template request for non-existent project {ProjectId}", projectId); + logger.LogError("Init-fwdata-project request for non-existent project {ProjectId}", projectId); return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound); } - await projectCreationService.CreateFromTemplate( + await projectCreationService.InitFwDataProject( projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi); return TypedResults.Ok(); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index ac20ca9606..169693468a 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -25,7 +25,7 @@ public class ProjectCreationService( // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge). private const string CustomPropertiesFilename = "FLExProject.CustomProperties"; - public async Task CreateFromTemplate( + public async Task InitFwDataProject( Guid projectId, string projectCode, IReadOnlyList vernacularWritingSystems, diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index c6b0b248e3..8b171857e0 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -36,13 +36,13 @@ public class ProjectController( /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. /// Optional display name; defaults to the code. - [HttpPost("createFromTemplate")] + [HttpPost("initFwDataProject")] [AdminRequired] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status400BadRequest)] [ProducesResponseType(StatusCodes.Status409Conflict)] [ProducesResponseType(StatusCodes.Status500InternalServerError)] - public async Task> CreateFromTemplate( + public async Task> InitFwDataProject( string code, [FromQuery] string[] wsVernacular, [FromQuery] string[]? wsAnalysis = null, @@ -76,7 +76,7 @@ public async Task> CreateFromTemplate( (HttpStatusCode statusCode, string? error) result; try { - result = await fwHeadlessClient.CreateProjectFromTemplate(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, cancellationToken); + result = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, cancellationToken); } catch { diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index b8768932ab..1842a1635b 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -55,7 +55,7 @@ public static void AddLexBoxApi(this IServiceCollection services, services.AddHttpClient(client => { client.BaseAddress = new ("http://fwHeadless"); - // create-from-template runs inline (clone empty repo + build the template project + push), + // init-fwdata-project runs inline (clone empty repo + build the template project + push), // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom. client.Timeout = TimeSpan.FromMinutes(5); }) diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 3c1b35c343..30693ef257 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -205,19 +205,19 @@ public async Task UnblockProject(Guid projectId) /// success; on failure returns the FwHeadless status code and error body so the caller can /// propagate the right status (e.g. surface a 400 rather than flattening it to 500). /// - public async Task<(HttpStatusCode StatusCode, string? Error)> CreateProjectFromTemplate(Guid projectId, + public async Task<(HttpStatusCode StatusCode, string? Error)> InitFwDataProject(Guid projectId, IReadOnlyList wsVernacular, IReadOnlyList wsAnalysis, string wsUi, CancellationToken cancellationToken = default) { - var input = new CreateProjectFromTemplateInput(wsVernacular, wsAnalysis, wsUi); - var response = await httpClient.PostAsJsonAsync($"/api/project/create-from-template?projectId={projectId}", + var input = new InitFwDataProjectInput(wsVernacular, wsAnalysis, wsUi); + var response = await httpClient.PostAsJsonAsync($"/api/project/init-fwdata-project?projectId={projectId}", input, cancellationToken); if (response.IsSuccessStatusCode) return (response.StatusCode, null); var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); - logger.LogError("Failed to create project from template: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + logger.LogError("Failed to init fwdata project: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", response.StatusCode, response.ReasonPhrase, projectId, diff --git a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs similarity index 67% rename from backend/LexCore/Entities/CreateProjectFromTemplateInput.cs rename to backend/LexCore/Entities/InitFwDataProjectInput.cs index c71a0c75fc..ba2e2b72a9 100644 --- a/backend/LexCore/Entities/CreateProjectFromTemplateInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -1,11 +1,11 @@ namespace LexCore.Entities; /// -/// Body of the internal FwHeadless "create project from template" call. The vernacular/analysis +/// Body of the internal FwHeadless "init fwdata project" call. The vernacular/analysis /// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one /// vernacular; analysis and UI default to "en"). /// -public record CreateProjectFromTemplateInput( +public record InitFwDataProjectInput( IReadOnlyList WsVernacular, IReadOnlyList WsAnalysis, string WsUi); diff --git a/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs similarity index 87% rename from backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs rename to backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index 7d1424d6a8..efd3a7a7f6 100644 --- a/backend/Testing/LexBoxApi/CreateFromTemplateValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -7,11 +7,11 @@ namespace Testing.LexBoxApi; /// -/// Unit tests for ProjectController.CreateFromTemplate input validation. The 400 branches run before +/// Unit tests for ProjectController.InitFwDataProject input validation. The 400 branches run before /// any injected service is touched, so the controller can be built with null dependencies; only a /// ProblemDetailsFactory (from AddControllers) is needed for Problem() to render. /// -public class CreateFromTemplateValidationTests +public class InitFwDataProjectValidationTests { private static ProjectController NewController() { @@ -30,7 +30,7 @@ private static ProjectController NewController() [Fact] public async Task Rejects_when_no_vernacular_writing_system_is_supplied() { - var result = await NewController().CreateFromTemplate("myproj", wsVernacular: []); + var result = await NewController().InitFwDataProject("myproj", wsVernacular: []); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); @@ -40,7 +40,7 @@ public async Task Rejects_when_no_vernacular_writing_system_is_supplied() [Fact] public async Task Rejects_an_invalid_project_code() { - var result = await NewController().CreateFromTemplate("Bad Code!", wsVernacular: ["fr"]); + var result = await NewController().InitFwDataProject("Bad Code!", wsVernacular: ["fr"]); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); diff --git a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs similarity index 89% rename from backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs rename to backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index 7e17ff5402..df3d7e05a2 100644 --- a/backend/Testing/SyncReverseProxy/CreateProjectFromTemplateTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -12,25 +12,25 @@ namespace Testing.SyncReverseProxy; /// -/// End-to-end coverage for the admin "create project from a template" API. This is the one path that +/// End-to-end coverage for the admin "init fwdata project" API. This is the one path that /// can't be unit-tested: LexBox creates an empty hg repo and FwHeadless does the first push of a /// template-built .fwdata into it (via LfMergeBridge/Chorus against the real hgweb). Requires the /// lexbox stack, so it only runs in CI. /// [Trait("Category", "Integration")] -public class CreateProjectFromTemplateTests : IClassFixture +public class InitFwDataProjectTests : IClassFixture { private readonly ITestOutputHelper _output; private readonly ApiTestBase _adminApiTester; - public CreateProjectFromTemplateTests(ITestOutputHelper output, IntegrationFixture fixture) + public InitFwDataProjectTests(ITestOutputHelper output, IntegrationFixture fixture) { _output = output; _adminApiTester = fixture.AdminApiTester; } [Fact] - public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritingSystems() + public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWritingSystems() { // Valid code (lowercase/digits/hyphen, doesn't start with a hyphen), unique per run. var code = $"tmpl-{Guid.NewGuid():N}"[..12]; @@ -54,7 +54,7 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin createClient.Timeout = TimeSpan.FromMinutes(5); await JwtHelper.ExecuteLogin(AdminAuth, includeDefaultScope: true, createClient); var response = await createClient.PostAsync( - $"{_adminApiTester.BaseUrl}/api/project/createFromTemplate{query}", null); + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject{query}", null); response.StatusCode.Should().Be(HttpStatusCode.OK, "creation should succeed; body: {0}", await response.Content.ReadAsStringAsync()); projectId = await response.Content.ReadFromJsonAsync(); @@ -100,11 +100,11 @@ public async Task CreateFromTemplate_PopulatesTheEmptyRepoWithTheRequestedWritin } [Fact] - public async Task CreateFromTemplate_RejectsWhenNoVernacularWritingSystem() + public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() { var code = $"tmpl-{Guid.NewGuid():N}"[..12]; var response = await _adminApiTester.HttpClient.PostAsync( - $"{_adminApiTester.BaseUrl}/api/project/createFromTemplate?code={code}&wsAnalysis=en", null); + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject?code={code}&wsAnalysis=en", null); response.StatusCode.Should().Be(HttpStatusCode.BadRequest); } @@ -130,7 +130,7 @@ await _adminApiTester.ExecuteGql($$""" } catch (Exception ex) { - _output.WriteLine($"[CreateProjectFromTemplateTests] Ignored cleanup exception: {ex}"); + _output.WriteLine($"[InitFwDataProjectTests] Ignored cleanup exception: {ex}"); } } } From c0fc9cbc549a57aba099661dd8d19668e851ddb5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 24 Aug 2026 12:16:13 +0700 Subject: [PATCH 26/60] Add admin-only `projectOrigin` param Only admins are allowed to pass this parameter, which sets the ProjectOrigin value in the newly-created project. This will be used for tagging previously-non-SR projects migrated from Language Forge. --- .../Controllers/ProjectController.cs | 25 ++++++- backend/LexBoxApi/Services/ProjectService.cs | 4 +- backend/LexCore/Entities/Project.cs | 1 + .../InitFwDataProjectValidationTests.cs | 66 ++++++++++++++++++- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 8b171857e0..5360d60d69 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -36,6 +36,7 @@ public class ProjectController( /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. /// Optional display name; defaults to the code. + /// Optional to record as the project's origin; admin-only. [HttpPost("initFwDataProject")] [AdminRequired] [ProducesResponseType(StatusCodes.Status200OK)] @@ -48,12 +49,24 @@ public async Task> InitFwDataProject( [FromQuery] string[]? wsAnalysis = null, string wsUi = "en", string? name = null, + string? projectOrigin = null, CancellationToken cancellationToken = default) { if (wsVernacular is null || wsVernacular.Length == 0) return Problem("At least one vernacular writing system is required", statusCode: StatusCodes.Status400BadRequest); if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); + + ProjectMigrationStatus? origin = null; + if (!string.IsNullOrEmpty(projectOrigin)) + { + // Only admins may say where a project came from. Currently redundant with [AdminRequired], + // but this endpoint is expected to open up to non-admins later; this check must stay. + permissionService.AssertIsAdmin(); + if (!TryParseProjectOrigin(projectOrigin, out var parsedOrigin)) + return Problem($"Invalid project origin '{projectOrigin}'", statusCode: StatusCodes.Status400BadRequest); + origin = parsedOrigin; + } if (await projectService.ProjectExists(code)) return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); @@ -68,7 +81,8 @@ public async Task> InitFwDataProject( RetentionPolicy: RetentionPolicy.Verified, IsConfidential: false, ProjectManagerId: null, - OrgId: null)); + OrgId: null), + projectOrigin: origin); // Saga: the project row + empty repo now exist. Have FwHeadless populate the repo with the // template .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs @@ -97,6 +111,15 @@ public async Task> InitFwDataProject( return projectId; } + /// + /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists + /// ("Migrated,Migrating"), so the parsed value must round-trip to the name that was passed in. + /// + public static bool TryParseProjectOrigin(string projectOrigin, out ProjectMigrationStatus origin) => + Enum.TryParse(projectOrigin, ignoreCase: true, out origin) + && Enum.IsDefined(origin) + && string.Equals(origin.ToString(), projectOrigin, StringComparison.OrdinalIgnoreCase); + /// Analysis writing systems to use, defaulting to English when none are supplied. public static string[] AnalysisWritingSystemsOrDefault(string[]? wsAnalysis) => wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 0568df2de8..5aa82c252e 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -25,7 +25,7 @@ public class ProjectService( IEmailService emailService, FwHeadlessClient fwHeadless) { - public async Task CreateProject(CreateProjectInput input) + public async Task CreateProject(CreateProjectInput input, ProjectMigrationStatus? projectOrigin = null) { await using var transaction = await dbContext.Database.BeginTransactionAsync(); var projectId = input.Id ?? Guid.NewGuid(); @@ -41,7 +41,7 @@ public async Task CreateProject(CreateProjectInput input) Id = projectId, Code = input.Code, Name = input.Name, - ProjectOrigin = ProjectMigrationStatus.Migrated, + ProjectOrigin = projectOrigin ?? ProjectMigrationStatus.Migrated, Description = input.Description, Type = input.Type, LastCommit = null, diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index ce1c2e0fa3..e554061755 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -97,6 +97,7 @@ public enum ProjectMigrationStatus Migrating, PrivateRedmine, PublicRedmine, + LanguageForgeNonSR, } public enum ResetStatus diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index efd3a7a7f6..d1db78622f 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -1,8 +1,11 @@ using LexBoxApi.Controllers; +using LexCore.Entities; +using LexCore.ServiceInterfaces; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; +using Moq; namespace Testing.LexBoxApi; @@ -13,12 +16,12 @@ namespace Testing.LexBoxApi; /// public class InitFwDataProjectValidationTests { - private static ProjectController NewController() + private static ProjectController NewController(IPermissionService? permissionService = null) { var services = new ServiceCollection(); services.AddLogging(); services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves - return new ProjectController(null!, null!, null!, null!, null!, null!, NullLogger.Instance) + return new ProjectController(null!, null!, null!, permissionService!, null!, null!, NullLogger.Instance) { ControllerContext = new ControllerContext { @@ -47,6 +50,65 @@ public async Task Rejects_an_invalid_project_code() problem.Detail.Should().Contain("Invalid project code"); } + /// A permission service that either passes or fails the admin assertion. + private static IPermissionService PermissionService(bool isAdmin) + { + var mock = new Mock(); + if (!isAdmin) mock.Setup(p => p.AssertIsAdmin()).Throws(); + return mock.Object; + } + + [Fact] + public async Task Rejects_a_project_origin_that_is_not_a_migration_status() + { + var result = await NewController(PermissionService(isAdmin: true)) + .InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin"); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project origin"); + } + + [Fact] + public async Task Rejects_a_project_origin_from_a_non_admin() + { + // The admin check must happen before the value is even parsed, and must outlive [AdminRequired], + // which is expected to be relaxed for this endpoint later. + var act = () => NewController(PermissionService(isAdmin: false)) + .InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: "Migrated"); + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task Ignores_an_empty_project_origin_without_requiring_admin() + { + // An empty value isn't "specified", so it needs no admin check; a null permission service + // proves AssertIsAdmin() was never called. Fails later (null project service) than validation. + var act = () => NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: ""); + await act.Should().ThrowAsync(); + } + + [Theory] + [InlineData("Migrated", ProjectMigrationStatus.Migrated)] + [InlineData("migrating", ProjectMigrationStatus.Migrating)] + [InlineData("PUBLICREDMINE", ProjectMigrationStatus.PublicRedmine)] + public void Project_origin_parses_case_insensitively(string input, ProjectMigrationStatus expected) + { + ProjectController.TryParseProjectOrigin(input, out var origin).Should().BeTrue(); + origin.Should().Be(expected); + } + + [Theory] + [InlineData("NotAnOrigin")] + [InlineData("1")] // Enum.TryParse accepts raw numbers; the API takes names only + [InlineData("42")] + [InlineData("Migrated,Migrating")] // ...and comma-separated lists, which OR into a defined value here + [InlineData("")] + public void Project_origin_rejects_values_that_are_not_named_members(string input) + { + ProjectController.TryParseProjectOrigin(input, out _).Should().BeFalse(); + } + [Fact] public void Analysis_writing_systems_default_to_english_when_null() { From be9f3faecc84e16ed90f095eb7d9019f73304b63 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 10 Sep 2026 12:01:28 +0700 Subject: [PATCH 27/60] Rename internal API for consistency Now we don't have two different casings (camelCase vs kebab-case) of what is essentially the same API, one externally-accessible endpoint and one for internal use only. --- backend/FwHeadless/Routes/ProjectRoutes.cs | 4 ++-- backend/LexBoxApi/LexBoxKernel.cs | 2 +- backend/LexBoxApi/Services/FwHeadlessClient.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index 3e8e42b52f..fefa65f5ba 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -10,7 +10,7 @@ public static class ProjectRoutes public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app) { var group = app.MapGroup("/api/project"); - group.MapPost("/init-fwdata-project", InitFwDataProject); + group.MapPost("/initFwDataProject", InitFwDataProject); return group; } @@ -42,7 +42,7 @@ private static async Task> InitFwDataProject( var projectCode = await projectLookupService.GetProjectCode(projectId); if (projectCode is null) { - logger.LogError("Init-fwdata-project request for non-existent project {ProjectId}", projectId); + logger.LogError("initFwDataProject request for non-existent project {ProjectId}", projectId); return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound); } diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 1842a1635b..9f571a638d 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -55,7 +55,7 @@ public static void AddLexBoxApi(this IServiceCollection services, services.AddHttpClient(client => { client.BaseAddress = new ("http://fwHeadless"); - // init-fwdata-project runs inline (clone empty repo + build the template project + push), + // initFwDataProject runs inline (clone empty repo + build the template project + push), // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom. client.Timeout = TimeSpan.FromMinutes(5); }) diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 30693ef257..252801beb7 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -212,7 +212,7 @@ public async Task UnblockProject(Guid projectId) CancellationToken cancellationToken = default) { var input = new InitFwDataProjectInput(wsVernacular, wsAnalysis, wsUi); - var response = await httpClient.PostAsJsonAsync($"/api/project/init-fwdata-project?projectId={projectId}", + var response = await httpClient.PostAsJsonAsync($"/api/project/initFwDataProject?projectId={projectId}", input, cancellationToken); if (response.IsSuccessStatusCode) return (response.StatusCode, null); From fe0e27ee422c0e763802c1faaf4a5b3f9de1fbaf Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 10 Sep 2026 12:32:16 +0700 Subject: [PATCH 28/60] Add new migration status enum to GraphQL schema --- frontend/schema.graphql | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/schema.graphql b/frontend/schema.graphql index 5fbee946fa..6a4c75ddba 100644 --- a/frontend/schema.graphql +++ b/frontend/schema.graphql @@ -1546,6 +1546,7 @@ enum ProjectMigrationStatus { MIGRATING PRIVATE_REDMINE PUBLIC_REDMINE + LANGUAGE_FORGE_NON_SR } enum ProjectRole { From 7dfe0efbdfa4a8a523ac161f35dbca547ef2eedf Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 10 Sep 2026 15:05:19 +0700 Subject: [PATCH 29/60] Ensure new project has correctly-ordered wses CreateNewLangProj takes a HashSet for additional writing systems, meaning it won't necessarily create them in the order we want. So immediately after creating the new project, we reorder its writing systems to match the order passed in the query params. We check whether liblcm has added any writing systems (for example, it will add `en` if it wasn't there), and we make sure those get kept. --- .../LcmUtils/ProjectLoader.cs | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index e9a2f6f52a..ec690e420a 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -2,8 +2,10 @@ using System.Diagnostics; using System.Runtime.InteropServices; using Microsoft.Extensions.Options; +using SIL.Extensions; using SIL.LCModel; using SIL.LCModel.Core.WritingSystems; +using SIL.LCModel.Infrastructure; using SIL.LCModel.Utils; using SIL.WritingSystems; @@ -105,7 +107,12 @@ public virtual LcmCache NewProject(FwDataProject project, uiWs, AdditionalWritingSystems(analysisDefinitions), AdditionalWritingSystems(vernacularDefinitions)); - return LoadCache(project); + // LcmCache.CreateNewLangProj takes additional writing systems as a HashSet, so it might have added + // them in the wrong order. Right now before anyone else does anything with this project, order + // them correctly if needed. + var lcmCache = LoadCache(project); + ReorderWritingSystems(lcmCache, analysisDefinitions, vernacularDefinitions); + return lcmCache; } private static CoreWritingSystemDefinition CreateWritingSystemDefinition(string ws) => new(ws) { Id = ws }; @@ -134,4 +141,49 @@ private static void NewProject(IThreadedProgress progress, LcmCache.CreateNewLangProj(progress, [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs, uiWs, additionalAnalysisWss, additionalVernacularWss]); } + + private static IList CorrectlyOrderedWritingSystems(ICollection lcmList, + IEnumerable preferredOrder) + { + var result = preferredOrder.ToList(); // Defensive copy + var preferredIds = result.Select(ws => ws.Id).ToHashSet(); + foreach (var ws in lcmList) + { + if (!preferredIds.Contains(ws.Id)) + { + result.Add(ws); + } + } + return result; + } + + private static void ReorderWritingSystems(ICollection lcmList, + IEnumerable preferredOrder) + { + var reordered = CorrectlyOrderedWritingSystems(lcmList, preferredOrder); + // Guard against making liblcm fire WritingSystemListChanged if it doesn't need to + if (reordered.Select(ws => ws.Id).SequenceEqual(lcmList.Select(ws => ws.Id))) + { + return; + } + lcmList.Clear(); + lcmList.AddRange(reordered); + } + + private static void ReorderWritingSystems(LcmCache lcmCache, + IEnumerable preferredOrderAnalysis, + IEnumerable preferredOrderVernacular) + { + var wsContainer = lcmCache.LangProject; + NonUndoableUnitOfWorkHelper.Do(lcmCache.ActionHandlerAccessor, () => + { + // Have to do both the "Current" and "All" lists, because liblcm doesn't automatically + // keep their order in sync. If a ws is added to Current them it's also added to All, + // but that's it. A reorder wouldn't be automatically synced up, we must do so here. + ReorderWritingSystems(wsContainer.CurrentAnalysisWritingSystems, preferredOrderAnalysis); + ReorderWritingSystems(wsContainer.AnalysisWritingSystems, preferredOrderAnalysis); + ReorderWritingSystems(wsContainer.CurrentVernacularWritingSystems, preferredOrderVernacular); + ReorderWritingSystems(wsContainer.VernacularWritingSystems, preferredOrderVernacular); + }); + } } From cfaf87cb0d76957678b6352e15a478a90e2cc301 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 10 Sep 2026 16:10:07 +0700 Subject: [PATCH 30/60] Ensure writing systems are in the right order Includes a unit test to ensure they're all ordered correctly --- .../FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs | 8 ++++++++ .../Testing/SyncReverseProxy/InitFwDataProjectTests.cs | 9 +++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index ec690e420a..15fff22ee0 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -112,6 +112,7 @@ public virtual LcmCache NewProject(FwDataProject project, // them correctly if needed. var lcmCache = LoadCache(project); ReorderWritingSystems(lcmCache, analysisDefinitions, vernacularDefinitions); + lcmCache.ActionHandlerAccessor.Commit(); // Ensure changes are written out return lcmCache; } @@ -161,6 +162,13 @@ private static void ReorderWritingSystems(ICollection preferredOrder) { var reordered = CorrectlyOrderedWritingSystems(lcmList, preferredOrder); + // TODO: Perhaps the ones added by Lcm, if any, should be considered NON-current? If so, + // then we should add a boolean param and the line above should change to: + // var reordered = + // includeNonPreferred ? + // CorrectlyOrderedWritingSystems(lcmList, preferredOrder) : + // preferredOrder.ToList(); + // Guard against making liblcm fire WritingSystemListChanged if it doesn't need to if (reordered.Select(ws => ws.Id).SequenceEqual(lcmList.Select(ws => ws.Id))) { diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index df3d7e05a2..1f5267a422 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -34,8 +34,8 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting { // Valid code (lowercase/digits/hyphen, doesn't start with a hyphen), unique per run. var code = $"tmpl-{Guid.NewGuid():N}"[..12]; - var vernacular = new[] { "fr", "es" }; - var analysis = new[] { "de", "pt" }; + var vernacular = new[] { "fr", "es", "ko", "ru", "my" }; + var analysis = new[] { "de", "pt", "th", "id", "he", "el" }; var query = $"?code={code}" + string.Concat(vernacular.Select(ws => $"&wsVernacular={ws}")) + string.Concat(analysis.Select(ws => $"&wsAnalysis={ws}")); @@ -92,6 +92,11 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting curAnalysisWss.Should().Contain(ws, "analysis writing system {0} should be current in the project", ws); foreach (var ws in vernacular) curVernWss.Should().Contain(ws, "vernacular writing system {0} should be current in the project", ws); + + // The requested writing systems should *also* be in the correct order. Any writing systems added + // by FieldWorks should come last. + curAnalysisWss.Take(analysis.Length).Should().Equal(analysis, "analysis writing systems were in the wrong order"); + curVernWss.Take(vernacular.Length).Should().Equal(vernacular, "vernacular writing systems were in the wrong order"); } finally { From f6c1c99fb0528357c39c0e5931904671de92d475 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 10 Sep 2026 16:53:34 +0700 Subject: [PATCH 31/60] Mark liblcm-added wses as NON-current This will ensure that we don't end up adding a visible "en" to projects that didn't actually want one. --- .../LcmUtils/ProjectLoader.cs | 25 +++++++++---------- .../InitFwDataProjectTests.cs | 8 ++++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index 15fff22ee0..0fe37b86c4 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -158,16 +158,15 @@ private static IList CorrectlyOrderedWritingSystems return result; } - private static void ReorderWritingSystems(ICollection lcmList, - IEnumerable preferredOrder) + private static void CorrectWritingSystemsList(ICollection lcmList, + IEnumerable preferredOrder, + bool includeNonPreferred = true) { - var reordered = CorrectlyOrderedWritingSystems(lcmList, preferredOrder); - // TODO: Perhaps the ones added by Lcm, if any, should be considered NON-current? If so, - // then we should add a boolean param and the line above should change to: - // var reordered = - // includeNonPreferred ? - // CorrectlyOrderedWritingSystems(lcmList, preferredOrder) : - // preferredOrder.ToList(); + // TODO: Don't like these names, let's find more self-explanatory ones + var reordered = + includeNonPreferred ? + CorrectlyOrderedWritingSystems(lcmList, preferredOrder) : + preferredOrder.ToList(); // Guard against making liblcm fire WritingSystemListChanged if it doesn't need to if (reordered.Select(ws => ws.Id).SequenceEqual(lcmList.Select(ws => ws.Id))) @@ -188,10 +187,10 @@ private static void ReorderWritingSystems(LcmCache lcmCache, // Have to do both the "Current" and "All" lists, because liblcm doesn't automatically // keep their order in sync. If a ws is added to Current them it's also added to All, // but that's it. A reorder wouldn't be automatically synced up, we must do so here. - ReorderWritingSystems(wsContainer.CurrentAnalysisWritingSystems, preferredOrderAnalysis); - ReorderWritingSystems(wsContainer.AnalysisWritingSystems, preferredOrderAnalysis); - ReorderWritingSystems(wsContainer.CurrentVernacularWritingSystems, preferredOrderVernacular); - ReorderWritingSystems(wsContainer.VernacularWritingSystems, preferredOrderVernacular); + CorrectWritingSystemsList(wsContainer.AnalysisWritingSystems, preferredOrderAnalysis); + CorrectWritingSystemsList(wsContainer.VernacularWritingSystems, preferredOrderVernacular); + CorrectWritingSystemsList(wsContainer.CurrentAnalysisWritingSystems, preferredOrderAnalysis, includeNonPreferred: false); + CorrectWritingSystemsList(wsContainer.CurrentVernacularWritingSystems, preferredOrderVernacular, includeNonPreferred: false); }); } } diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index 1f5267a422..340a9a0e1f 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -83,6 +83,8 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting // fr es ... var langProject = XDocument.Parse(langprojXml).Root?.Element("LangProject"); langProject.Should().NotBeNull("LanguageProject.langproj should contain a LangProject element"); + var allAnalysisWss = SpaceSeparatedUni(langProject!, "AnalysisWss"); + var allVernWss = SpaceSeparatedUni(langProject!, "VernWss"); var curAnalysisWss = SpaceSeparatedUni(langProject!, "CurAnalysisWss"); var curVernWss = SpaceSeparatedUni(langProject!, "CurVernWss"); @@ -95,8 +97,10 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting // The requested writing systems should *also* be in the correct order. Any writing systems added // by FieldWorks should come last. - curAnalysisWss.Take(analysis.Length).Should().Equal(analysis, "analysis writing systems were in the wrong order"); - curVernWss.Take(vernacular.Length).Should().Equal(vernacular, "vernacular writing systems were in the wrong order"); + allAnalysisWss.Take(analysis.Length).Should().Equal(analysis, "analysis writing systems were in the wrong order"); + allVernWss.Take(vernacular.Length).Should().Equal(vernacular, "vernacular writing systems were in the wrong order"); + curAnalysisWss.Should().Equal(analysis, "current analysis writing systems should not contain any extras"); + curVernWss.Should().Equal(vernacular, "current vernacular writing systems should not contain any extras"); } finally { From 5b37c5d9a20a79cee804a9732d7ca842ec605d29 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 11 Sep 2026 13:32:53 +0700 Subject: [PATCH 32/60] Combine SetBranch and CommitEmpty into SwitchBranch These two methods were always called one after another, and CommitEmpty has no other reason to exist. So we combine them. New method changes branch and creates an empty commit to switch to it immediately, but can optionally be told not to switch immediately. --- .../Services/ISendReceiveService.cs | 3 +- .../Services/ProjectCreationService.cs | 12 ++----- .../FwHeadless/Services/SendReceiveHelpers.cs | 36 ++++++++++--------- .../FwHeadless/Services/SendReceiveService.cs | 9 ++--- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs index 98c9a573e9..b1835ec7f5 100644 --- a/backend/FwHeadless/Services/ISendReceiveService.cs +++ b/backend/FwHeadless/Services/ISendReceiveService.cs @@ -10,7 +10,6 @@ public interface ISendReceiveService Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode); Task PendingCommitCountBothWays(FwDataProject project, string? projectCode); Task CommitFile(string filePath, string commitMessage); - Task CommitEmpty(string folder, string commitMessage); Task InitRepo(string folder); - Task SetBranch(string folder, string branchName); + Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = ""); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 169693468a..8a2308dbc0 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -49,8 +49,8 @@ public async Task InitFwDataProject( // FLExBridge create a repo, since Clone can't establish an empty remote and // Language_Forge_Send_Receive refuses to make the *first* commit itself: // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). - // 2. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all - // of the requested writing systems. + // 2. Build fw.fwdata into that folder from the SIL.LCModel NewLangProj template, + // configured with all of the requested writing systems. // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch -- the same // genesis file, on the same branch, FieldWorks commits first. The fwdata itself is NOT // committed; Send/Receive splits it into the nested files Mercurial actually tracks @@ -68,13 +68,7 @@ public async Task InitFwDataProject( var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject); await srService.CommitFile(customPropertiesFile, InitialCommitMessage); - await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); - // BUG WORKAROUND (remove once the LanguageForgeSendReceiveActionHandler bug is fixed): that - // handler refuses to commit when doing so "could possibly create a new branch", so it won't - // establish the model-version branch itself. This empty commit (no file changes -- it only - // records the branch change from SetBranch above) creates a head on the branch so Send/Receive - // will proceed. - await srService.CommitEmpty(fwDataProject.ProjectFolder, InitialCommitMessage); + await srService.SwitchBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage); if (!pushResult.Success) throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 04b7c6f0e3..78630a0d47 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -160,17 +160,6 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", fileDir, progress); } - public static async Task CommitEmpty(string folder, string commitMessage, IProgress? progress = null) - { - using var activity = FwHeadlessActivitySource.Value.StartActivity(); - activity?.SetTag("app.folder", folder); - progress ??= new NullProgress(); - // No `hg add` and no file changes: this succeeds only because a pending branch change (from a - // preceding SetBranch) is itself a committable change in Mercurial, so it records a new head on - // that branch. A plain commit with truly nothing changed would exit 1 and throw. - await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress); - } - public static async Task InitRepo(string folder, IProgress? progress = null) { using var activity = FwHeadlessActivitySource.Value.StartActivity(); @@ -182,17 +171,32 @@ public static async Task InitRepo(string folder, IProgress? progress = null) await Task.Run(() => HgRepository.CreateOrUseExisting(folder, progress)); } - public static async Task SetBranch(string folder, string branchName, IProgress? progress = null) + public static async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "", IProgress? progress = null) { using var activity = FwHeadlessActivitySource.Value.StartActivity(); activity?.SetTag("app.branch", branchName); progress ??= new NullProgress(); // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version - // (e.g. 7500002.7000072); a clone looking for that branch reports "no such branch" if the initial - // commit landed on 'default' instead. hg records the branch for the *next* commit, so this must - // run before the first commit. Numeric/dotted branch names are accepted thanks to the fixutf8 - // extension wired into Mercurial/mercurial.ini. + // (e.g. 7500002.7000072); the initial commit of a repo needs to be on the 'default' branch + // so that Chorus can use rev 0 to compare project origin, but then we need to switch branches + // to the numbered branch. The `hg branch` command records the branch that will be used for + // the next commit, but we need to add an empty commit in order to actually create the branch, + // otherwise LfMergeBridge will refuse to create a new branch (since in the Language Forge code + // that LfMergeBridge targets, creating a new branch would be an error). This results in a new + // project being created by this process having three initial commits, rather than the two + // initial commits that it would have if created by FieldWorks Classic, but that's an acceptable + // price to pay for such a simple workaround. If LfMergeBridge is updated to be okay with creating + // new branches on its own, then this method can be passed "createBranch: false" to skip running + // the `hg commit` step that creates that empty commit. + // + // Commit message defaults to "Switching to branch {branchName}", pass `commitMessage` param to change await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress); + if (createBranch) + { + // Could run "hg log -r . --template {desc}" to reuse previous commit message if we wanted to + if (string.IsNullOrEmpty(commitMessage)) commitMessage = $"Switching to branch {branchName}"; + await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress); + } } private static string EscapeShellArg(string arg) diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs index 3ef8955a36..0c410ba1f9 100644 --- a/backend/FwHeadless/Services/SendReceiveService.cs +++ b/backend/FwHeadless/Services/SendReceiveService.cs @@ -71,18 +71,13 @@ public async Task CommitFile(string filePath, string commitMessage) await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress); } - public async Task CommitEmpty(string folder, string commitMessage) - { - await SendReceiveHelpers.CommitEmpty(folder, commitMessage, progress); - } - public async Task InitRepo(string folder) { await SendReceiveHelpers.InitRepo(folder, progress); } - public async Task SetBranch(string folder, string branchName) + public async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "") { - await SendReceiveHelpers.SetBranch(folder, branchName, progress); + await SendReceiveHelpers.SwitchBranch(folder, branchName, createBranch, commitMessage, progress); } } From 0dc17b7ef9607029ef5b002045e3321c22b9b432 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 18 Sep 2026 11:13:47 +0700 Subject: [PATCH 33/60] Explain why Chorus wants rev 0 on default branch --- .../FwHeadless/Services/ProjectCreationService.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 8a2308dbc0..d0848157f8 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -50,11 +50,14 @@ public async Task InitFwDataProject( // Language_Forge_Send_Receive refuses to make the *first* commit itself: // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). // 2. Build fw.fwdata into that folder from the SIL.LCModel NewLangProj template, - // configured with all of the requested writing systems. - // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch -- the same - // genesis file, on the same branch, FieldWorks commits first. The fwdata itself is NOT - // committed; Send/Receive splits it into the nested files Mercurial actually tracks - // (fwdata is excluded from tracking). + // configured with all of the requested writing systems, but do not commit it yet. + // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch to be + // revision 0 -- Chorus expects revision 0 to be on the default branch, because it uses + // that rev 0 commit to track a repo's history and catch when someone would try to push + // an unrelated repo (which would otherwise create two Mercurial heads). This is the + // same initial commit, with the same file, that FieldWorks creates. The fwdata itself + // is NOT committed; Send/Receive splits it into the nested files Mercurial actually + // tracks (fwdata is excluded from tracking). // 4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, // e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on // that branch, so the split data must land there while the genesis stays on 'default'. From d746810ac0a8f220206abe790a784d344ed98ee5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 18 Sep 2026 16:36:41 +0700 Subject: [PATCH 34/60] Allow specifying null for UI writing system This will let liblcm make the choice, instead of us enforcing an "en" default. Currently liblcm's default is already "en", but this would allow us to follow liblcm's lead if they change that default in the future. --- backend/FwHeadless/Routes/ProjectRoutes.cs | 4 ++-- backend/FwHeadless/Services/ProjectCreationService.cs | 6 ++---- .../Fixtures/MockFwProjectLoader.cs | 4 ++-- .../FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs | 10 +++++----- backend/LexCore/Entities/InitFwDataProjectInput.cs | 2 +- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index fefa65f5ba..b5c57a30e1 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -35,8 +35,8 @@ private static async Task> InitFwDataProject( if (invalidWs is not null) return TypedResults.Problem($"Invalid writing system code: {invalidWs}", statusCode: StatusCodes.Status400BadRequest); - if (string.IsNullOrEmpty(input.WsUi) || !IetfLanguageTag.IsValid(input.WsUi)) - return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", + if (input.WsUi is string wsUi && !IetfLanguageTag.IsValid(wsUi)) + return TypedResults.Problem($"Invalid UI writing system code: {wsUi}", statusCode: StatusCodes.Status400BadRequest); var projectCode = await projectLookupService.GetProjectCode(projectId); diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index d0848157f8..ecaa6dee3c 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -30,14 +30,12 @@ public async Task InitFwDataProject( string projectCode, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, - string uiWritingSystem) + string? uiWritingSystem = null) { if (vernacularWritingSystems.Count == 0) throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); if (analysisWritingSystems.Count == 0) throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); - if (string.IsNullOrEmpty(uiWritingSystem)) - throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem)); // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. if (!syncHostedService.TryStartProjectCreation(projectId)) @@ -93,7 +91,7 @@ private void BuildFromTemplate( FwDataProject fwDataProject, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, - string uiWs) + string? uiWs = null) { // NewProject copies the SIL.LCModel template, configured with all of the requested writing // systems (the first of each list is the default of that type), and returns a loaded cache; diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs index 4123cdd39f..ceb72ef36b 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs @@ -19,7 +19,7 @@ public override LcmCache LoadCache(FwDataProject project) return cache; } - public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") + public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string? uiWs = null) { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -28,7 +28,7 @@ public override LcmCache NewProject(FwDataProject project, string analysisWs, st new SimpleProjectId(BackendProviderType.kMemoryOnly, Path.GetFullPath(project.FilePath)), analysisWs, vernacularWs, - uiWs, + uiWs ?? "en", new LfLcmUi(progress.SynchronizeInvoke), lcmDirectories, new LcmSettings()); diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index 0fe37b86c4..e600e5b2c9 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -21,13 +21,13 @@ public interface IProjectLoader /// LcmCache LoadCache(FwDataProject project); - LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en"); + LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string? uiWs = null); /// /// Like the single-WS overload, but the new project also gets every analysis/vernacular writing /// system in the given lists. The first item of each list is the default of that type. /// - LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, string uiWs = "en"); + LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, string? uiWs = null); } public class ProjectLoader(IOptions config) : IProjectLoader @@ -83,7 +83,7 @@ public virtual LcmCache LoadCache(FwDataProject project) return cache; } - public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") + public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string? uiWs = null) { return NewProject(project, [analysisWs], [vernacularWs], uiWs); } @@ -91,7 +91,7 @@ public virtual LcmCache NewProject(FwDataProject project, string analysisWs, str public virtual LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, - string uiWs = "en") + string? uiWs = null) { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -133,7 +133,7 @@ private static void NewProject(IThreadedProgress progress, ISynchronizeInvoke syncInvoke, CoreWritingSystemDefinition analysisWs, CoreWritingSystemDefinition vernacularWs, - string uiWs, + string? uiWs, HashSet additionalAnalysisWss, HashSet additionalVernacularWss) { diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs index ba2e2b72a9..277e779cc4 100644 --- a/backend/LexCore/Entities/InitFwDataProjectInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -8,4 +8,4 @@ namespace LexCore.Entities; public record InitFwDataProjectInput( IReadOnlyList WsVernacular, IReadOnlyList WsAnalysis, - string WsUi); + string? WsUi); From b7c75adb3ac9bd172daf88dba823958d3e8c8f88 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 18 Sep 2026 16:37:55 +0700 Subject: [PATCH 35/60] Revert "Allow specifying null for UI writing system" This reverts commit d746810ac0a8f220206abe790a784d344ed98ee5. Decided not to do this, for now, and stick with "en" as default UI WS. --- backend/FwHeadless/Routes/ProjectRoutes.cs | 4 ++-- backend/FwHeadless/Services/ProjectCreationService.cs | 6 ++++-- .../Fixtures/MockFwProjectLoader.cs | 4 ++-- .../FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs | 10 +++++----- backend/LexCore/Entities/InitFwDataProjectInput.cs | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index b5c57a30e1..fefa65f5ba 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -35,8 +35,8 @@ private static async Task> InitFwDataProject( if (invalidWs is not null) return TypedResults.Problem($"Invalid writing system code: {invalidWs}", statusCode: StatusCodes.Status400BadRequest); - if (input.WsUi is string wsUi && !IetfLanguageTag.IsValid(wsUi)) - return TypedResults.Problem($"Invalid UI writing system code: {wsUi}", + if (string.IsNullOrEmpty(input.WsUi) || !IetfLanguageTag.IsValid(input.WsUi)) + return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", statusCode: StatusCodes.Status400BadRequest); var projectCode = await projectLookupService.GetProjectCode(projectId); diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index ecaa6dee3c..d0848157f8 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -30,12 +30,14 @@ public async Task InitFwDataProject( string projectCode, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, - string? uiWritingSystem = null) + string uiWritingSystem) { if (vernacularWritingSystems.Count == 0) throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); if (analysisWritingSystems.Count == 0) throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); + if (string.IsNullOrEmpty(uiWritingSystem)) + throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem)); // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. if (!syncHostedService.TryStartProjectCreation(projectId)) @@ -91,7 +93,7 @@ private void BuildFromTemplate( FwDataProject fwDataProject, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, - string? uiWs = null) + string uiWs) { // NewProject copies the SIL.LCModel template, configured with all of the requested writing // systems (the first of each list is the default of that type), and returns a loaded cache; diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs index ceb72ef36b..4123cdd39f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs @@ -19,7 +19,7 @@ public override LcmCache LoadCache(FwDataProject project) return cache; } - public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string? uiWs = null) + public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -28,7 +28,7 @@ public override LcmCache NewProject(FwDataProject project, string analysisWs, st new SimpleProjectId(BackendProviderType.kMemoryOnly, Path.GetFullPath(project.FilePath)), analysisWs, vernacularWs, - uiWs ?? "en", + uiWs, new LfLcmUi(progress.SynchronizeInvoke), lcmDirectories, new LcmSettings()); diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index e600e5b2c9..0fe37b86c4 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -21,13 +21,13 @@ public interface IProjectLoader /// LcmCache LoadCache(FwDataProject project); - LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string? uiWs = null); + LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en"); /// /// Like the single-WS overload, but the new project also gets every analysis/vernacular writing /// system in the given lists. The first item of each list is the default of that type. /// - LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, string? uiWs = null); + LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, string uiWs = "en"); } public class ProjectLoader(IOptions config) : IProjectLoader @@ -83,7 +83,7 @@ public virtual LcmCache LoadCache(FwDataProject project) return cache; } - public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string? uiWs = null) + public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") { return NewProject(project, [analysisWs], [vernacularWs], uiWs); } @@ -91,7 +91,7 @@ public virtual LcmCache NewProject(FwDataProject project, string analysisWs, str public virtual LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, - string? uiWs = null) + string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -133,7 +133,7 @@ private static void NewProject(IThreadedProgress progress, ISynchronizeInvoke syncInvoke, CoreWritingSystemDefinition analysisWs, CoreWritingSystemDefinition vernacularWs, - string? uiWs, + string uiWs, HashSet additionalAnalysisWss, HashSet additionalVernacularWss) { diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs index 277e779cc4..ba2e2b72a9 100644 --- a/backend/LexCore/Entities/InitFwDataProjectInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -8,4 +8,4 @@ namespace LexCore.Entities; public record InitFwDataProjectInput( IReadOnlyList WsVernacular, IReadOnlyList WsAnalysis, - string? WsUi); + string WsUi); From b808a83db70506d6aee6f23884b951f4fbe050be Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 21 Sep 2026 13:09:18 +0700 Subject: [PATCH 36/60] Rename BuildFromTemplate method We don't actually use a template in our code; that's a liblcm internal detail that we shouldn't be worrying about. --- backend/FwHeadless/Services/ProjectCreationService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index d0848157f8..58344ad82a 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -64,7 +64,7 @@ public async Task InitFwDataProject( // 5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push. await srService.InitRepo(fwDataProject.ProjectFolder); - BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); + BuildFwDataFile(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); FixLinkedFilesRootDirSeparator(fwDataProject); AssertModelVersionMatches(fwDataProject); @@ -89,7 +89,7 @@ public async Task InitFwDataProject( } } - private void BuildFromTemplate( + private void BuildFwDataFile( FwDataProject fwDataProject, IReadOnlyList vernacularWritingSystems, IReadOnlyList analysisWritingSystems, From 3cb538450b085bcbf43e3cbe49c74f02ab63610d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 21 Sep 2026 14:09:44 +0700 Subject: [PATCH 37/60] Consistency edits Add another OTel tag for consistency with other Mercurial helpers. Also make wording of various comments more consistent. --- backend/FwHeadless/Services/ProjectCreationService.cs | 5 ----- backend/FwHeadless/Services/SendReceiveHelpers.cs | 1 + backend/FwHeadless/Services/SyncHostedService.cs | 7 ++++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 58344ad82a..68a366428b 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -144,11 +144,6 @@ private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) File.Move(tempPath, sourcePath, overwrite: true); } - /// - /// Writes the genesis FLExProject.CustomProperties file into the repo root: an empty custom-property - /// list (just an <AdditionalFields /> root). Uses SIL's canonical XML settings so the bytes match - /// exactly what the first Send/Receive's project splitter regenerates, leaving no spurious diff. - /// private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) { var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename); diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 78630a0d47..7d1974ad6e 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -174,6 +174,7 @@ public static async Task InitRepo(string folder, IProgress? progress = null) public static async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "", IProgress? progress = null) { using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); activity?.SetTag("app.branch", branchName); progress ??= new NullProgress(); // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index d01b878593..8983dda87e 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -20,12 +20,13 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsToSync = Channel.CreateUnbounded(); private readonly ConcurrentDictionary> _projectsQueuedOrRunning = new(); - // Projects currently being created from a template. Tracked here (not just in ProjectCreationService) + // Projects currently being created and populated with an initial .fwdata file. Tracked here (not just in ProjectCreationService) // so a sync job can't be queued for a project mid-creation and race on the same fw/ folder and repo. private readonly ConcurrentDictionary _projectsBeingCreated = new(); // Serialises the check-then-add across the two dictionaries above, so a creation and a sync can't // both slip through by interleaving their checks. + // This requires that SyncHostedService be a singleton, of course private readonly Lock _reservationLock = new(); protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -66,7 +67,7 @@ public virtual bool IsJobQueuedOrRunning(Guid projectId) } /// - /// Reserves a project for creation-from-template, blocking a concurrent creation or sync of the + /// Reserves a project for creation-and-population, blocking a concurrent creation or sync of the /// same project. Returns false if a sync is already queued/running or another creation is in flight. /// Pair with in a finally block. /// @@ -96,7 +97,7 @@ public bool QueueJob(Guid projectId) bool addedToQueue; lock (_reservationLock) { - //don't sync a project while it's still being created from a template (would race on the repo) + //don't sync a project while it's still being created and populated with initial .fwdata (would race on the repo) if (_projectsBeingCreated.ContainsKey(projectId)) { logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); From adabe5b998db78184069dd1e8e5487d84ba5d4bf Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 21 Sep 2026 16:09:00 +0700 Subject: [PATCH 38/60] Add new test checking projectOrigin is admin-only Currently projectOrigin is effectively admin-only because the entire API call is admin-only. But at some point we will remove `[AdminRequired]` and then we want the projectOrigin parameter to still be restricted to only site admins even when the rest of the API is opened up. Also have the projectOrigin test check that LanguageForgeNonSR is parsed case-insensitively, because we expect to actually use that one. --- .../InitFwDataProjectValidationTests.cs | 1 + .../InitFwDataProjectTests.cs | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index d1db78622f..c549b0796e 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -92,6 +92,7 @@ public async Task Ignores_an_empty_project_origin_without_requiring_admin() [InlineData("Migrated", ProjectMigrationStatus.Migrated)] [InlineData("migrating", ProjectMigrationStatus.Migrating)] [InlineData("PUBLICREDMINE", ProjectMigrationStatus.PublicRedmine)] + [InlineData("languageforgenonsr", ProjectMigrationStatus.LanguageForgeNonSR)] public void Project_origin_parses_case_insensitively(string input, ProjectMigrationStatus expected) { ProjectController.TryParseProjectOrigin(input, out var origin).Should().BeTrue(); diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index 340a9a0e1f..a574160307 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -1,8 +1,10 @@ using System.Net; using System.Net.Http.Json; +using System.Text.Json; using System.Text.Json.Nodes; using System.Xml.Linq; using FluentAssertions; +using LexCore.Entities; using Testing.ApiTests; using Testing.Fixtures; using Testing.Services; @@ -108,6 +110,57 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting } } + /// + /// Only admins may say where a project came from, so a non-admin passing projectOrigin + /// must be rejected. Today it's [AdminRequired] on the endpoint that rejects it (403); once this + /// endpoint opens up to non-admins and that attribute goes away, the controller's own + /// AssertIsAdmin() will reject it (401). Either way the request must fail and no project may be + /// created, which is what this test pins down across that change. + /// To check it isn't passing for the wrong reason, temporarily delete [AdminRequired] from + /// ProjectController.InitFwDataProject, rebuild, and re-run: it must still pass. + /// + [Fact] + public async Task InitFwDataProject_RejectsProjectOriginFromANonAdmin() + { + var managerApiTester = new ApiTestBase(); + await managerApiTester.LoginAs(ManagerAuth.Username, ManagerAuth.Password); + + // Everything except projectOrigin is valid, so a 400 from input validation can't be what + // rejects this, and the origin itself is a real ProjectMigrationStatus name. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await managerApiTester.HttpClient.PostAsync( + $"{managerApiTester.BaseUrl}/api/project/initFwDataProject" + + $"?code={code}&wsVernacular=fr&projectOrigin={nameof(ProjectMigrationStatus.LanguageForgeNonSR)}", null); + + if (response.StatusCode == HttpStatusCode.InternalServerError) + { + // In developer mode, a thrown System.UnauthorizedAccessException exception results in an HTTP 500 + // instead of a 4xx code, with a JSON structure that includes `"title":"System.UnauthorizedAccessException"`. + // We check for that special case here; any non-JSON or JSON that has a different title would indicate a + // different problem and be a test failure. + JsonNode? problemDetails = null; + try + { + var body = await response.Content.ReadAsStringAsync(); + problemDetails = JsonNode.Parse(body); + } + catch (JsonException) + { + // Do nothing here, the Should().NotBeNull() check below handles this + } + var failureReasonMsg = "HTTP 500 should have been caused by developer page printing " + + "a traceback due to " + nameof(UnauthorizedAccessException); + problemDetails.Should().NotBeNull(failureReasonMsg); + problemDetails?["title"]?.ToString().Should().EndWithEquivalentOf( + nameof(UnauthorizedAccessException), failureReasonMsg); + } + else + { + response.StatusCode.Should().BeOneOf([HttpStatusCode.Forbidden, HttpStatusCode.Unauthorized], + "a non-admin may not set projectOrigin; body: {0}", await response.Content.ReadAsStringAsync()); + } + } + [Fact] public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() { From 341bc346068abeaf068c14a58463a087ad08eac6 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 11:34:23 +0700 Subject: [PATCH 39/60] Better documentation phrasing in ProjectController --- backend/LexBoxApi/Controllers/ProjectController.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 5360d60d69..c050758360 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -31,10 +31,14 @@ public class ProjectController( /// /// Admin-only: create a new FLEx project whose repo is populated with a template .fwdata /// (from the SIL.LCModel package) configured for the requested writing systems. + /// + /// At some future date this API may be opened up to trusted users, but the projectOrigin + /// parameter will remain admin-only/ /// /// Project code for the new project. /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. + /// Optional user interface writing system id; defaults to "en" if not given. /// Optional display name; defaults to the code. /// Optional to record as the project's origin; admin-only. [HttpPost("initFwDataProject")] @@ -84,8 +88,8 @@ public async Task> InitFwDataProject( OrgId: null), projectOrigin: origin); - // Saga: the project row + empty repo now exist. Have FwHeadless populate the repo with the - // template .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs + // At this point, the project row and empty repo exist. Have FwHeadless populate the repo with + // the initial .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs // on a fresh token so a client disconnect can't skip it. (HttpStatusCode statusCode, string? error) result; try From d73ba6645bf5e2034131b9e0f092d78d09d043b2 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 11:38:34 +0700 Subject: [PATCH 40/60] Typo fix --- backend/LexBoxApi/Controllers/ProjectController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index c050758360..1fe17acc5e 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -33,7 +33,7 @@ public class ProjectController( /// (from the SIL.LCModel package) configured for the requested writing systems. /// /// At some future date this API may be opened up to trusted users, but the projectOrigin - /// parameter will remain admin-only/ + /// parameter will remain admin-only. /// /// Project code for the new project. /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. From 04cf3fde04c0ff09d2f758d32615829818a7ada7 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 14:32:43 +0700 Subject: [PATCH 41/60] Better comment wording --- backend/LexBoxApi/Services/FwHeadlessClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 252801beb7..9b2c75a583 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -200,7 +200,7 @@ public async Task UnblockProject(Guid projectId) } /// - /// Populates a newly-created (empty) project's repo with a template .fwdata configured for the + /// Initializes a newly-created (empty) project's repo with a new .fwdata configured for the /// given writing systems. Runs inline (a new empty project is small). Returns a null Error on /// success; on failure returns the FwHeadless status code and error body so the caller can /// propagate the right status (e.g. surface a 400 rather than flattening it to 500). From a7e203dd238fdc2eaf33a7287cc79a7dd8571bc1 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 14:36:03 +0700 Subject: [PATCH 42/60] Better comment about integration tests --- backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index a574160307..252750f97f 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -17,7 +17,7 @@ namespace Testing.SyncReverseProxy; /// End-to-end coverage for the admin "init fwdata project" API. This is the one path that /// can't be unit-tested: LexBox creates an empty hg repo and FwHeadless does the first push of a /// template-built .fwdata into it (via LfMergeBridge/Chorus against the real hgweb). Requires the -/// lexbox stack, so it only runs in CI. +/// lexbox stack to be up and running, so only run it in CI or with `task test:integration`. /// [Trait("Category", "Integration")] public class InitFwDataProjectTests : IClassFixture From f9e036a20fbdc889997c5a787670eb362e89a02b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 16:35:57 +0700 Subject: [PATCH 43/60] Do not change current directory in FwHeadless No longer needed to work around Chorus plugin-finding bug now that LibChorus version 6.0.0-beta0074 is out. --- backend/FwHeadless/Program.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index cdc459031b..ed832e53e9 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -12,17 +12,6 @@ using WebServiceDefaults; using AppVersion = LexCore.AppVersion; -// Chorus discovers its file-type handler plugins with `new DirectoryCatalog(".", "*-ChorusPlugin.dll")`, -// which scans the *current working directory* -- not the app base dir -- and WebApplication.CreateBuilder -// does not change CWD. When launched from anywhere other than the folder holding the co-published -// LibFLExBridge-ChorusPlugin.dll (e.g. `dotnet run`, whose CWD is the project dir), that plugin isn't -// found. Without it, FieldWorks extensions like `.list` fall back to Chorus's 1 MB LargeFileFilter -// default, so the >1 MB SemanticDomainList.list is silently filtered out of a new project's first push -// (and later merges lose their FieldWorks handlers too). Pin CWD to the base dir so the plugin is always -// discovered. Safe here: hg runs with explicit working dirs and TemplatesFolder resolves against -// AppContext.BaseDirectory already. -Environment.CurrentDirectory = AppContext.BaseDirectory; - var builder = WebApplication.CreateBuilder(args); // Add services to the container. From a81fced371de89b5ec0c94bd0cea1937d4bff0e1 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 16:47:25 +0700 Subject: [PATCH 44/60] Add test against possible Chorus regression This test addition should fail if Chorus ever starts failing to find its file-handler plugins again, because the symptom that would occur if that happened (the semantic domain list file would not be pushed) is exactly what this extra text check looks for. --- .../SyncReverseProxy/InitFwDataProjectTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index 252750f97f..42b5e7f464 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -90,6 +90,21 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting var curAnalysisWss = SpaceSeparatedUni(langProject!, "CurAnalysisWss"); var curVernWss = SpaceSeparatedUni(langProject!, "CurVernWss"); + // 4. The largest file survived the push. Chorus's LargeFileFilter caps a file at 1 MB unless + // some loaded file-type handler raises the cap for that extension, and the handler that knows + // about .list ships in LibFLExBridge-ChorusPlugin.dll. If plugin discovery ever regresses + // (sillsdev/chorus#393), .list silently falls back to the 1 MB default and Chorus *forgets* + // SemanticDomainList.list instead of pushing it -- no error, no failed send/receive, just a + // project that looks fine here and is missing its semantic domains when FLEx opens it. + // Every other assertion in this test reads a small file, so they all stay green through + // that failure; this is the only one that catches it. + var semDomResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/Linguistics/Lexicon/SemanticDomainList.list"); + semDomResponse.StatusCode.Should().Be(HttpStatusCode.OK, + "SemanticDomainList.list should have been pushed; anything else (404 in particular) means " + + "Chorus filtered it out as oversized, i.e. the FieldWorks file-type handlers never loaded"); + var semDomSize = (await semDomResponse.Content.ReadAsByteArrayAsync()).Length; + // The requested writing systems should be current. FieldWorks may add its own defaults (e.g. // "en" as an analysis WS), so assert each requested code is present rather than exact equality. foreach (var ws in analysis) From f659ba7ddbaa962bae0ab565fc254fcc4d21b1f4 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 17:00:54 +0700 Subject: [PATCH 45/60] Do not cancel project creation when the request is aborted An action's CancellationToken parameter binds to HttpContext.RequestAborted, which trips whenever the caller hangs up or its client times out. FwHeadless's initFwDataProject takes no cancellation token of its own, so it runs creation through to completion regardless. Aborting our wait therefore told us nothing about whether the project was created, while the catch block tore it down -- deleting the project row and the hg repo that FwHeadless was still pushing into. The 409 guard in CleanupFailedProjectCreation does not close the window, because the creation reservation is released as soon as creation returns. Drop the parameter rather than substituting another token: HttpClient.Timeout already bounds the call, and an unused token in scope invites someone to thread it back in. Compensate only when FwHeadless has answered with a failure; on no answer at all, log and leave the project in place, since an orphaned empty project is recoverable by an admin and a half-deleted one is not. Co-Authored-By: Claude Opus 5 (1M context) --- .../Controllers/ProjectController.cs | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 1fe17acc5e..4664429862 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -53,8 +53,7 @@ public async Task> InitFwDataProject( [FromQuery] string[]? wsAnalysis = null, string wsUi = "en", string? name = null, - string? projectOrigin = null, - CancellationToken cancellationToken = default) + string? projectOrigin = null) { if (wsVernacular is null || wsVernacular.Length == 0) return Problem("At least one vernacular writing system is required", statusCode: StatusCodes.Status400BadRequest); @@ -89,20 +88,37 @@ public async Task> InitFwDataProject( projectOrigin: origin); // At this point, the project row and empty repo exist. Have FwHeadless populate the repo with - // the initial .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs - // on a fresh token so a client disconnect can't skip it. + // the initial .fwdata; if that fails, compensate by tearing the project back down. + // + // Deliberately no CancellationToken here. An action's CancellationToken parameter binds to + // HttpContext.RequestAborted, which trips whenever the caller hangs up or its client times out -- + // routine for an operation this slow. FwHeadless's initFwDataProject takes no cancellation token + // of its own, so it runs the creation through to completion regardless of what happens to this + // connection. Cancelling our wait would therefore tell us nothing about whether the project was + // created, while making us tear down one that FwHeadless is still building. HttpClient.Timeout + // (5 minutes, set in LexBoxKernel) is what bounds this call. (HttpStatusCode statusCode, string? error) result; try { - result = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, cancellationToken); + result = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi); } - catch + catch (Exception ex) { - await CleanupFailedCreation(projectId, code); + // FwHeadless never answered, so we cannot tell a project that was never created from one + // still being built -- or one finished just as we gave up. Tearing it down here would race + // FwHeadless writing into the very repo we would delete. An orphaned empty project is + // recoverable by an admin; a project deleted out from under an in-flight creation is not. + logger.LogError(ex, + "initFwDataProject did not return for project {ProjectId} ({Code}); leaving it in place because FwHeadless may still be creating it. It may need manual cleanup", + projectId, + code); throw; } if (result.error is not null) { + // FwHeadless answered and the answer was failure, so creation is definitively not running + // (it releases its own reservation and cleans up locally before responding). Only here do we + // know enough to compensate. await CleanupFailedCreation(projectId, code); // Surface a bad request from FwHeadless (e.g. an invalid writing-system tag) as 400, not 500. var statusCode = result.statusCode == HttpStatusCode.BadRequest From cb2cc880069dc8118f89b6d7074ae15d49e5d99a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 17:11:30 +0700 Subject: [PATCH 46/60] Delete leftover bit of code --- backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index 42b5e7f464..d99dd1dce2 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -103,7 +103,6 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting semDomResponse.StatusCode.Should().Be(HttpStatusCode.OK, "SemanticDomainList.list should have been pushed; anything else (404 in particular) means " + "Chorus filtered it out as oversized, i.e. the FieldWorks file-type handlers never loaded"); - var semDomSize = (await semDomResponse.Content.ReadAsByteArrayAsync()).Length; // The requested writing systems should be current. FieldWorks may add its own defaults (e.g. // "en" as an analysis WS), so assert each requested code is present rather than exact equality. From 69176e08ff8fcb9001449adbf71ab73390d8a5ca Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 17:26:32 +0700 Subject: [PATCH 47/60] Add FwHeadless endpoints for project creation status Creation is a long-running operation with no way to ask how it is going, so a caller has to hold a request open for the whole thing. Give it the same polling shape the merge routes already have: GET /api/project/creation-status (mirrors /api/merge/status) GET /api/project/await-creation-finished (mirrors /api/merge/await-finished) The creation reservation in SyncHostedService now carries a TaskCompletionSource, exactly as the sync reservation does, so ProjectCreationService can publish its outcome to anyone waiting. Results are cached for 5 minutes rather than the sync path's 30 seconds, because a caller whose own request timed out partway through needs to be able to come back later and learn how it ended. When nothing is in flight and no result is remembered, the status is derived from durable state (the .fwdata is on disk after a success, and the project folder is deleted after a failure), so it survives a FwHeadless restart. Giving up on the await reports TimedOutAwaitingCreation rather than an error: creation takes no cancellation token, so abandoning the wait says nothing about whether the project was created. Nothing consumes these yet; the POST remains blocking. Co-Authored-By: Claude Opus 5 (1M context) --- backend/FwHeadless/Routes/ProjectRoutes.cs | 93 +++++++++++++++++++ .../Services/ProjectCreationService.cs | 10 +- .../FwHeadless/Services/SyncHostedService.cs | 46 ++++++++- backend/LexCore/Sync/ProjectCreationStatus.cs | 43 +++++++++ ...ncHostedServiceCreationReservationTests.cs | 66 ++++++++++++- 5 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 backend/LexCore/Sync/ProjectCreationStatus.cs diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index fefa65f5ba..c84501f662 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -1,6 +1,9 @@ +using System.Diagnostics; using FwHeadless.Services; using LexCore.Entities; +using LexCore.Sync; using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.Extensions.Options; using SIL.WritingSystems; namespace FwHeadless.Routes; @@ -11,6 +14,8 @@ public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication ap { var group = app.MapGroup("/api/project"); group.MapPost("/initFwDataProject", InitFwDataProject); + group.MapGet("/creation-status", GetCreationStatus); + group.MapGet("/await-creation-finished", AwaitCreationFinished); return group; } @@ -50,4 +55,92 @@ await projectCreationService.InitFwDataProject( projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi); return TypedResults.Ok(); } + + // Instantaneous creation status, for a caller that wants to poll. Mirrors /api/merge/status. + private static async Task, NotFound>> GetCreationStatus( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.project_id", projectId); + + var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config); + if (status is null) + { + activity?.SetStatus(ActivityStatusCode.Error, "Project not found"); + return TypedResults.NotFound(); + } + activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString()); + return TypedResults.Ok(status); + } + + // Long-poll until an in-flight creation finishes. Mirrors /api/merge/await-finished: giving up on the + // wait is reported as a status, not an error, because the creation itself keeps running either way. + private static async Task, NotFound>> AwaitCreationFinished( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config, + CancellationToken cancellationToken) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.project_id", projectId); + + try + { + var result = await syncHostedService.AwaitCreationFinished(projectId, cancellationToken); + if (result is not null) + { + activity?.SetStatus(ActivityStatusCode.Ok, result.Status.ToString()); + return TypedResults.Ok(result); + } + } + catch (OperationCanceledException) + { + // The caller's wait was cancelled; the creation was not (it takes no cancellation token). + // Report that distinctly so a caller can't read a timeout as a failed creation. + activity?.SetStatus(ActivityStatusCode.Unset, "Timed out awaiting creation"); + return TypedResults.Ok(ProjectCreationStatus.TimedOutAwaitingCreation); + } + + // Nothing in flight and nothing remembered, so fall back to what the project itself tells us. + var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config); + if (status is null) + { + activity?.SetStatus(ActivityStatusCode.Error, "Project not found"); + return TypedResults.NotFound(); + } + activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString()); + return TypedResults.Ok(status); + } + + /// + /// Creation status for a project, or null when LexBox has no such project. In-memory state wins + /// when we have it; otherwise the status is derived from durable state, so it survives a FwHeadless + /// restart and a result that has aged out of the cache. + /// + private static async Task DetermineCreationStatus( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config) + { + if (syncHostedService.IsProjectBeingCreated(projectId)) return ProjectCreationStatus.Creating; + var recent = syncHostedService.TryGetRecentCreationResult(projectId); + if (recent is not null) return recent; + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) return null; + + // A successful creation leaves the .fwdata on disk; a failed one deletes the project folder + // (ProjectCreationService.CleanupLocalProject). Note this says "project data is here", not "this + // process created it" -- a project cloned by an ordinary sync looks the same, which is the right + // answer for a caller asking whether the project is ready to use. + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + return File.Exists(fwDataProject.FilePath) + ? ProjectCreationStatus.Created + : ProjectCreationStatus.NotCreated; + } } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 68a366428b..8f36c3d83a 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -3,6 +3,7 @@ using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.LcmUtils; using LexCore.Exceptions; +using LexCore.Sync; using Microsoft.Extensions.Options; using SIL.Xml; @@ -43,6 +44,9 @@ public async Task InitFwDataProject( if (!syncHostedService.TryStartProjectCreation(projectId)) throw new ProjectSyncInProgressException(projectId); var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + // Published to status pollers in the finally below. Pre-set to a failure so an exit path that + // somehow reaches neither the success line nor the catch still releases pollers with an answer. + var outcome = ProjectCreationStatus.Failed("Project creation ended without reporting a result"); try { // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/ @@ -77,15 +81,17 @@ public async Task InitFwDataProject( throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId); + outcome = ProjectCreationStatus.Created; } - catch + catch (Exception ex) { + outcome = ProjectCreationStatus.Failed(ex.Message); CleanupLocalProject(fwDataProject); throw; } finally { - syncHostedService.EndProjectCreation(projectId); + syncHostedService.EndProjectCreation(projectId, outcome); } } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 8983dda87e..0d6f944348 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -22,7 +22,8 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsBeingCreated = new(); + // The TaskCompletionSource carries the outcome to anyone long-polling AwaitCreationFinished. + private readonly ConcurrentDictionary> _projectsBeingCreated = new(); // Serialises the check-then-add across the two dictionaries above, so a creation and a sync can't // both slip through by interleaving their checks. @@ -76,15 +77,28 @@ public bool TryStartProjectCreation(Guid projectId) lock (_reservationLock) { if (_projectsQueuedOrRunning.ContainsKey(projectId)) return false; - return _projectsBeingCreated.TryAdd(projectId, 0); + // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on + // the thread that finished the creation. + return _projectsBeingCreated.TryAdd(projectId, + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); } } - public void EndProjectCreation(Guid projectId) + /// + /// Releases the reservation and publishes the outcome to pollers. Must be called on every exit path + /// from a creation, or an AwaitCreationFinished caller waits until its own token trips. + /// + public void EndProjectCreation(Guid projectId, ProjectCreationStatus result) { - _projectsBeingCreated.TryRemove(projectId, out _); + // Cache before removing the reservation, so a poller that sees "not in flight" always finds the + // result waiting for it rather than a gap. + CacheRecentCreationResult(projectId, result); + _projectsBeingCreated.TryRemove(projectId, out var tcs); + tcs?.TrySetResult(result); } + public bool IsProjectBeingCreated(Guid projectId) => _projectsBeingCreated.ContainsKey(projectId); + public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) { if (_projectsQueuedOrRunning.TryGetValue(projectId, out var tcs)) @@ -92,6 +106,30 @@ public void EndProjectCreation(Guid projectId) return TryGetRecentSyncResult(projectId); } + /// + /// Waits for an in-flight creation to finish. Returns the recent result if one just finished, or + /// null when this FwHeadless has no record of a creation for the project (the caller should then + /// derive the status from durable state instead). + /// + public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken) + { + if (_projectsBeingCreated.TryGetValue(projectId, out var tcs)) + return await tcs.Task.WaitAsync(cancellationToken); + return TryGetRecentCreationResult(projectId); + } + + public ProjectCreationStatus? TryGetRecentCreationResult(Guid projectId) + { + return memoryCache.Get($"CreationResult|{projectId}"); + } + + private void CacheRecentCreationResult(Guid projectId, ProjectCreationStatus result) + { + // Longer than the sync equivalent's 30s: a caller whose own request timed out partway through a + // creation should still be able to come back and learn how it ended. + memoryCache.Set($"CreationResult|{projectId}", result, TimeSpan.FromMinutes(5)); + } + public bool QueueJob(Guid projectId) { bool addedToQueue; diff --git a/backend/LexCore/Sync/ProjectCreationStatus.cs b/backend/LexCore/Sync/ProjectCreationStatus.cs new file mode 100644 index 0000000000..67260ec19b --- /dev/null +++ b/backend/LexCore/Sync/ProjectCreationStatus.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; + +namespace LexCore.Sync; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ProjectCreationStatusEnum +{ + /// No creation is in flight and no project data exists in FwHeadless for this project. + NotCreated, + /// A creation is running right now. + Creating, + /// Project data exists in FwHeadless, i.e. a creation finished successfully. + Created, + /// A creation was attempted and failed. says why. + CreationFailed, + /// + /// The caller's await gave up before creation finished. The creation itself is unaffected and is + /// still running -- poll again rather than treating this as a failure. + /// + TimedOutAwaitingCreation, +} + +/// +/// Status of a FwHeadless project creation, for callers that want to poll (or long-poll) instead of +/// holding a request open for the whole operation. Mirrors , which does +/// the same job for merges. +/// +public record ProjectCreationStatus( + [property: JsonConverter(typeof(JsonStringEnumConverter))] + ProjectCreationStatusEnum Status, + string? Error = null) +{ + public static ProjectCreationStatus NotCreated => new(ProjectCreationStatusEnum.NotCreated); + public static ProjectCreationStatus Creating => new(ProjectCreationStatusEnum.Creating); + public static ProjectCreationStatus Created => new(ProjectCreationStatusEnum.Created); + public static ProjectCreationStatus TimedOutAwaitingCreation => new(ProjectCreationStatusEnum.TimedOutAwaitingCreation); + + public static ProjectCreationStatus Failed(string error) => new(ProjectCreationStatusEnum.CreationFailed, error); + + /// True once creation has stopped running, whether it succeeded or failed. + [JsonIgnore] + public bool IsFinished => Status is ProjectCreationStatusEnum.Created or ProjectCreationStatusEnum.CreationFailed; +} diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs index 1846cebb08..12cad3cd9e 100644 --- a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -1,4 +1,5 @@ using FwHeadless.Services; +using LexCore.Sync; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging.Abstractions; @@ -10,6 +11,9 @@ namespace Testing.FwHeadless; /// public class SyncHostedServiceCreationReservationTests { + /// Bounded so a regression that never completes the wait fails the test instead of hanging it. + private static CancellationToken TestTimeout => new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token; + private static SyncHostedService NewService() => new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); @@ -23,7 +27,7 @@ public void TryStartProjectCreation_blocks_a_second_concurrent_creation_and_is_r svc.TryStartProjectCreation(projectId).Should().BeFalse("a creation is already in flight for this project"); svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); - svc.EndProjectCreation(projectId); + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); svc.TryStartProjectCreation(projectId).Should().BeTrue("the reservation is released and reusable"); } @@ -37,7 +41,7 @@ public void QueueJob_is_refused_while_a_project_is_being_created() svc.TryStartProjectCreation(projectId).Should().BeTrue(); svc.QueueJob(projectId).Should().BeFalse("a sync must not race a project that's still being created"); - svc.EndProjectCreation(projectId); + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); svc.QueueJob(projectId).Should().BeTrue("syncing is allowed once creation has finished"); } @@ -50,4 +54,62 @@ public void A_queued_sync_blocks_a_creation() svc.QueueJob(projectId).Should().BeTrue(); svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); } + + [Fact] + public async Task AwaitCreationFinished_returns_the_outcome_the_creation_reported() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + + var awaiting = svc.AwaitCreationFinished(projectId, TestTimeout); + awaiting.IsCompleted.Should().BeFalse("the creation is still in flight"); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Failed("the push failed")); + + var result = await awaiting; + result.Should().NotBeNull(); + result!.Status.Should().Be(ProjectCreationStatusEnum.CreationFailed); + result.Error.Should().Be("the push failed"); + } + + [Fact] + public async Task AwaitCreationFinished_returns_a_result_that_finished_before_the_caller_asked() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + + // The caller shows up late (e.g. its own request timed out and it came back to ask). It must + // still learn the outcome rather than being told nothing ever happened. + var result = await svc.AwaitCreationFinished(projectId, TestTimeout); + result.Should().NotBeNull(); + result!.Status.Should().Be(ProjectCreationStatusEnum.Created); + } + + [Fact] + public async Task AwaitCreationFinished_returns_null_when_no_creation_is_known() + { + var svc = NewService(); + + var result = await svc.AwaitCreationFinished(Guid.NewGuid(), TestTimeout); + result.Should().BeNull("the caller should fall back to deriving the status from the project itself"); + } + + [Fact] + public async Task AwaitCreationFinished_honours_the_caller_token_without_ending_the_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + + using var cts = new CancellationTokenSource(); + var awaiting = svc.AwaitCreationFinished(projectId, cts.Token); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => awaiting); + svc.IsProjectBeingCreated(projectId).Should().BeTrue("giving up on the wait must not cancel the creation"); + } } From f6f1fe799245a80e1684835801161ef16b4dd78e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 22 Sep 2026 17:32:42 +0700 Subject: [PATCH 48/60] Add FwHeadlessClient methods for project creation status Wraps the two endpoints added in the previous commit: CreationStatus -> GET /api/project/creation-status AwaitCreationFinished -> GET /api/project/await-creation-finished Both take a cancellation token and callers may pass RequestAborted, which looks like it contradicts the earlier fix but does not: abandoning a status read starts no durable work and leaves nothing to act on, while abandoning the creation call leaves work running that we cannot observe. The rule is about what a call sets in motion, not about tokens; the reason is recorded on CreationStatus so it isn't rediscovered the hard way. AwaitCreationFinished translates its own timeout into TimedOutAwaitingCreation. FwHeadless only produces that status when its RequestAborted trips, which happens because we hung up, so its response arrives on a connection we already dropped. Reporting it ourselves gives callers one contract no matter which side stopped waiting. (The merge pair has the same gap; left alone as out of scope here.) Also sketches the return-202-and-poll alternative in ProjectController as commented-out code, so the trade-offs can be discussed against something concrete. It records what FwHeadless would need, and the open question that keeps it a comment: with nobody waiting, who compensates a failed creation, and where does the creation state live once it has to survive a restart? Nothing calls the new methods yet; the POST stays blocking. Co-Authored-By: Claude Opus 5 (1M context) --- .../Controllers/ProjectController.cs | 63 +++++++++++++++++++ .../LexBoxApi/Services/FwHeadlessClient.cs | 60 ++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 4664429862..32e21e6f60 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -129,8 +129,71 @@ public async Task> InitFwDataProject( await projectService.UpdateLastCommit(code); return projectId; + + // ==================================================================================== + // NOT ENABLED. Sketch of the "return 202 and let the caller poll" alternative, kept here + // deliberately so the trade-offs can be discussed against real code rather than in the + // abstract. It would replace everything from the try/catch above down to the return. + // + // What it buys: nobody holds a request open for the length of a creation, so the client + // timeout that this method currently has to reason about stops existing as a concern. + // + // What FwHeadless would need: its POST /api/project/initFwDataProject would take the + // reservation (SyncHostedService.TryStartProjectCreation), hand the work to a background + // runner the way SyncHostedService.QueueJob already does for merges, and return 202 at + // once. The two status endpoints the caller would then poll already exist and work today: + // GET /api/project/creation-status and GET /api/project/await-creation-finished, reachable + // via FwHeadlessClient.CreationStatus / AwaitCreationFinished. + // + // The open question, and the reason this is a comment and not the implementation: with + // nobody waiting, who compensates a creation that fails? Today this method does it inline + // because it is still on the stack when the answer arrives. Options: + // (a) FwHeadless cleans up after itself -- but that means granting it permission to + // delete a LexBox project row, which it deliberately cannot do today. + // (b) A Quartz job sweeps projects left in a "creating" state past some age. Closest to + // how the rest of the system already handles background work. + // (c) Leave failures in place for an admin and surface them in the UI. + // (b) or (c) also need the creation state persisted on the project row: FwHeadless's + // status is in-memory, so it does not survive a restart on either side, and a client that + // reconnects later has nothing to poll against. + // + // The LexBox side would then be roughly: + // + // // Fire and forget: FwHeadless queues the work and answers immediately. + // var accepted = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi); + // if (accepted.error is not null) + // { + // // A rejection here is from the queueing step only (bad writing system, project + // // not found), so nothing has been built yet and compensating is unambiguous. + // await CleanupFailedCreation(projectId, code); + // var statusCode = accepted.statusCode == HttpStatusCode.BadRequest + // ? StatusCodes.Status400BadRequest + // : StatusCodes.Status500InternalServerError; + // return Problem($"Failed to start creating the project: {accepted.error}", statusCode: statusCode); + // } + // // UpdateLastCommit moves to whatever observes the creation finishing; there is no + // // commit to record yet at this point. + // return Accepted(projectId); + // ==================================================================================== } + // NOT ENABLED. The companion to the 202 sketch above: what the frontend would poll while a + // creation runs. Uncomment along with the 202 change. + // + // [HttpGet("fwDataProjectCreationStatus/{projectId}")] + // [AdminRequired] + // [ProducesResponseType(StatusCodes.Status200OK)] + // [ProducesResponseType(StatusCodes.Status404NotFound)] + // public async Task> FwDataProjectCreationStatus( + // Guid projectId, + // CancellationToken cancellationToken) + // { + // // Passing RequestAborted is fine here, unlike on the creation call itself: a status read + // // starts no durable work, so abandoning it costs nothing. See FwHeadlessClient.CreationStatus. + // var status = await fwHeadlessClient.CreationStatus(projectId, cancellationToken); + // return status is null ? NotFound() : status; + // } + /// /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists /// ("Migrated,Migrating"), so the parsed value must round-trip to the name that was passed in. diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 9b2c75a583..8665e6df0f 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -225,4 +225,64 @@ public async Task UnblockProject(Guid projectId) return (response.StatusCode, responseBody); } + /// + /// Current creation status for a project, or null if FwHeadless doesn't know the project or the + /// call failed (both are logged). Cheap and safe to poll. + /// + /// + /// Unlike , this one does take a cancellation token, and callers may + /// safely pass HttpContext.RequestAborted. Abandoning a status read costs nothing: it starts no + /// durable work and tells the caller nothing it has to act on. Abandoning the creation call is what + /// is dangerous, because the creation keeps running on the FwHeadless side either way. + /// + public async Task CreationStatus(Guid projectId, CancellationToken cancellationToken = default) + { + var response = await httpClient.GetAsync($"/api/project/creation-status?projectId={projectId}", cancellationToken); + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(cancellationToken); + // 404 is an ordinary answer ("no such project"), not a fault worth an error log. + if (response.StatusCode != HttpStatusCode.NotFound) + { + logger.LogError("Failed to get project creation status: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync(cancellationToken)); + } + return null; + } + + /// + /// Waits for an in-flight creation to finish and returns how it ended, or null if FwHeadless doesn't + /// know the project or the call failed. Returns TimedOutAwaitingCreation if the wait is given up on + /// -- by either side -- which means "still running, ask again", never "creation failed". + /// + public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken = default) + { + HttpResponseMessage response; + try + { + response = await httpClient.GetAsync($"/api/project/await-creation-finished?projectId={projectId}", cancellationToken); + } + catch (OperationCanceledException) + { + // We gave up first, so FwHeadless's own TimedOutAwaitingCreation response is on a connection + // we already dropped. Report the same thing ourselves, so a caller sees one contract no + // matter which side stopped waiting. The creation itself is untouched and still running. + logger.LogInformation("Gave up awaiting creation of project {ProjectId}; it is still running", projectId); + return ProjectCreationStatus.TimedOutAwaitingCreation; + } + + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(cancellationToken); + if (response.StatusCode != HttpStatusCode.NotFound) + { + logger.LogError("Failed to await project creation: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync(cancellationToken)); + } + return null; + } } From 19edb58d107cca516a060a4b8dda91a2d4025165 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 26 Oct 2026 14:03:09 +0700 Subject: [PATCH 49/60] Simply ProjectCreationService XML handling 1. No need to use a full XmlWriter for a two-line file 2. When replacing forward slash with backslash in LinkedFilesDirectory in a newly-created .fwdata file, exit early if we can detect that we made no changes, avoiding the need to copy the rest of the file. --- .../Services/ProjectCreationService.cs | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs index 8f36c3d83a..dfa84441b0 100644 --- a/backend/FwHeadless/Services/ProjectCreationService.cs +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -1,11 +1,9 @@ using System.Xml; -using System.Xml.Linq; using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.LcmUtils; using LexCore.Exceptions; using LexCore.Sync; using Microsoft.Extensions.Options; -using SIL.Xml; namespace FwHeadless.Services; @@ -126,6 +124,7 @@ private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) var sourcePath = fwDataProject.FilePath; var tempPath = sourcePath + ".tmp"; + var madeChanges = false; using (var reader = new StreamReader(sourcePath)) using (var writer = new StreamWriter(tempPath)) { @@ -136,7 +135,14 @@ private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) { if (nextLineIsValue) { - line = line.Replace(forwardSlashValue, backslashValue); + var changedLine = line.Replace(forwardSlashValue, backslashValue); + if (string.Equals(changedLine, line, StringComparison.InvariantCulture)) + { + // Exit early and don't overwrite since we dind't need this change + break; + } + madeChanges = true; + line = changedLine; nextLineIsValue = false; done = true; } @@ -147,17 +153,19 @@ private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) writer.WriteLine(line); } } - File.Move(tempPath, sourcePath, overwrite: true); + if (madeChanges) + File.Move(tempPath, sourcePath, overwrite: true); + else + File.Delete(tempPath); } private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) { var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename); - using (var writer = XmlWriter.Create(path, CanonicalXmlSettings.CreateXmlWriterSettings())) - { - writer.WriteStartDocument(); - new XElement("AdditionalFields").WriteTo(writer); - } + File.WriteAllText(path, """ + + + """); return path; } From da2a31b0fc519140bb34201e32baa4cd8ff236e5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 11:02:33 +0700 Subject: [PATCH 50/60] Merge sync and creation reservations into one dictionary A project may be queued/running a sync or being created, never both. Tracking both in a single ConcurrentDictionary lets one atomic TryAdd/GetOrAdd decide who gets the project, so the separate lock that serialised check-then-add across two dictionaries is no longer needed. Removals are conditional on the exact entry, so ending one kind of job can never clear the other kind. Co-Authored-By: Claude Opus 5.5 --- .../FwHeadless/Services/SyncHostedService.cs | 83 +++++++++++-------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 0d6f944348..3777b720aa 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -18,17 +18,37 @@ namespace FwHeadless.Services; public class SyncHostedService(IServiceProvider services, ILogger logger, IMemoryCache memoryCache) : BackgroundService { private readonly Channel _projectsToSync = Channel.CreateUnbounded(); - private readonly ConcurrentDictionary> _projectsQueuedOrRunning = new(); - // Projects currently being created and populated with an initial .fwdata file. Tracked here (not just in ProjectCreationService) - // so a sync job can't be queued for a project mid-creation and race on the same fw/ folder and repo. - // The TaskCompletionSource carries the outcome to anyone long-polling AwaitCreationFinished. - private readonly ConcurrentDictionary> _projectsBeingCreated = new(); + // Projects with a sync queued or running, or currently being created and populated with an initial .fwdata file. + // Creations are tracked here (not just in ProjectCreationService) so a sync job can't be queued for a project + // mid-creation and race on the same fw/ folder and repo. Keeping both in one dictionary means a single atomic + // TryAdd/GetOrAdd decides who gets the project, so no lock is needed (this requires that SyncHostedService be a singleton). + private readonly ConcurrentDictionary _projectJobs = new(); - // Serialises the check-then-add across the two dictionaries above, so a creation and a sync can't - // both slip through by interleaving their checks. - // This requires that SyncHostedService be a singleton, of course - private readonly Lock _reservationLock = new(); + /// + /// Exactly one of or is set. The TaskCompletionSource carries the + /// outcome to anyone long-polling AwaitSyncFinished or AwaitCreationFinished. + /// + private record ProjectJob( + TaskCompletionSource? Sync, + TaskCompletionSource? Creation) + { + public static ProjectJob ForSync() => new(new TaskCompletionSource(), null); + + // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on + // the thread that finished the creation. + public static ProjectJob ForCreation() => + new(null, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + } + + private TaskCompletionSource? GetSyncJob(Guid projectId) => + _projectJobs.TryGetValue(projectId, out var job) ? job.Sync : null; + + private TaskCompletionSource? GetCreationJob(Guid projectId) => + _projectJobs.TryGetValue(projectId, out var job) ? job.Creation : null; + + // Only removes the entry if it's still the given job, so ending one kind of job can never clear the other kind + private bool TryRemoveJob(Guid projectId, ProjectJob job) => _projectJobs.TryRemove(KeyValuePair.Create(projectId, job)); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -57,14 +77,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Give clients a bit more time to poll the status CacheRecentSyncResult(projectId, result); - _projectsQueuedOrRunning.TryRemove(projectId, out var tcs); - tcs?.TrySetResult(result); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is { } tcs && TryRemoveJob(projectId, job)) + tcs.TrySetResult(result); } } public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _projectsQueuedOrRunning.ContainsKey(projectId) || _projectsBeingCreated.ContainsKey(projectId); + return _projectJobs.ContainsKey(projectId); } /// @@ -74,14 +94,7 @@ public virtual bool IsJobQueuedOrRunning(Guid projectId) /// public bool TryStartProjectCreation(Guid projectId) { - lock (_reservationLock) - { - if (_projectsQueuedOrRunning.ContainsKey(projectId)) return false; - // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on - // the thread that finished the creation. - return _projectsBeingCreated.TryAdd(projectId, - new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); - } + return _projectJobs.TryAdd(projectId, ProjectJob.ForCreation()); } /// @@ -93,15 +106,15 @@ public void EndProjectCreation(Guid projectId, ProjectCreationStatus result) // Cache before removing the reservation, so a poller that sees "not in flight" always finds the // result waiting for it rather than a gap. CacheRecentCreationResult(projectId, result); - _projectsBeingCreated.TryRemove(projectId, out var tcs); - tcs?.TrySetResult(result); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Creation is { } tcs && TryRemoveJob(projectId, job)) + tcs.TrySetResult(result); } - public bool IsProjectBeingCreated(Guid projectId) => _projectsBeingCreated.ContainsKey(projectId); + public bool IsProjectBeingCreated(Guid projectId) => GetCreationJob(projectId) is not null; public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) { - if (_projectsQueuedOrRunning.TryGetValue(projectId, out var tcs)) + if (GetSyncJob(projectId) is { } tcs) return await tcs.Task.WaitAsync(cancellationToken); return TryGetRecentSyncResult(projectId); } @@ -113,7 +126,7 @@ public void EndProjectCreation(Guid projectId, ProjectCreationStatus result) /// public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken) { - if (_projectsBeingCreated.TryGetValue(projectId, out var tcs)) + if (GetCreationJob(projectId) is { } tcs) return await tcs.Task.WaitAsync(cancellationToken); return TryGetRecentCreationResult(projectId); } @@ -132,24 +145,22 @@ private void CacheRecentCreationResult(Guid projectId, ProjectCreationStatus res public bool QueueJob(Guid projectId) { - bool addedToQueue; - lock (_reservationLock) + var newJob = ProjectJob.ForSync(); + var job = _projectJobs.GetOrAdd(projectId, newJob); + //don't sync a project while it's still being created and populated with initial .fwdata (would race on the repo) + if (job.Creation is not null) { - //don't sync a project while it's still being created and populated with initial .fwdata (would race on the repo) - if (_projectsBeingCreated.ContainsKey(projectId)) - { - logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); - return false; - } - //will only queue job if it's not already queued - addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); + logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); + return false; } + //will only queue job if it's not already queued + var addedToQueue = ReferenceEquals(job, newJob); if (addedToQueue) { if (!_projectsToSync.Writer.TryWrite(projectId)) { logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); - _projectsQueuedOrRunning.TryRemove(projectId, out _); + TryRemoveJob(projectId, newJob); return false; } From a457b8d4d5000a6387e8fd96dff3594c46af9064 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 11:29:37 +0700 Subject: [PATCH 51/60] Report why a sync request was not queued QueueJob returned a bool that conflated "already queued", "project is still being created" and "channel write failed", and the execute route ignored it, answering 200 "Sync job queued" even when no sync would run. Return a QueueJobResult instead, and have the route answer 409 while the project is still being created and 503 if the job couldn't be queued. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Routes/MergeRoutes.cs | 11 +++++- .../FwHeadless/Services/SyncHostedService.cs | 36 +++++++++++-------- ...ncHostedServiceCreationReservationTests.cs | 7 ++-- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index 0ed8387032..da46f43046 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -72,7 +72,16 @@ static async Task> ExecuteMergeRequest( return TypedResults.Problem($"Project is blocked from syncing. Reason: {blockInfo.Reason}", statusCode: StatusCodes.Status423Locked); } - syncHostedService.QueueJob(projectId); + switch (syncHostedService.QueueJob(projectId)) + { + case QueueJobResult.ProjectBeingCreated: + activity?.SetStatus(ActivityStatusCode.Ok, "Project is still being created"); + return TypedResults.Problem("Project is still being created, try again once creation has finished", + statusCode: StatusCodes.Status409Conflict); + case QueueJobResult.QueueFull: + activity?.SetStatus(ActivityStatusCode.Error, "Sync queue is full"); + return TypedResults.Problem("Unable to queue sync job", statusCode: StatusCodes.Status503ServiceUnavailable); + } activity?.SetStatus(ActivityStatusCode.Ok, "Sync job queued"); return TypedResults.Ok(); } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 3777b720aa..935a522d92 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -15,6 +15,15 @@ namespace FwHeadless.Services; +public enum QueueJobResult +{ + Queued, + AlreadyQueued, + // A sync must not race a project that's still being created from a template; the caller should retry later + ProjectBeingCreated, + QueueFull, +} + public class SyncHostedService(IServiceProvider services, ILogger logger, IMemoryCache memoryCache) : BackgroundService { private readonly Channel _projectsToSync = Channel.CreateUnbounded(); @@ -143,7 +152,7 @@ private void CacheRecentCreationResult(Guid projectId, ProjectCreationStatus res memoryCache.Set($"CreationResult|{projectId}", result, TimeSpan.FromMinutes(5)); } - public bool QueueJob(Guid projectId) + public QueueJobResult QueueJob(Guid projectId) { var newJob = ProjectJob.ForSync(); var job = _projectJobs.GetOrAdd(projectId, newJob); @@ -151,27 +160,24 @@ public bool QueueJob(Guid projectId) if (job.Creation is not null) { logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); - return false; + return QueueJobResult.ProjectBeingCreated; } //will only queue job if it's not already queued - var addedToQueue = ReferenceEquals(job, newJob); - if (addedToQueue) + if (!ReferenceEquals(job, newJob)) { - if (!_projectsToSync.Writer.TryWrite(projectId)) - { - logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); - TryRemoveJob(projectId, newJob); - return false; - } - - logger.LogInformation("Queued sync job for project {ProjectId}", projectId); + logger.LogInformation("Project {ProjectId} is already queued", projectId); + return QueueJobResult.AlreadyQueued; } - else + + if (!_projectsToSync.Writer.TryWrite(projectId)) { - logger.LogInformation("Project {ProjectId} is already queued", projectId); + logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); + TryRemoveJob(projectId, newJob); + return QueueJobResult.QueueFull; } - return addedToQueue; + logger.LogInformation("Queued sync job for project {ProjectId}", projectId); + return QueueJobResult.Queued; } private void CacheRecentSyncResult(Guid projectId, SyncJobResult result) diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs index 12cad3cd9e..677d03dd1c 100644 --- a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -39,10 +39,10 @@ public void QueueJob_is_refused_while_a_project_is_being_created() var projectId = Guid.NewGuid(); svc.TryStartProjectCreation(projectId).Should().BeTrue(); - svc.QueueJob(projectId).Should().BeFalse("a sync must not race a project that's still being created"); + svc.QueueJob(projectId).Should().Be(QueueJobResult.ProjectBeingCreated, "a sync must not race a project that's still being created"); svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); - svc.QueueJob(projectId).Should().BeTrue("syncing is allowed once creation has finished"); + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued, "syncing is allowed once creation has finished"); } [Fact] @@ -51,7 +51,8 @@ public void A_queued_sync_blocks_a_creation() var svc = NewService(); var projectId = Guid.NewGuid(); - svc.QueueJob(projectId).Should().BeTrue(); + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued); svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); } From e3bd3518531e76d75634cfa1822f916179ad79a3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 11:31:15 +0700 Subject: [PATCH 52/60] Refuse a sync request while that project is already syncing A sync request that arrived while the project's sync was already running got a 200, as if a sync covering the caller's changes had been queued. But the running sync may have read the repo before those changes were pushed, so the caller could wrongly conclude its changes were synced. Track whether a queued sync has started. A request that finds the sync still waiting in the queue is covered by it and still gets a 200; one that finds it already running now gets 429 Too Many Requests, telling the caller to try again once the running sync finishes. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Routes/MergeRoutes.cs | 4 ++++ .../FwHeadless/Services/SyncHostedService.cs | 20 ++++++++++++++++++- ...ncHostedServiceCreationReservationTests.cs | 15 ++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index da46f43046..56dafea3d2 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -78,6 +78,10 @@ static async Task> ExecuteMergeRequest( activity?.SetStatus(ActivityStatusCode.Ok, "Project is still being created"); return TypedResults.Problem("Project is still being created, try again once creation has finished", statusCode: StatusCodes.Status409Conflict); + case QueueJobResult.SyncAlreadyRunning: + activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running"); + return TypedResults.Problem("A sync is already running for this project, try again once it has finished", + statusCode: StatusCodes.Status429TooManyRequests); case QueueJobResult.QueueFull: activity?.SetStatus(ActivityStatusCode.Error, "Sync queue is full"); return TypedResults.Problem("Unable to queue sync job", statusCode: StatusCodes.Status503ServiceUnavailable); diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 935a522d92..0f43c3a23e 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -21,6 +21,8 @@ public enum QueueJobResult AlreadyQueued, // A sync must not race a project that's still being created from a template; the caller should retry later ProjectBeingCreated, + // The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes + SyncAlreadyRunning, QueueFull, } @@ -37,10 +39,13 @@ public class SyncHostedService(IServiceProvider services, ILogger /// Exactly one of or is set. The TaskCompletionSource carries the /// outcome to anyone long-polling AwaitSyncFinished or AwaitCreationFinished. + /// distinguishes a queued sync (which will pick up any changes made before it starts) + /// from a running one (which might not). /// private record ProjectJob( TaskCompletionSource? Sync, - TaskCompletionSource? Creation) + TaskCompletionSource? Creation, + bool SyncStarted = false) { public static ProjectJob ForSync() => new(new TaskCompletionSource(), null); @@ -63,6 +68,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) { + MarkSyncStarted(projectId); using var activity = FwHeadlessActivitySource.Value.StartActivity("SyncHostedService.ExecuteAsync"); await using var scope = services.CreateAsyncScope(); var syncWorker = ActivatorUtilities.CreateInstance(scope.ServiceProvider, projectId); @@ -91,6 +97,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + // QueueJob never modifies an existing sync entry, and only ExecuteAsync removes one, so nothing can race this update + internal void MarkSyncStarted(Guid projectId) + { + if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is not null) + _projectJobs.TryUpdate(projectId, job with { SyncStarted = true }, job); + } + public virtual bool IsJobQueuedOrRunning(Guid projectId) { return _projectJobs.ContainsKey(projectId); @@ -165,6 +178,11 @@ public QueueJobResult QueueJob(Guid projectId) //will only queue job if it's not already queued if (!ReferenceEquals(job, newJob)) { + if (job.SyncStarted) + { + logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId); + return QueueJobResult.SyncAlreadyRunning; + } logger.LogInformation("Project {ProjectId} is already queued", projectId); return QueueJobResult.AlreadyQueued; } diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs index 677d03dd1c..bb8b796ec8 100644 --- a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -56,6 +56,21 @@ public void A_queued_sync_blocks_a_creation() svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); } + [Fact] + public void QueueJob_is_refused_while_a_sync_is_running() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.MarkSyncStarted(projectId); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning, + "the running sync may have missed the caller's changes, so it must not be reported as covering them"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is running for this project"); + } + [Fact] public async Task AwaitCreationFinished_returns_the_outcome_the_creation_reported() { From d2fe9977818327ad56c2879cecc93fbe6bf3d173 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 11:37:19 +0700 Subject: [PATCH 53/60] Run sync awaiters' continuations asynchronously The creation TaskCompletionSource already used RunContinuationsAsynchronously, but the sync one didn't. So when ExecuteAsync completed a sync, a long-polling await-finished request could run its response handling inline on the sync loop, delaying the next project's sync until that handler hit a real await. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Services/SyncHostedService.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 0f43c3a23e..89a0c8009d 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -47,10 +47,11 @@ private record ProjectJob( TaskCompletionSource? Creation, bool SyncStarted = false) { - public static ProjectJob ForSync() => new(new TaskCompletionSource(), null); - // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on - // the thread that finished the creation. + // the thread that finished the sync or creation. + public static ProjectJob ForSync() => + new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), null); + public static ProjectJob ForCreation() => new(null, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); } From 49c53f5fbc720e0b8828acec3bc9f498406beebc Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 11:43:19 +0700 Subject: [PATCH 54/60] Explicit comment on AlreadyQueued result If a sync is already queued, we don't return an error but let it fall through to the "Sync job queued" HTTP 200 result. This might look like a bug, so add a comment explaining that it's intentional. --- backend/FwHeadless/Routes/MergeRoutes.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index 56dafea3d2..79f83eed8d 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -85,6 +85,9 @@ static async Task> ExecuteMergeRequest( case QueueJobResult.QueueFull: activity?.SetStatus(ActivityStatusCode.Error, "Sync queue is full"); return TypedResults.Problem("Unable to queue sync job", statusCode: StatusCodes.Status503ServiceUnavailable); + // case QueueJobResult.AlreadyQueued: + // // Not handled, fall through to returning HTTP 200 "Sync job queued" + // break; } activity?.SetStatus(ActivityStatusCode.Ok, "Sync job queued"); return TypedResults.Ok(); From ad0f00a5594f4bafe0938c830de5fd08f23b8176 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 12:15:38 +0700 Subject: [PATCH 55/60] Limit writing-system lists when creating a project Neither LexBoxApi nor FwHeadless limited how many writing systems a project-creation request could list, how long each tag could be, or whether a tag was repeated. LexBoxApi also created the project row and repo before any tag was checked, so a bad request was only rejected by FwHeadless after there was already a project to tear down. Add a shared shape check on InitFwDataProjectInput: at most 100 writing systems per list, at most 128 characters per tag, no empty tags, and no case-insensitive duplicates within a list. The limits are deliberately generous so that metadata in private-use (-x-) subtags isn't forbidden. LexBoxApi runs the check before creating anything, and FwHeadless runs it again before its BCP 47 check. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Routes/ProjectRoutes.cs | 14 ++-- .../Controllers/ProjectController.cs | 10 ++- .../Entities/InitFwDataProjectInput.cs | 31 +++++++- .../InitFwDataProjectValidationTests.cs | 22 ++++++ .../LexCore/InitFwDataProjectInputTests.cs | 75 +++++++++++++++++++ 5 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 backend/Testing/LexCore/InitFwDataProjectInputTests.cs diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index c84501f662..290bd4012d 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -30,17 +30,17 @@ private static async Task> InitFwDataProject( ILogger logger) { // Null-safe against a malformed body (System.Text.Json leaves an absent list null). - if (input.WsVernacular is not { Count: > 0 }) - return TypedResults.Problem("At least one vernacular writing system is required", - statusCode: StatusCodes.Status400BadRequest); - if (input.WsAnalysis is not { Count: > 0 }) - return TypedResults.Problem("At least one analysis writing system is required", - statusCode: StatusCodes.Status400BadRequest); + var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", input.WsVernacular) + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", input.WsAnalysis); + if (wsError is not null) + return TypedResults.Problem(wsError, statusCode: StatusCodes.Status400BadRequest); var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); if (invalidWs is not null) return TypedResults.Problem($"Invalid writing system code: {invalidWs}", statusCode: StatusCodes.Status400BadRequest); - if (string.IsNullOrEmpty(input.WsUi) || !IetfLanguageTag.IsValid(input.WsUi)) + if (string.IsNullOrEmpty(input.WsUi) + || input.WsUi.Length > InitFwDataProjectInput.MaxWritingSystemTagLength + || !IetfLanguageTag.IsValid(input.WsUi)) return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", statusCode: StatusCodes.Status400BadRequest); diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 32e21e6f60..5b22f632bd 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -55,8 +55,12 @@ public async Task> InitFwDataProject( string? name = null, string? projectOrigin = null) { - if (wsVernacular is null || wsVernacular.Length == 0) - return Problem("At least one vernacular writing system is required", statusCode: StatusCodes.Status400BadRequest); + // Validate before anything is created, so a bad request never leaves a project to tear down. + var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); + var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", wsVernacular) + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", wsAnalysisOrDefault); + if (wsError is not null) + return Problem(wsError, statusCode: StatusCodes.Status400BadRequest); if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); @@ -73,8 +77,6 @@ public async Task> InitFwDataProject( if (await projectService.ProjectExists(code)) return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); - var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); - var projectId = await projectService.CreateProject(new CreateProjectInput( Id: null, Name: string.IsNullOrWhiteSpace(name) ? code : name, diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs index ba2e2b72a9..edb234a02e 100644 --- a/backend/LexCore/Entities/InitFwDataProjectInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -3,9 +3,36 @@ namespace LexCore.Entities; /// /// Body of the internal FwHeadless "init fwdata project" call. The vernacular/analysis /// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one -/// vernacular; analysis and UI default to "en"). +/// vernacular; analysis and UI default to "en"); FwHeadless validates them again before using them. /// public record InitFwDataProjectInput( IReadOnlyList WsVernacular, IReadOnlyList WsAnalysis, - string WsUi); + string WsUi) +{ + // Deliberately generous: real projects use far fewer writing systems than this, and real tags are far + // shorter, but private-use subtags after "-x-" may carry arbitrary metadata and we don't want to forbid + // any reasonable use of it. + public const int MaxWritingSystemsPerList = 100; + public const int MaxWritingSystemTagLength = 128; + + /// + /// Shape checks shared by LexBoxApi (before it creates anything) and FwHeadless. Returns an error + /// message, or null if the list is acceptable. FwHeadless additionally checks that each tag is valid BCP 47. + /// + public static string? ValidateWritingSystemList(string kind, IReadOnlyList? tags) + { + if (tags is not { Count: > 0 }) + return $"At least one {kind} writing system is required"; + if (tags.Count > MaxWritingSystemsPerList) + return $"At most {MaxWritingSystemsPerList} {kind} writing systems are allowed"; + if (tags.Any(string.IsNullOrWhiteSpace)) + return $"Empty {kind} writing system code"; + if (tags.FirstOrDefault(t => t.Length > MaxWritingSystemTagLength) is { } tooLong) + return $"The {kind} writing system code is longer than {MaxWritingSystemTagLength} characters: {tooLong[..32]}…"; + // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system + if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate) + return $"Duplicate {kind} writing system: {duplicate.Key}"; + return null; + } +} diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index c549b0796e..ab9a7911b0 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -40,6 +40,28 @@ public async Task Rejects_when_no_vernacular_writing_system_is_supplied() problem.Detail.Should().Contain("vernacular"); } + // The controller's services are all null, so reaching the 400 proves nothing was created first + [Fact] + public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_anything() + { + var result = await NewController().InitFwDataProject("myproj", wsVernacular: ["fr", "fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Duplicate vernacular writing system"); + } + + [Fact] + public async Task Rejects_too_many_analysis_writing_systems_before_creating_anything() + { + var analysis = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); + var result = await NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], wsAnalysis: analysis); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("analysis writing systems are allowed"); + } + [Fact] public async Task Rejects_an_invalid_project_code() { diff --git a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs new file mode 100644 index 0000000000..2e65458fcb --- /dev/null +++ b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs @@ -0,0 +1,75 @@ +using LexCore.Entities; +using FluentAssertions; + +namespace Testing.LexCore; + +public class InitFwDataProjectInputTests +{ + private static string? Validate(params string[] tags) => + InitFwDataProjectInput.ValidateWritingSystemList("vernacular", tags); + + [Fact] + public void Accepts_a_typical_list() + { + Validate("fr", "en-US", "qaa-Latn-x-mycustom").Should().BeNull(); + } + + [Fact] + public void Rejects_a_null_list() + { + InitFwDataProjectInput.ValidateWritingSystemList("vernacular", null) + .Should().Contain("At least one vernacular writing system is required"); + } + + [Fact] + public void Rejects_an_empty_list() + { + Validate().Should().Contain("At least one vernacular writing system is required"); + } + + [Fact] + public void Accepts_the_maximum_number_of_writing_systems() + { + var tags = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList).Select(i => $"qaa-x-ws{i}").ToArray(); + Validate(tags).Should().BeNull(); + } + + [Fact] + public void Rejects_more_than_the_maximum_number_of_writing_systems() + { + var tags = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); + Validate(tags).Should().Contain($"At most {InitFwDataProjectInput.MaxWritingSystemsPerList}"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Rejects_an_empty_tag(string tag) + { + Validate("fr", tag).Should().Contain("Empty vernacular writing system code"); + } + + [Fact] + public void Accepts_a_tag_of_the_maximum_length() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength - "qaa-x-".Length); + Validate(tag).Should().BeNull(); + } + + [Fact] + public void Rejects_a_tag_longer_than_the_maximum_and_shows_its_start() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var error = Validate("fr", tag); + error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); + error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); + } + + [Theory] + [InlineData("fr", "fr")] + [InlineData("fr", "FR")] // BCP 47 tags are case-insensitive + public void Rejects_a_duplicate_tag(string first, string second) + { + Validate(first, "en", second).Should().Contain($"Duplicate vernacular writing system: {first}"); + } +} From a7579ccb2199430252cb5f6bae194b8e4b581532 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 15:19:54 +0700 Subject: [PATCH 56/60] Delete the repo before the row when cleaning up a failed creation CleanupFailedProjectCreation deleted the project row first, so if deleting the repo then failed, the repo was left with no row pointing at it: invisible to admins, and it made the next creation attempt with that code fail. Deleting the repo first means any failure leaves a project row an admin can see and delete, and a retry gets a clear "already exists" conflict. The cache invalidations now run in a finally, so they're no longer skipped when a step fails. Co-Authored-By: Claude Opus 5.5 --- backend/LexBoxApi/Services/ProjectService.cs | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 5aa82c252e..d90c7e50aa 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -244,17 +244,25 @@ public async Task CleanupFailedProjectCreation(Guid projectId, string code) // do this first, because it throws if a creation or sync is in progress, which stops us from // tearing down an active or already-completed creation (mirrors DeleteProjectPermanently) await fwHeadless.DeleteProject(projectId); - var project = await dbContext.Projects.FindAsync(projectId); - if (project is not null) + try { - dbContext.Projects.Remove(project); - await dbContext.SaveChangesAsync(); + // Repo before row: if deleting the repo fails, the project row survives for an admin to see and + // delete, instead of an orphaned repo with no row pointing at it + await hgService.DeleteRepoIfExists(code); + var project = await dbContext.Projects.FindAsync(projectId); + if (project is not null) + { + dbContext.Projects.Remove(project); + await dbContext.SaveChangesAsync(); + } + } + finally + { + // Don't forget to add more Invalidate calls here if we add new caches + InvalidateProjectCodeCache(code); + InvalidateProjectConfidentialityCache(projectId); + InvalidateProjectOrgIdsCache(projectId); } - await hgService.DeleteRepoIfExists(code); - // Don't forget to add more Invalidate calls here if we add new caches - InvalidateProjectCodeCache(code); - InvalidateProjectConfidentialityCache(projectId); - InvalidateProjectOrgIdsCache(projectId); } public async ValueTask LookupProjectOrgIds(Guid projectId) From da27256cca637ea7c01b219f15d62c4a6bc25b8e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 16:13:37 +0700 Subject: [PATCH 57/60] Check the UI writing system before creating a project InitFwDataProject validated the vernacular and analysis lists before creating anything, but not wsUi. An empty or over-long UI tag created the project row and repo, was then rejected by FwHeadless, and had to be torn down again. ValidateUiWritingSystem does the same shape checks as the list validator (present, at most MaxWritingSystemTagLength characters) and is shared by LexBoxApi and FwHeadless, so both reject the same tags. FwHeadless still checks BCP 47 validity itself, since LexBoxApi doesn't reference SIL.WritingSystems. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Routes/ProjectRoutes.cs | 7 ++--- .../Controllers/ProjectController.cs | 3 +- .../Entities/InitFwDataProjectInput.cs | 13 ++++++++ .../InitFwDataProjectValidationTests.cs | 11 +++++++ .../LexCore/InitFwDataProjectInputTests.cs | 31 +++++++++++++++++++ 5 files changed, 60 insertions(+), 5 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index 290bd4012d..dd3e540c4f 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -31,16 +31,15 @@ private static async Task> InitFwDataProject( { // Null-safe against a malformed body (System.Text.Json leaves an absent list null). var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", input.WsVernacular) - ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", input.WsAnalysis); + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", input.WsAnalysis) + ?? InitFwDataProjectInput.ValidateUiWritingSystem(input.WsUi); if (wsError is not null) return TypedResults.Problem(wsError, statusCode: StatusCodes.Status400BadRequest); var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); if (invalidWs is not null) return TypedResults.Problem($"Invalid writing system code: {invalidWs}", statusCode: StatusCodes.Status400BadRequest); - if (string.IsNullOrEmpty(input.WsUi) - || input.WsUi.Length > InitFwDataProjectInput.MaxWritingSystemTagLength - || !IetfLanguageTag.IsValid(input.WsUi)) + if (!IetfLanguageTag.IsValid(input.WsUi)) return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", statusCode: StatusCodes.Status400BadRequest); diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 5b22f632bd..a38f01ac13 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -58,7 +58,8 @@ public async Task> InitFwDataProject( // Validate before anything is created, so a bad request never leaves a project to tear down. var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", wsVernacular) - ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", wsAnalysisOrDefault); + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", wsAnalysisOrDefault) + ?? InitFwDataProjectInput.ValidateUiWritingSystem(wsUi); if (wsError is not null) return Problem(wsError, statusCode: StatusCodes.Status400BadRequest); if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs index edb234a02e..1785126854 100644 --- a/backend/LexCore/Entities/InitFwDataProjectInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -35,4 +35,17 @@ public record InitFwDataProjectInput( return $"Duplicate {kind} writing system: {duplicate.Key}"; return null; } + + /// + /// Shape checks for the UI writing system, shared like . Returns an + /// error message, or null if the tag is acceptable. FwHeadless additionally checks that it is valid BCP 47. + /// + public static string? ValidateUiWritingSystem(string? tag) + { + if (string.IsNullOrWhiteSpace(tag)) + return "A UI writing system is required"; + if (tag.Length > MaxWritingSystemTagLength) + return $"The UI writing system code is longer than {MaxWritingSystemTagLength} characters: {tag[..32]}…"; + return null; + } } diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index ab9a7911b0..9b49e8ce20 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -62,6 +62,17 @@ public async Task Rejects_too_many_analysis_writing_systems_before_creating_anyt problem.Detail.Should().Contain("analysis writing systems are allowed"); } + [Fact] + public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything() + { + var wsUi = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var result = await NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], wsUi: wsUi); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("UI writing system code is longer than"); + } + [Fact] public async Task Rejects_an_invalid_project_code() { diff --git a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs index 2e65458fcb..7940d6b65f 100644 --- a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs +++ b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs @@ -72,4 +72,35 @@ public void Rejects_a_duplicate_tag(string first, string second) { Validate(first, "en", second).Should().Contain($"Duplicate vernacular writing system: {first}"); } + + [Fact] + public void Accepts_a_typical_UI_writing_system() + { + InitFwDataProjectInput.ValidateUiWritingSystem("en").Should().BeNull(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Rejects_a_missing_UI_writing_system(string? tag) + { + InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().Contain("A UI writing system is required"); + } + + [Fact] + public void Accepts_a_UI_writing_system_of_the_maximum_length() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength - "qaa-x-".Length); + InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().BeNull(); + } + + [Fact] + public void Rejects_a_UI_writing_system_longer_than_the_maximum_and_shows_its_start() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var error = InitFwDataProjectInput.ValidateUiWritingSystem(tag); + error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); + error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); + } } From 0bfd8e2b2e0db7b5300616d7b36eb221502772a3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 27 Oct 2026 17:06:46 +0700 Subject: [PATCH 58/60] Return 409 when a concurrent request takes the project code InitFwDataProject's ProjectExists check is only a read, so two requests for the same new code can both pass it. The unique index on Project.Code then makes the second insert fail, and that DbUpdateException surfaced as a 500 instead of the documented 409. Catch the duplicate-project-code violation around CreateProject using the existing DbError mapping (as the GraphQL mutations already do) and return the same 409 as the early check. The losing request has created nothing: its insert fails before the repo is initialised, and its transaction rolls back. Co-Authored-By: Claude Opus 5.5 --- .../Controllers/ProjectController.cs | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index a38f01ac13..9aea06c7c9 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -78,17 +78,27 @@ public async Task> InitFwDataProject( if (await projectService.ProjectExists(code)) return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); - var projectId = await projectService.CreateProject(new CreateProjectInput( - Id: null, - Name: string.IsNullOrWhiteSpace(name) ? code : name, - Description: string.Empty, - Code: code, - Type: ProjectType.FLEx, - RetentionPolicy: RetentionPolicy.Verified, - IsConfidential: false, - ProjectManagerId: null, - OrgId: null), - projectOrigin: origin); + Guid projectId; + try + { + projectId = await projectService.CreateProject(new CreateProjectInput( + Id: null, + Name: string.IsNullOrWhiteSpace(name) ? code : name, + Description: string.Empty, + Code: code, + Type: ProjectType.FLEx, + RetentionPolicy: RetentionPolicy.Verified, + IsConfidential: false, + ProjectManagerId: null, + OrgId: null), + projectOrigin: origin); + } + catch (DbUpdateException e) when (DbError.CreateErrorFrom(e).Code == DbErrorCode.DuplicateProjectCode) + { + // ProjectExists above is only a fast path: a concurrent request can take the code after that check, + // and the unique index on Project.Code is what actually catches it. Nothing was created by this request. + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + } // At this point, the project row and empty repo exist. Have FwHeadless populate the repo with // the initial .fwdata; if that fails, compensate by tearing the project back down. From 02df23561e19a4ef24701a550f01074df07d5c79 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 3 Nov 2026 11:52:27 +0700 Subject: [PATCH 59/60] Create projects from the template in a background job InitFwDataProject used to hold the request open while FwHeadless built and pushed the project, which can take longer than the ingress's 60-second proxy timeout. A caller could then get a 504 for a creation that went on to succeed, and a LexBox restart mid-request skipped both the rollback on failure and recording the commit on success. The endpoint now creates the project row (marked FwDataInitStatus InProgress in the same insert) and the empty repo, queues InitFwDataProjectJob, and returns 202 with a Location header for the new fwDataProjectCreationStatus endpoint, which reports Creating, Created or CreationFailed. The job runs on Quartz's persistent store with RequestRecovery, so a run interrupted by LexBox stopping is rerun, and every run is written to be safe to repeat: - A non-empty repo means an earlier attempt's push landed, so the project is marked ready without asking FwHeadless. - A creation FwHeadless is still running is waited for, not restarted. - Otherwise FwHeadless's local project state is cleared (an interrupted attempt may have left some) and the creation started. - A failure FwHeadless reports is rolled back with CleanupFailedProjectCreation. - An unknown outcome (FwHeadless unreachable or busy, our request timing out, or a 409 because something else holds the project) is retried a minute later, up to 10 attempts, after which the project stays marked InProgress for an admin. The reason for a failure is kept in memory for an hour after the rollback has deleted the row, so the status endpoint can report it. Co-Authored-By: Claude Opus 5.5 --- .../Controllers/ProjectController.cs | 145 +- .../LexBoxApi/Jobs/InitFwDataProjectJob.cs | 211 +++ backend/LexBoxApi/ScheduledTasksKernel.cs | 3 + backend/LexBoxApi/Services/HgService.cs | 2 +- backend/LexBoxApi/Services/ProjectService.cs | 20 +- backend/LexCore/Entities/Project.cs | 8 + ...859_AddProjectFwDataInitStatus.Designer.cs | 1422 +++++++++++++++++ ...261103044859_AddProjectFwDataInitStatus.cs | 29 + .../LexBoxDbContextModelSnapshot.cs | 44 +- .../LexBoxApi/InitFwDataProjectJobTests.cs | 196 +++ .../InitFwDataProjectValidationTests.cs | 2 +- .../InitFwDataProjectTests.cs | 42 +- frontend/schema.graphql | 15 + 13 files changed, 1998 insertions(+), 141 deletions(-) create mode 100644 backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs create mode 100644 backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.Designer.cs create mode 100644 backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.cs create mode 100644 backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 9aea06c7c9..e82c8147ec 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,4 +1,3 @@ -using System.Net; using System.Text.RegularExpressions; using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; @@ -8,9 +7,11 @@ using LexCore.Entities; using LexCore.Exceptions; using LexCore.ServiceInterfaces; +using LexCore.Sync; using LexData; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; using Quartz; using Path = System.IO.Path; @@ -24,7 +25,6 @@ public class ProjectController( LexBoxDbContext lexBoxDbContext, IPermissionService permissionService, ISchedulerFactory scheduler, - FwHeadlessClient fwHeadlessClient, ILogger logger) : ControllerBase { @@ -34,6 +34,10 @@ public class ProjectController( /// /// At some future date this API may be opened up to trusted users, but the projectOrigin /// parameter will remain admin-only. + /// + /// Building and pushing the project takes too long to hold a request open for, so this creates the + /// project row and empty repo, queues the rest as a background job, and returns 202 Accepted with the + /// new project's ID. Poll to learn when it's ready. /// /// Project code for the new project. /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. @@ -43,7 +47,7 @@ public class ProjectController( /// Optional to record as the project's origin; admin-only. [HttpPost("initFwDataProject")] [AdminRequired] - [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status202Accepted)] [ProducesResponseType(StatusCodes.Status400BadRequest)] [ProducesResponseType(StatusCodes.Status409Conflict)] [ProducesResponseType(StatusCodes.Status500InternalServerError)] @@ -91,7 +95,9 @@ public async Task> InitFwDataProject( IsConfidential: false, ProjectManagerId: null, OrgId: null), - projectOrigin: origin); + projectOrigin: origin, + // In the same insert as the row itself, so the project is never visible without it + fwDataInitStatus: FwDataInitStatus.InProgress); } catch (DbUpdateException e) when (DbError.CreateErrorFrom(e).Code == DbErrorCode.DuplicateProjectCode) { @@ -100,112 +106,47 @@ public async Task> InitFwDataProject( return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); } - // At this point, the project row and empty repo exist. Have FwHeadless populate the repo with - // the initial .fwdata; if that fails, compensate by tearing the project back down. - // - // Deliberately no CancellationToken here. An action's CancellationToken parameter binds to - // HttpContext.RequestAborted, which trips whenever the caller hangs up or its client times out -- - // routine for an operation this slow. FwHeadless's initFwDataProject takes no cancellation token - // of its own, so it runs the creation through to completion regardless of what happens to this - // connection. Cancelling our wait would therefore tell us nothing about whether the project was - // created, while making us tear down one that FwHeadless is still building. HttpClient.Timeout - // (5 minutes, set in LexBoxKernel) is what bounds this call. - (HttpStatusCode statusCode, string? error) result; + // The project row (marked as being initialised) and its empty repo now exist. Populating the repo + // happens in the job, which also marks the project ready or rolls it back. try { - result = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi); + await InitFwDataProjectJob.Queue(scheduler, projectId, + new InitFwDataProjectInput(wsVernacular, wsAnalysisOrDefault, wsUi)); } catch (Exception ex) { - // FwHeadless never answered, so we cannot tell a project that was never created from one - // still being built -- or one finished just as we gave up. Tearing it down here would race - // FwHeadless writing into the very repo we would delete. An orphaned empty project is - // recoverable by an admin; a project deleted out from under an in-flight creation is not. - logger.LogError(ex, - "initFwDataProject did not return for project {ProjectId} ({Code}); leaving it in place because FwHeadless may still be creating it. It may need manual cleanup", - projectId, - code); - throw; - } - if (result.error is not null) - { - // FwHeadless answered and the answer was failure, so creation is definitively not running - // (it releases its own reservation and cleans up locally before responding). Only here do we - // know enough to compensate. + // Nothing has been sent to FwHeadless yet, so tearing the project down is unambiguous + logger.LogError(ex, "Failed to queue initialising project {ProjectId} ({Code}), rolling it back", projectId, code); await CleanupFailedCreation(projectId, code); - // Surface a bad request from FwHeadless (e.g. an invalid writing-system tag) as 400, not 500. - var statusCode = result.statusCode == HttpStatusCode.BadRequest - ? StatusCodes.Status400BadRequest - : StatusCodes.Status500InternalServerError; - return Problem($"Failed to create the project from the template: {result.error}", statusCode: statusCode); + throw; } - - await projectService.UpdateLastCommit(code); - return projectId; - - // ==================================================================================== - // NOT ENABLED. Sketch of the "return 202 and let the caller poll" alternative, kept here - // deliberately so the trade-offs can be discussed against real code rather than in the - // abstract. It would replace everything from the try/catch above down to the return. - // - // What it buys: nobody holds a request open for the length of a creation, so the client - // timeout that this method currently has to reason about stops existing as a concern. - // - // What FwHeadless would need: its POST /api/project/initFwDataProject would take the - // reservation (SyncHostedService.TryStartProjectCreation), hand the work to a background - // runner the way SyncHostedService.QueueJob already does for merges, and return 202 at - // once. The two status endpoints the caller would then poll already exist and work today: - // GET /api/project/creation-status and GET /api/project/await-creation-finished, reachable - // via FwHeadlessClient.CreationStatus / AwaitCreationFinished. - // - // The open question, and the reason this is a comment and not the implementation: with - // nobody waiting, who compensates a creation that fails? Today this method does it inline - // because it is still on the stack when the answer arrives. Options: - // (a) FwHeadless cleans up after itself -- but that means granting it permission to - // delete a LexBox project row, which it deliberately cannot do today. - // (b) A Quartz job sweeps projects left in a "creating" state past some age. Closest to - // how the rest of the system already handles background work. - // (c) Leave failures in place for an admin and surface them in the UI. - // (b) or (c) also need the creation state persisted on the project row: FwHeadless's - // status is in-memory, so it does not survive a restart on either side, and a client that - // reconnects later has nothing to poll against. - // - // The LexBox side would then be roughly: - // - // // Fire and forget: FwHeadless queues the work and answers immediately. - // var accepted = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi); - // if (accepted.error is not null) - // { - // // A rejection here is from the queueing step only (bad writing system, project - // // not found), so nothing has been built yet and compensating is unambiguous. - // await CleanupFailedCreation(projectId, code); - // var statusCode = accepted.statusCode == HttpStatusCode.BadRequest - // ? StatusCodes.Status400BadRequest - // : StatusCodes.Status500InternalServerError; - // return Problem($"Failed to start creating the project: {accepted.error}", statusCode: statusCode); - // } - // // UpdateLastCommit moves to whatever observes the creation finishing; there is no - // // commit to record yet at this point. - // return Accepted(projectId); - // ==================================================================================== + return AcceptedAtAction(nameof(FwDataProjectCreationStatus), new { projectId }, projectId); } - // NOT ENABLED. The companion to the 202 sketch above: what the frontend would poll while a - // creation runs. Uncomment along with the 202 change. - // - // [HttpGet("fwDataProjectCreationStatus/{projectId}")] - // [AdminRequired] - // [ProducesResponseType(StatusCodes.Status200OK)] - // [ProducesResponseType(StatusCodes.Status404NotFound)] - // public async Task> FwDataProjectCreationStatus( - // Guid projectId, - // CancellationToken cancellationToken) - // { - // // Passing RequestAborted is fine here, unlike on the creation call itself: a status read - // // starts no durable work, so abandoning it costs nothing. See FwHeadlessClient.CreationStatus. - // var status = await fwHeadlessClient.CreationStatus(projectId, cancellationToken); - // return status is null ? NotFound() : status; - // } + /// + /// Admin-only: how the creation started by is going. + /// Creating while the job is still running; Created once the project exists and isn't being + /// initialised (so also for any project that wasn't created that way). A failed creation is rolled + /// back, deleting the project, after which this reports CreationFailed with the reason for an hour, + /// as long as LexBox hasn't restarted, and 404 after that. + /// + [HttpGet("fwDataProjectCreationStatus/{projectId}")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task> FwDataProjectCreationStatus( + Guid projectId, + [FromServices] IMemoryCache memoryCache) + { + var initStatus = await lexBoxDbContext.Projects + .Where(p => p.Id == projectId) + .Select(p => (FwDataInitStatus?)p.FwDataInitStatus) + .SingleOrDefaultAsync(); + if (initStatus is not null) + return initStatus == FwDataInitStatus.InProgress ? ProjectCreationStatus.Creating : ProjectCreationStatus.Created; + var failure = InitFwDataProjectJob.TryGetRecentFailure(memoryCache, projectId); + return failure is null ? NotFound() : ProjectCreationStatus.Failed(failure); + } /// /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists @@ -228,7 +169,7 @@ private async Task CleanupFailedCreation(Guid projectId, string code) } catch (Exception ex) { - logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}) after a failed template creation", projectId, code); + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}); it may need manual cleanup", projectId, code); } } diff --git a/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs new file mode 100644 index 0000000000..38c90de1de --- /dev/null +++ b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs @@ -0,0 +1,211 @@ +using System.Net; +using System.Text.Json; +using LexBoxApi.Services; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexCore.Sync; +using LexData; +using Microsoft.Extensions.Caching.Memory; +using Quartz; + +namespace LexBoxApi.Jobs; + +/// +/// Has FwHeadless populate the empty repo of a project created by ProjectController.InitFwDataProject, +/// then marks the project ready, or rolls it back if FwHeadless reports that the creation failed. +/// +/// +/// Quartz keeps this job in the database and reruns it if LexBox stops partway through (RequestRecovery), +/// so every run must be safe to repeat. Each run therefore starts by finding out how far an earlier run +/// got, using LexBox's own repo as the source of truth. When the outcome is unknown (FwHeadless is +/// unreachable, busy, or still running a creation), the job schedules another attempt rather than guess. +/// +public class InitFwDataProjectJob( + LexBoxDbContext dbContext, + ProjectService projectService, + FwHeadlessClient fwHeadlessClient, + IHgService hgService, + ISchedulerFactory schedulerFactory, + IMemoryCache memoryCache, + ILogger logger) : LexJob +{ + public const int MaxAttempts = 10; + public static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + public static JobKey Key { get; } = new("InitFwDataProjectJob", "ProjectCreation"); + + public static async Task Queue(ISchedulerFactory schedulerFactory, + Guid projectId, + InitFwDataProjectInput input, + int attempt = 1, + TimeSpan? delay = null, + CancellationToken cancellationToken = default) + { + await QueueJob(schedulerFactory, + Key, + new JobDataMap + { + { nameof(ProjectId), projectId.ToString() }, + // The job store only takes strings, so the writing systems travel as JSON + { nameof(Input), JsonSerializer.Serialize(input) }, + { nameof(Attempt), attempt.ToString() }, + }, + delay, + cancellationToken); + } + + public string? ProjectId { get; set; } + public string? Input { get; set; } + public string? Attempt { get; set; } + + protected override async Task ExecuteJob(IJobExecutionContext context) + { + ArgumentException.ThrowIfNullOrEmpty(ProjectId); + ArgumentException.ThrowIfNullOrEmpty(Input); + ArgumentException.ThrowIfNullOrEmpty(Attempt); + if (!Guid.TryParse(ProjectId, out var projectId)) throw new ArgumentException($"Invalid project ID {ProjectId}", nameof(ProjectId)); + if (!int.TryParse(Attempt, out var attempt)) throw new ArgumentException($"Invalid number {Attempt}", nameof(Attempt)); + var input = JsonSerializer.Deserialize(Input) + ?? throw new ArgumentException("Missing creation input", nameof(Input)); + + var project = await dbContext.Projects.FindAsync(projectId); + if (project is null) + { + logger.LogInformation("Project {ProjectId} no longer exists, nothing to initialise", projectId); + return; + } + if (project.FwDataInitStatus != FwDataInitStatus.InProgress) + { + // An earlier run finished the job and this is a duplicate (e.g. a recovered run) + logger.LogInformation("Project {ProjectId} ({Code}) has already been initialised", projectId, project.Code); + return; + } + + var outcome = await AttemptInit(fwHeadlessClient, hgService, logger, projectId, project.Code, input); + switch (outcome.Result) + { + case InitAttemptResult.Created: + await projectService.FinishFwDataInit(projectId); + logger.LogInformation("Initialised project {ProjectId} ({Code}) from the template", projectId, project.Code); + return; + + case InitAttemptResult.Failed: + logger.LogError("Creating project {ProjectId} ({Code}) from the template failed, rolling it back: {Error}", + projectId, project.Code, outcome.Detail); + // Before the rollback, so a poller that finds the project gone always finds the reason too + RecordFailure(memoryCache, projectId, outcome.Detail); + try + { + await projectService.CleanupFailedProjectCreation(projectId, project.Code); + return; + } + catch (Exception ex) + { + // The project is still marked as being initialised, so a later attempt starts over: it + // either creates the project after all, or fails again and retries the rollback + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code})", projectId, project.Code); + } + break; + + case InitAttemptResult.RetryLater: + logger.LogWarning("Couldn't tell whether project {ProjectId} ({Code}) was created (attempt {Attempt} of {MaxAttempts}): {Reason}", + projectId, project.Code, attempt, MaxAttempts, outcome.Detail); + break; + } + + if (attempt >= MaxAttempts) + { + logger.LogError( + "Giving up on initialising project {ProjectId} ({Code}) after {Attempts} attempts; it is still marked as being initialised and needs manual cleanup", + projectId, project.Code, attempt); + return; + } + await Queue(schedulerFactory, projectId, input, attempt + 1, RetryDelay); + } + + /// + /// One attempt at getting the project populated, picking up wherever an earlier attempt got to. + /// Never throws: anything unexpected means the outcome is unknown, so it's reported as RetryLater. + /// + /// + /// None of these calls take a cancellation token, for the same reason ProjectController used not to + /// pass one: FwHeadless finishes a creation whatever happens to our side of the connection, so + /// abandoning the wait would only stop us learning the outcome. If LexBox shuts down mid-attempt, + /// Quartz recovers the job and the next run picks the creation up again. + /// + public static async Task AttemptInit(FwHeadlessClient fwHeadlessClient, + IHgService hgService, + ILogger logger, + Guid projectId, + string code, + InitFwDataProjectInput input) + { + try + { + // FwHeadless's push is the last step of a creation, so a non-empty repo means an earlier attempt + // succeeded, even if we never heard back. Trust this over FwHeadless's own status, which says + // Created as soon as the .fwdata exists on its disk, even if FwHeadless died before pushing it. + if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created; + + var status = await fwHeadlessClient.CreationStatus(projectId); + if (status?.Status == ProjectCreationStatusEnum.Creating) + { + // An earlier attempt is still running in FwHeadless (e.g. this job was interrupted and + // recovered); starting another would be refused, so wait for that one instead + status = await fwHeadlessClient.AwaitCreationFinished(projectId); + if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created; + return status?.Status == ProjectCreationStatusEnum.CreationFailed + ? InitAttemptOutcome.Failed(status.Error ?? "FwHeadless reported that the creation failed") + : InitAttemptOutcome.RetryLater($"Waiting for FwHeadless's creation ended with {status?.Status.ToString() ?? "no status"}"); + } + + // Clear anything an interrupted attempt left in FwHeadless's local project folder, which would + // otherwise break the new attempt. Throws ProjectSyncInProgressException if FwHeadless is busy + // with this project. + await fwHeadlessClient.DeleteProject(projectId); + var (statusCode, error) = await fwHeadlessClient.InitFwDataProject(projectId, input.WsVernacular, input.WsAnalysis, input.WsUi); + if (error is null) return InitAttemptOutcome.Created; + // FwHeadless refused to start because something else holds the project, so nothing was built + if (statusCode == HttpStatusCode.Conflict) return InitAttemptOutcome.RetryLater(error); + // FwHeadless answered with a failure, so the creation isn't running and FwHeadless has already + // cleaned up its own local state + return InitAttemptOutcome.Failed(error); + } + catch (Exception ex) + { + // Most likely FwHeadless is unreachable (e.g. restarting) or our request timed out. FwHeadless + // may still be creating the project, so only a later attempt can tell how this ended. + logger.LogWarning(ex, "Attempt to initialise project {ProjectId} ({Code}) ended without an answer", projectId, code); + return InitAttemptOutcome.RetryLater(ex.Message); + } + } + + private static async Task RepoHasCommits(IHgService hgService, string code) => + await hgService.GetTipHash(code) != HgService.AllZeroHash; + + // In memory only: once the rollback has deleted the project row, this is all that's left to tell a + // poller why. It doesn't survive a LexBox restart, after which the poller sees "not found". + private static string FailureCacheKey(Guid projectId) => $"InitFwDataProjectFailure|{projectId}"; + + private static void RecordFailure(IMemoryCache memoryCache, Guid projectId, string? error) => + memoryCache.Set(FailureCacheKey(projectId), error ?? "Creating the project from the template failed", TimeSpan.FromHours(1)); + + public static string? TryGetRecentFailure(IMemoryCache memoryCache, Guid projectId) => + memoryCache.Get(FailureCacheKey(projectId)); +} + +public enum InitAttemptResult +{ + Created, + // FwHeadless definitely isn't creating the project and didn't create it + Failed, + // The outcome is unknown, e.g. FwHeadless was unreachable or is still busy + RetryLater, +} + +public record InitAttemptOutcome(InitAttemptResult Result, string? Detail = null) +{ + public static InitAttemptOutcome Created { get; } = new(InitAttemptResult.Created); + public static InitAttemptOutcome Failed(string error) => new(InitAttemptResult.Failed, error); + public static InitAttemptOutcome RetryLater(string reason) => new(InitAttemptResult.RetryLater, reason); +} diff --git a/backend/LexBoxApi/ScheduledTasksKernel.cs b/backend/LexBoxApi/ScheduledTasksKernel.cs index 5293d20502..5345071d68 100644 --- a/backend/LexBoxApi/ScheduledTasksKernel.cs +++ b/backend/LexBoxApi/ScheduledTasksKernel.cs @@ -37,6 +37,9 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur q.AddJob(DeleteTempDirectoryJob.Key, j => j.StoreDurably()); q.AddJob(UpdateProjectMetadataJob.Key, j => j.StoreDurably()); q.AddJob(RetryEmailJob.Key, j => j.StoreDurably()); + // RequestRecovery: rerun a creation that was interrupted by LexBox stopping, rather than leave the + // project marked as being initialised forever. The job is written to be safe to rerun. + q.AddJob(InitFwDataProjectJob.Key, j => j.StoreDurably().RequestRecovery()); q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key) .WithIdentity("WeeklyCleanupTrigger") //every sunday at 2am diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 8aa3d17ec9..db663ad2f7 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -22,7 +22,7 @@ public partial class HgService : IHgService, IHostedService private const string DELETED_REPO_FOLDER = ProjectCode.DELETED_REPO_FOLDER; private const string TEMP_REPO_FOLDER = ProjectCode.TEMP_REPO_FOLDER; - private const string AllZeroHash = "0000000000000000000000000000000000000000"; + public const string AllZeroHash = "0000000000000000000000000000000000000000"; private readonly IOptions _options; private readonly Lazy _hgClient; diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index d90c7e50aa..b1ddfdd8c4 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -25,7 +25,9 @@ public class ProjectService( IEmailService emailService, FwHeadlessClient fwHeadless) { - public async Task CreateProject(CreateProjectInput input, ProjectMigrationStatus? projectOrigin = null) + public async Task CreateProject(CreateProjectInput input, + ProjectMigrationStatus? projectOrigin = null, + FwDataInitStatus fwDataInitStatus = FwDataInitStatus.None) { await using var transaction = await dbContext.Database.BeginTransactionAsync(); var projectId = input.Id ?? Guid.NewGuid(); @@ -42,6 +44,7 @@ public async Task CreateProject(CreateProjectInput input, ProjectMigration Code = input.Code, Name = input.Name, ProjectOrigin = projectOrigin ?? ProjectMigrationStatus.Migrated, + FwDataInitStatus = fwDataInitStatus, Description = input.Description, Type = input.Type, LastCommit = null, @@ -232,12 +235,25 @@ public async Task FinishReset(string code, Stream? zipFile = null) return project; } + /// + /// Marks a project whose repo FwHeadless has populated from the template as ready, recording the + /// commit it pushed. + /// + public async Task FinishFwDataInit(Guid projectId) + { + var project = await dbContext.Projects.FindAsync(projectId); + if (project is null) return; + project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code); + project.FwDataInitStatus = FwDataInitStatus.None; + await dbContext.SaveChangesAsync(); + } + /// /// Compensating cleanup for a project whose template population failed after CreateProject: /// deletes FwHeadless's local project state, then removes the repo and the project row and /// invalidates the same caches as a permanent delete. /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a - /// project this same request just created. + /// project that InitFwDataProject just created. /// public async Task CleanupFailedProjectCreation(Guid projectId, string code) { diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index e554061755..dcdd5718bd 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -25,6 +25,7 @@ public class Project : EntityBase public int? RepoSizeInKb { get; set; } public DateTimeOffset? DeletedDate { get; set; } public ResetStatus ResetStatus { get; set; } = ResetStatus.None; + public FwDataInitStatus FwDataInitStatus { get; set; } = FwDataInitStatus.None; //historical reference for if this project originated here (migrated), or came from redmine, public or private public ProjectMigrationStatus ProjectOrigin { get; set; } = ProjectMigrationStatus.Migrated; @@ -106,6 +107,13 @@ public enum ResetStatus InProgress = 1 } +public enum FwDataInitStatus +{ + None = 0, + // Created by InitFwDataProject, and FwHeadless hasn't yet finished populating its repo from the template + InProgress = 1 +} + public enum ProjectType { Unknown = 0, diff --git a/backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.Designer.cs b/backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.Designer.cs new file mode 100644 index 0000000000..4d342fcf16 --- /dev/null +++ b/backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.Designer.cs @@ -0,0 +1,1422 @@ +// +using System; +using System.Collections.Generic; +using LexData; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace LexData.Migrations +{ + [DbContext(typeof(LexBoxDbContext))] + [Migration("20261103044859_AddProjectFwDataInitStatus")] + partial class AddProjectFwDataInitStatus + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "10.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("bytea") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_blob_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("qrtz_calendars", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_cron_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("text") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_qrtz_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_qrtz_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_qrtz_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_qrtz_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_nm_gp"); + + b.ToTable("qrtz_fired_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bool") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bool") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_j_req_recovery"); + + b.ToTable("qrtz_job_details", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("text") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("qrtz_locks", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("qrtz_paused_trigger_grps", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("qrtz_scheduler_state", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bool") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bool") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("integer") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("integer") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("text") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("text") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("text") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_simprop_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_simple_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("smallint") + .HasColumnName("misfire_instr"); + + b.Property("MisfireOriginalFireTime") + .HasColumnType("bigint") + .HasColumnName("misfire_orig_fire_time"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_qrtz_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_qrtz_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_qrtz_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("qrtz_triggers", "quartz"); + }); + + modelBuilder.Entity("LexCore.Entities.DraftProject", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsConfidential") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("ProjectManagerId") + .HasColumnType("uuid"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ProjectManagerId"); + + b.ToTable("DraftProjects"); + }); + + modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b => + { + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("FlexModelVersion") + .HasColumnType("integer"); + + b.Property("LangProjectId") + .HasColumnType("uuid"); + + b.Property("LexEntryCount") + .HasColumnType("integer"); + + b.HasKey("ProjectId"); + + b.ToTable("FlexProjectMetadata"); + }); + + modelBuilder.Entity("LexCore.Entities.MediaFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("text"); + + b.Property("Metadata") + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValueSql("'{}'"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Revision") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("LexCore.Entities.OrgMember", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("OrgId"); + + b.HasIndex("UserId", "OrgId") + .IsUnique(); + + b.ToTable("OrgMembers", (string)null); + }); + + modelBuilder.Entity("LexCore.Entities.OrgProjects", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("OrgId", "ProjectId") + .IsUnique(); + + b.ToTable("OrgProjects"); + }); + + modelBuilder.Entity("LexCore.Entities.Organization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Orgs", (string)null); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("DeletedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("FwDataInitStatus") + .HasColumnType("integer"); + + b.Property("IsConfidential") + .HasColumnType("boolean"); + + b.Property("LastCommit") + .HasColumnType("timestamp with time zone"); + + b.Property("MigratedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("ProjectOrigin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("RepoSizeInKb") + .HasColumnType("integer"); + + b.Property("ResetStatus") + .HasColumnType("integer"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ParentId"); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("UserId", "ProjectId") + .IsUnique(); + + b.ToTable("ProjectUsers"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CanCreateProjects") + .HasColumnType("boolean"); + + b.Property("CreatedById") + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Email") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("EmailVerified") + .HasColumnType("boolean"); + + b.Property>("FeatureFlags") + .HasColumnType("text[]"); + + b.Property("GoogleId") + .HasColumnType("text"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("LastActive") + .HasColumnType("timestamp with time zone"); + + b.Property("LocalizationCode") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("en"); + + b.Property("Locked") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("PasswordStrength") + .HasColumnType("integer"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Username") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("Email") + .IsUnique(); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ClientId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ClientSecret") + .HasColumnType("text"); + + b.Property("ClientType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ConsentType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("DisplayNames") + .HasColumnType("text"); + + b.Property("JsonWebKeySet") + .HasColumnType("text"); + + b.Property("Permissions") + .HasColumnType("text"); + + b.Property("PostLogoutRedirectUris") + .HasColumnType("text"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RedirectUris") + .HasColumnType("text"); + + b.Property("Requirements") + .HasColumnType("text"); + + b.Property("Settings") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ClientId") + .IsUnique(); + + b.ToTable("OpenIddictApplications", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("Scopes") + .HasColumnType("text"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)"); + + b.Property("Type") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("Id"); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type"); + + b.ToTable("OpenIddictAuthorizations", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreScope", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Descriptions") + .HasColumnType("text"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("DisplayNames") + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("Resources") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("OpenIddictScopes", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("AuthorizationId") + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Payload") + .HasColumnType("text"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RedemptionDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ReferenceId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)"); + + b.Property("Type") + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.HasKey("Id"); + + b.HasIndex("AuthorizationId"); + + b.HasIndex("ReferenceId") + .IsUnique(); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type"); + + b.ToTable("OpenIddictTokens", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChangeEntities") + .HasColumnType("jsonb"); + + b.Property("ClientId") + .HasColumnType("uuid"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 => + { + b1.IsRequired(); + + b1.Property("Counter") + .HasColumnType("bigint"); + + b1.Property("DateTime") + .HasColumnType("timestamp with time zone"); + }); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("CrdtCommits", (string)null); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("LexCore.Entities.DraftProject", b => + { + b.HasOne("LexCore.Entities.User", "ProjectManager") + .WithMany() + .HasForeignKey("ProjectManagerId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ProjectManager"); + }); + + modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithOne("FlexProjectMetadata") + .HasForeignKey("LexCore.Entities.FlexProjectMetadata", "ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("LexCore.Entities.ProjectWritingSystems", "WritingSystems", b1 => + { + b1.Property("FlexProjectMetadataProjectId"); + + b1.HasKey("FlexProjectMetadataProjectId"); + + b1.ToTable("FlexProjectMetadata"); + + b1 + .ToJson("WritingSystems") + .HasColumnType("jsonb"); + + b1.WithOwner() + .HasForeignKey("FlexProjectMetadataProjectId"); + + b1.OwnsMany("LexCore.Entities.FLExWsId", "AnalysisWss", b2 => + { + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); + + b2.Property("__synthesizedOrdinal") + .ValueGeneratedOnAdd(); + + b2.Property("IsActive"); + + b2.Property("IsDefault"); + + b2.Property("Tag") + .IsRequired(); + + b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); + + b2.ToTable("FlexProjectMetadata"); + + b2.WithOwner() + .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId"); + }); + + b1.OwnsMany("LexCore.Entities.FLExWsId", "VernacularWss", b2 => + { + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); + + b2.Property("__synthesizedOrdinal") + .ValueGeneratedOnAdd(); + + b2.Property("IsActive"); + + b2.Property("IsDefault"); + + b2.Property("Tag") + .IsRequired(); + + b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); + + b2.ToTable("FlexProjectMetadata"); + + b2.WithOwner() + .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId"); + }); + + b1.Navigation("AnalysisWss"); + + b1.Navigation("VernacularWss"); + }); + + b.Navigation("WritingSystems"); + }); + + modelBuilder.Entity("LexCore.Entities.MediaFile", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("LexCore.Entities.OrgMember", b => + { + b.HasOne("LexCore.Entities.Organization", "Organization") + .WithMany("Members") + .HasForeignKey("OrgId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Organizations") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Organization"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.OrgProjects", b => + { + b.HasOne("LexCore.Entities.Organization", "Org") + .WithMany() + .HasForeignKey("OrgId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Org"); + + b.Navigation("Project"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ParentId"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany("Users") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Projects") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Project"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.HasOne("LexCore.Entities.User", "CreatedBy") + .WithMany("UsersICreated") + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Authorizations") + .HasForeignKey("ApplicationId"); + + b.Navigation("Application"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Tokens") + .HasForeignKey("ApplicationId"); + + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", "Authorization") + .WithMany("Tokens") + .HasForeignKey("AuthorizationId"); + + b.Navigation("Application"); + + b.Navigation("Authorization"); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany("HarmonyCommits") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); + + modelBuilder.Entity("LexCore.Entities.Organization", b => + { + b.Navigation("Members"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Navigation("FlexProjectMetadata"); + + b.Navigation("HarmonyCommits"); + + b.Navigation("Users"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Navigation("Organizations"); + + b.Navigation("Projects"); + + b.Navigation("UsersICreated"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Navigation("Authorizations"); + + b.Navigation("Tokens"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Navigation("Tokens"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.cs b/backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.cs new file mode 100644 index 0000000000..7fd0ffdad8 --- /dev/null +++ b/backend/LexData/Migrations/20261103044859_AddProjectFwDataInitStatus.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace LexData.Migrations +{ + /// + public partial class AddProjectFwDataInitStatus : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "FwDataInitStatus", + table: "Projects", + type: "integer", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "FwDataInitStatus", + table: "Projects"); + } + } +} diff --git a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs index d4199964ea..0ad21a65dd 100644 --- a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs +++ b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs @@ -19,7 +19,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) #pragma warning disable 612, 618 modelBuilder .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "und-u-ks-level2,und-u-ks-level2,icu,False") - .HasAnnotation("ProductVersion", "9.0.16") + .HasAnnotation("ProductVersion", "10.0.8") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); @@ -694,6 +694,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Description") .HasColumnType("text"); + b.Property("FwDataInitStatus") + .HasColumnType("integer"); + b.Property("IsConfidential") .HasColumnType("boolean"); @@ -1086,7 +1089,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("ProjectId") .HasColumnType("uuid"); - b.ComplexProperty>("HybridDateTime", "SIL.Harmony.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 => + b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 => { b1.IsRequired(); @@ -1179,8 +1182,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.OwnsOne("LexCore.Entities.ProjectWritingSystems", "WritingSystems", b1 => { - b1.Property("FlexProjectMetadataProjectId") - .HasColumnType("uuid"); + b1.Property("FlexProjectMetadataProjectId"); b1.HasKey("FlexProjectMetadataProjectId"); @@ -1195,22 +1197,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b1.OwnsMany("LexCore.Entities.FLExWsId", "AnalysisWss", b2 => { - b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId") - .HasColumnType("uuid"); + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); b2.Property("__synthesizedOrdinal") - .ValueGeneratedOnAdd() - .HasColumnType("integer"); + .ValueGeneratedOnAdd(); - b2.Property("IsActive") - .HasColumnType("boolean"); + b2.Property("IsActive"); - b2.Property("IsDefault") - .HasColumnType("boolean"); + b2.Property("IsDefault"); b2.Property("Tag") - .IsRequired() - .HasColumnType("text"); + .IsRequired(); b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); @@ -1222,22 +1219,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b1.OwnsMany("LexCore.Entities.FLExWsId", "VernacularWss", b2 => { - b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId") - .HasColumnType("uuid"); + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); b2.Property("__synthesizedOrdinal") - .ValueGeneratedOnAdd() - .HasColumnType("integer"); + .ValueGeneratedOnAdd(); - b2.Property("IsActive") - .HasColumnType("boolean"); + b2.Property("IsActive"); - b2.Property("IsDefault") - .HasColumnType("boolean"); + b2.Property("IsDefault"); b2.Property("Tag") - .IsRequired() - .HasColumnType("text"); + .IsRequired(); b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); @@ -1365,7 +1357,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => { b.HasOne("LexCore.Entities.Project", null) - .WithMany() + .WithMany("HarmonyCommits") .HasForeignKey("ProjectId") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); @@ -1396,6 +1388,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) { b.Navigation("FlexProjectMetadata"); + b.Navigation("HarmonyCommits"); + b.Navigation("Users"); }); diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs new file mode 100644 index 0000000000..0029e420fb --- /dev/null +++ b/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs @@ -0,0 +1,196 @@ +using System.Net; +using System.Net.Http.Json; +using LexBoxApi.Jobs; +using LexBoxApi.Services; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexCore.Sync; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Testing.LexBoxApi; + +/// +/// Unit tests for how one run of InitFwDataProjectJob decides what happened to a creation. Every run must be +/// safe to repeat (Quartz reruns the job after an interruption), so these cover picking up after each way an +/// earlier run could have stopped. FwHeadless is a fake HTTP handler and the repo is a mocked IHgService. +/// +public class InitFwDataProjectJobTests +{ + private const string Code = "test-proj"; + private const string PushedTip = "1234567890abcdef1234567890abcdef12345678"; + private static readonly Guid ProjectId = Guid.NewGuid(); + private static readonly InitFwDataProjectInput Input = new(["fr"], ["en"], "en"); + + private string _tip = HgService.AllZeroHash; + private readonly List _requests = []; + private readonly Mock _hgService = new(); + + // Responses by request, as "METHOD path". An unlisted request throws, which AttemptInit reports as + // RetryLater like any other exception, so the tests also check exactly which requests were made. + private readonly Dictionary> _fwHeadless = []; + + private const string CreationStatus = "GET /api/project/creation-status"; + private const string AwaitCreation = "GET /api/project/await-creation-finished"; + private static readonly string DeleteLocalProject = $"DELETE /api/manage/project/{ProjectId}"; + private const string InitProject = "POST /api/project/initFwDataProject"; + + public InitFwDataProjectJobTests() + { + _hgService.Setup(h => h.GetTipHash(Code, It.IsAny())).ReturnsAsync(() => _tip); + } + + private Task Attempt() + { + var client = new FwHeadlessClient(new HttpClient(new FakeHandler(this)) { BaseAddress = new Uri("http://fwheadless/") }, + NullLogger.Instance); + return InitFwDataProjectJob.AttemptInit(client, _hgService.Object, NullLogger.Instance, ProjectId, Code, Input); + } + + private static HttpResponseMessage Status(ProjectCreationStatus status) => + new(HttpStatusCode.OK) { Content = JsonContent.Create(status) }; + + private static HttpResponseMessage Response(HttpStatusCode code, string body = "") => + new(code) { Content = new StringContent(body) }; + + [Fact] + public async Task A_project_whose_push_already_landed_is_created_without_asking_FwHeadless() + { + _tip = PushedTip; + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().BeEmpty("the repo already shows that an earlier run's creation succeeded"); + } + + [Fact] + public async Task Waits_for_a_creation_still_running_in_FwHeadless_instead_of_starting_another() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => + { + _tip = PushedTip; + return Status(ProjectCreationStatus.Created); + }; + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().Equal(CreationStatus, AwaitCreation); + } + + [Fact] + public async Task A_creation_FwHeadless_reports_as_failed_while_waiting_is_a_failure() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => Status(ProjectCreationStatus.Failed("the push failed")); + + var outcome = await Attempt(); + + outcome.Should().Be(InitAttemptOutcome.Failed("the push failed")); + } + + [Fact] + public async Task A_creation_still_running_after_the_wait_is_retried_later() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => Status(ProjectCreationStatus.TimedOutAwaitingCreation); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, AwaitCreation); + } + + [Fact] + public async Task Clears_FwHeadless_local_state_before_starting_a_creation() + { + _fwHeadless[CreationStatus] = () => Response(HttpStatusCode.NotFound); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => Response(HttpStatusCode.OK); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task FwHeadless_being_busy_with_the_project_is_retried_later_without_starting_a_creation() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.Conflict); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, DeleteLocalProject); + } + + [Theory] + [InlineData(HttpStatusCode.Conflict, InitAttemptResult.RetryLater)] // something else holds the project, so nothing was built + [InlineData(HttpStatusCode.BadRequest, InitAttemptResult.Failed)] + [InlineData(HttpStatusCode.InternalServerError, InitAttemptResult.Failed)] + public async Task An_answer_from_FwHeadless_decides_the_outcome(HttpStatusCode statusCode, InitAttemptResult expected) + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => Response(statusCode, "what went wrong"); + + var outcome = await Attempt(); + + outcome.Should().Be(new InitAttemptOutcome(expected, "what went wrong")); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task No_answer_from_FwHeadless_is_retried_later_because_it_may_still_be_creating() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => throw new HttpRequestException("Connection refused"); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task Not_being_able_to_read_the_repo_is_retried_later() + { + _hgService.Setup(h => h.GetTipHash(Code, It.IsAny())).ThrowsAsync(new HttpRequestException("hg is down")); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().BeEmpty(); + } + + [Fact] + public void The_creation_input_survives_the_job_store() + { + // The job store only takes strings, so the input travels as JSON; it's a positional record with + // IReadOnlyList properties, which System.Text.Json has to bind through the constructor + var roundTripped = System.Text.Json.JsonSerializer.Deserialize( + System.Text.Json.JsonSerializer.Serialize(new InitFwDataProjectInput(["fr", "es"], ["en", "de"], "pt"))); + + roundTripped.Should().NotBeNull(); + roundTripped!.WsVernacular.Should().Equal("fr", "es"); + roundTripped.WsAnalysis.Should().Equal("en", "de"); + roundTripped.WsUi.Should().Be("pt"); + } + + private class FakeHandler(InitFwDataProjectJobTests test) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var key = $"{request.Method} {request.RequestUri!.AbsolutePath}"; + test._requests.Add(key); + if (!test._fwHeadless.TryGetValue(key, out var respond)) + throw new InvalidOperationException($"Unexpected request to FwHeadless: {key}"); + return Task.FromResult(respond()); + } + } +} diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index 9b49e8ce20..f32dbc745b 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -21,7 +21,7 @@ private static ProjectController NewController(IPermissionService? permissionSer var services = new ServiceCollection(); services.AddLogging(); services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves - return new ProjectController(null!, null!, null!, permissionService!, null!, null!, NullLogger.Instance) + return new ProjectController(null!, null!, null!, permissionService!, null!, NullLogger.Instance) { ControllerContext = new ControllerContext { diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index d99dd1dce2..33c30b48d4 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -5,6 +5,7 @@ using System.Xml.Linq; using FluentAssertions; using LexCore.Entities; +using LexCore.Sync; using Testing.ApiTests; using Testing.Fixtures; using Testing.Services; @@ -45,22 +46,25 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting Guid projectId = default; try { - // 1. Create the project via the admin endpoint (creates the DB row + empty repo, then has - // FwHeadless build the template .fwdata and push it into the empty repo). - // This is a long-running, non-idempotent operation. The shared tester's HttpClient retries - // transient failures/timeouts, so a slow first attempt (which already created the project - // row) would be retried and the retry would return 409 "already exists". Use a client that - // doesn't retry and allows more time. Log it in itself (cookie auth) rather than reusing the - // shared tester's captured JWT, which may be stale. + // 1. Create the project via the admin endpoint. It creates the DB row + empty repo and returns + // 202 at once; a background job then has FwHeadless build the template .fwdata and push it + // into the empty repo, and we poll until that has finished. + // The POST isn't idempotent. The shared tester's HttpClient retries transient failures, and a + // retried attempt (after the first already created the project row) would return 409 "already + // exists", so use a client that doesn't retry. Log it in itself (cookie auth) rather than + // reusing the shared tester's captured JWT, which may be stale. using var createClient = ApiTestBase.NewHttpClient(_adminApiTester.BaseUrl, retryTransientFailures: false).Client; - createClient.Timeout = TimeSpan.FromMinutes(5); await JwtHelper.ExecuteLogin(AdminAuth, includeDefaultScope: true, createClient); var response = await createClient.PostAsync( $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject{query}", null); - response.StatusCode.Should().Be(HttpStatusCode.OK, - "creation should succeed; body: {0}", await response.Content.ReadAsStringAsync()); + response.StatusCode.Should().Be(HttpStatusCode.Accepted, + "creation should be accepted; body: {0}", await response.Content.ReadAsStringAsync()); projectId = await response.Content.ReadFromJsonAsync(); projectId.Should().NotBe(Guid.Empty); + response.Headers.Location?.ToString().Should().EndWith($"/api/project/fwDataProjectCreationStatus/{projectId}"); + + var status = await AwaitCreationFinished(createClient, projectId); + status.Status.Should().Be(ProjectCreationStatusEnum.Created, "creation should succeed; error: {0}", status.Error); // 2. The first push landed a commit server-side (the empty repo is no longer empty). var lastCommit = await _adminApiTester.GetProjectLastCommit(code); @@ -184,6 +188,24 @@ public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() response.StatusCode.Should().Be(HttpStatusCode.BadRequest); } + /// Polls the creation status until the background job has finished, for up to five minutes. + private async Task AwaitCreationFinished(HttpClient client, Guid projectId) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + while (true) + { + var response = await client.GetAsync( + $"{_adminApiTester.BaseUrl}/api/project/fwDataProjectCreationStatus/{projectId}", timeout.Token); + response.StatusCode.Should().Be(HttpStatusCode.OK, + "the project's creation status should be known; body: {0}", await response.Content.ReadAsStringAsync()); + var status = await response.Content.ReadFromJsonAsync(timeout.Token); + status.Should().NotBeNull(); + _output.WriteLine($"Creation status: {status!.Status}"); + if (status.IsFinished) return status; + await Task.Delay(TimeSpan.FromSeconds(2), timeout.Token); + } + } + // Reads a LangProject child element's text (a space-separated writing-system list) and splits it. private static string[] SpaceSeparatedUni(XElement langProject, string elementName) { diff --git a/frontend/schema.graphql b/frontend/schema.graphql index 6a4c75ddba..48ea090b12 100644 --- a/frontend/schema.graphql +++ b/frontend/schema.graphql @@ -450,6 +450,7 @@ type Project { repoSizeInKb: Int deletedDate: DateTime resetStatus: ResetStatus! + fwDataInitStatus: FwDataInitStatus! projectOrigin: ProjectMigrationStatus! migratedDate: DateTime userCount: Int! @@ -976,6 +977,13 @@ input FlexProjectMetadataSortInput { flexModelVersion: SortEnumType @cost(weight: "10") } +input FwDataInitStatusOperationFilterInput { + eq: FwDataInitStatus @cost(weight: "10") + neq: FwDataInitStatus @cost(weight: "10") + in: [FwDataInitStatus!] @cost(weight: "10") + nin: [FwDataInitStatus!] @cost(weight: "10") +} + input HybridDateTimeFilterInput { and: [HybridDateTimeFilterInput!] or: [HybridDateTimeFilterInput!] @@ -1224,6 +1232,7 @@ input ProjectFilterInput { repoSizeInKb: IntOperationFilterInput deletedDate: DateTimeOperationFilterInput resetStatus: ResetStatusOperationFilterInput + fwDataInitStatus: FwDataInitStatusOperationFilterInput projectOrigin: ProjectMigrationStatusOperationFilterInput migratedDate: DateTimeOperationFilterInput harmonyCommits: ListFilterInputTypeOfServerCommitFilterInput @@ -1260,6 +1269,7 @@ input ProjectSortInput { repoSizeInKb: SortEnumType @cost(weight: "10") deletedDate: SortEnumType @cost(weight: "10") resetStatus: SortEnumType @cost(weight: "10") + fwDataInitStatus: SortEnumType @cost(weight: "10") projectOrigin: SortEnumType @cost(weight: "10") migratedDate: SortEnumType @cost(weight: "10") userCount: SortEnumType @cost(weight: "10") @@ -1497,6 +1507,11 @@ enum FeatureFlag { FW_LITE_BETA } +enum FwDataInitStatus { + NONE + IN_PROGRESS +} + enum JsonValueKind { UNDEFINED OBJECT From c45152355871d0f0ea456458345f0d1b7089d15a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 3 Nov 2026 14:33:11 +0700 Subject: [PATCH 60/60] Require a name when creating a project from the template InitFwDataProject used to fall back to the project code when no name was given. Make name a required parameter instead, and reject a blank one with a 400 before anything is created. Co-Authored-By: Claude Opus 5.5 --- .../Controllers/ProjectController.cs | 8 +++-- .../InitFwDataProjectValidationTests.cs | 29 ++++++++++++++----- .../InitFwDataProjectTests.cs | 6 ++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index e82c8147ec..739a4001ff 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -40,10 +40,10 @@ public class ProjectController( /// new project's ID. Poll to learn when it's ready. /// /// Project code for the new project. + /// Display name for the new project. /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. /// Optional user interface writing system id; defaults to "en" if not given. - /// Optional display name; defaults to the code. /// Optional to record as the project's origin; admin-only. [HttpPost("initFwDataProject")] [AdminRequired] @@ -53,10 +53,10 @@ public class ProjectController( [ProducesResponseType(StatusCodes.Status500InternalServerError)] public async Task> InitFwDataProject( string code, + string name, [FromQuery] string[] wsVernacular, [FromQuery] string[]? wsAnalysis = null, string wsUi = "en", - string? name = null, string? projectOrigin = null) { // Validate before anything is created, so a bad request never leaves a project to tear down. @@ -68,6 +68,8 @@ public async Task> InitFwDataProject( return Problem(wsError, statusCode: StatusCodes.Status400BadRequest); if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); + if (string.IsNullOrWhiteSpace(name)) + return Problem("A project name is required", statusCode: StatusCodes.Status400BadRequest); ProjectMigrationStatus? origin = null; if (!string.IsNullOrEmpty(projectOrigin)) @@ -87,7 +89,7 @@ public async Task> InitFwDataProject( { projectId = await projectService.CreateProject(new CreateProjectInput( Id: null, - Name: string.IsNullOrWhiteSpace(name) ? code : name, + Name: name, Description: string.Empty, Code: code, Type: ProjectType.FLEx, diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index f32dbc745b..20256dd027 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -33,7 +33,7 @@ private static ProjectController NewController(IPermissionService? permissionSer [Fact] public async Task Rejects_when_no_vernacular_writing_system_is_supplied() { - var result = await NewController().InitFwDataProject("myproj", wsVernacular: []); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: []); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); @@ -44,7 +44,7 @@ public async Task Rejects_when_no_vernacular_writing_system_is_supplied() [Fact] public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_anything() { - var result = await NewController().InitFwDataProject("myproj", wsVernacular: ["fr", "fr"]); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr", "fr"]); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); @@ -55,7 +55,7 @@ public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_ public async Task Rejects_too_many_analysis_writing_systems_before_creating_anything() { var analysis = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); - var result = await NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], wsAnalysis: analysis); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsAnalysis: analysis); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); @@ -66,7 +66,7 @@ public async Task Rejects_too_many_analysis_writing_systems_before_creating_anyt public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything() { var wsUi = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); - var result = await NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], wsUi: wsUi); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsUi: wsUi); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); @@ -76,13 +76,26 @@ public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything( [Fact] public async Task Rejects_an_invalid_project_code() { - var result = await NewController().InitFwDataProject("Bad Code!", wsVernacular: ["fr"]); + var result = await NewController().InitFwDataProject("Bad Code!", name: "My project", wsVernacular: ["fr"]); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); problem.Detail.Should().Contain("Invalid project code"); } + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task Rejects_a_missing_project_name(string? name) + { + var result = await NewController().InitFwDataProject("myproj", name: name!, wsVernacular: ["fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("A project name is required"); + } + /// A permission service that either passes or fails the admin assertion. private static IPermissionService PermissionService(bool isAdmin) { @@ -95,7 +108,7 @@ private static IPermissionService PermissionService(bool isAdmin) public async Task Rejects_a_project_origin_that_is_not_a_migration_status() { var result = await NewController(PermissionService(isAdmin: true)) - .InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin"); + .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin"); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); @@ -108,7 +121,7 @@ public async Task Rejects_a_project_origin_from_a_non_admin() // The admin check must happen before the value is even parsed, and must outlive [AdminRequired], // which is expected to be relaxed for this endpoint later. var act = () => NewController(PermissionService(isAdmin: false)) - .InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: "Migrated"); + .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "Migrated"); await act.Should().ThrowAsync(); } @@ -117,7 +130,7 @@ public async Task Ignores_an_empty_project_origin_without_requiring_admin() { // An empty value isn't "specified", so it needs no admin check; a null permission service // proves AssertIsAdmin() was never called. Fails later (null project service) than validation. - var act = () => NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: ""); + var act = () => NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: ""); await act.Should().ThrowAsync(); } diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs index 33c30b48d4..8ec6054831 100644 --- a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -39,7 +39,7 @@ public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWriting var code = $"tmpl-{Guid.NewGuid():N}"[..12]; var vernacular = new[] { "fr", "es", "ko", "ru", "my" }; var analysis = new[] { "de", "pt", "th", "id", "he", "el" }; - var query = $"?code={code}" + var query = $"?code={code}&name={code}" + string.Concat(vernacular.Select(ws => $"&wsVernacular={ws}")) + string.Concat(analysis.Select(ws => $"&wsAnalysis={ws}")); @@ -148,7 +148,7 @@ public async Task InitFwDataProject_RejectsProjectOriginFromANonAdmin() var code = $"tmpl-{Guid.NewGuid():N}"[..12]; var response = await managerApiTester.HttpClient.PostAsync( $"{managerApiTester.BaseUrl}/api/project/initFwDataProject" + - $"?code={code}&wsVernacular=fr&projectOrigin={nameof(ProjectMigrationStatus.LanguageForgeNonSR)}", null); + $"?code={code}&name={code}&wsVernacular=fr&projectOrigin={nameof(ProjectMigrationStatus.LanguageForgeNonSR)}", null); if (response.StatusCode == HttpStatusCode.InternalServerError) { @@ -184,7 +184,7 @@ public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() { var code = $"tmpl-{Guid.NewGuid():N}"[..12]; var response = await _adminApiTester.HttpClient.PostAsync( - $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject?code={code}&wsAnalysis=en", null); + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject?code={code}&name={code}&wsAnalysis=en", null); response.StatusCode.Should().Be(HttpStatusCode.BadRequest); }