From d288df480580653d0db6c3c5984e35bbdf846628 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 9 Nov 2026 16:03:53 +0700 Subject: [PATCH 1/6] Add an admin API to create a FLEx project with a populated repo POST api/project/initFwDataProject creates the project row (marked FwDataInitStatus.InProgress) and an empty repo, then returns 202 and queues InitFwDataProjectJob. The job asks FwHeadless to build the project from the SIL.LCModel template with the requested writing systems and push it, then marks the project ready, or rolls it back if FwHeadless reports a failure. The job is safe to rerun: it uses the repo's tip as the source of truth and retries while the outcome is unknown. GET api/project/fwDataProjectCreationStatus/{projectId} reports progress. In FwHeadless, ProjectCreationService does hg init, builds the .fwdata, commits FLExProject.CustomProperties on the default branch as rev 0, switches to the send/receive branch with an empty commit (LfMergeBridge won't create a branch itself), then sends/receives. A new project therefore starts with three commits instead of FieldWorks' two. SyncHostedService now also tracks creations, so a sync can't race a creation of the same project (409 on the merge route). Also adds the LanguageForgeNonSR project origin, which admins can pass as projectOrigin. Split out of #2439. Fixes #2589. Co-authored-by: Robin Munn Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/FwHeadless.csproj | 8 + backend/FwHeadless/FwHeadlessConfig.cs | 16 + backend/FwHeadless/FwHeadlessKernel.cs | 9 + backend/FwHeadless/Program.cs | 1 + backend/FwHeadless/Routes/MergeRoutes.cs | 4 + backend/FwHeadless/Routes/ProjectRoutes.cs | 145 ++ .../Services/ISendReceiveService.cs | 2 + .../Services/ProjectCreationService.cs | 216 +++ .../FwHeadless/Services/SendReceiveHelpers.cs | 40 + .../FwHeadless/Services/SendReceiveService.cs | 10 + .../FwHeadless/Services/SyncHostedService.cs | 116 +- .../Controllers/ProjectController.cs | 154 +- .../LexBoxApi/Jobs/InitFwDataProjectJob.cs | 211 +++ backend/LexBoxApi/LexBoxKernel.cs | 8 +- backend/LexBoxApi/ScheduledTasksKernel.cs | 3 + .../LexBoxApi/Services/FwHeadlessClient.cs | 89 + backend/LexBoxApi/Services/HgService.cs | 2 +- backend/LexBoxApi/Services/ProjectService.cs | 53 +- .../Entities/InitFwDataProjectInput.cs | 51 + backend/LexCore/Entities/Project.cs | 9 + backend/LexCore/Sync/ProjectCreationStatus.cs | 43 + ...102_AddProjectFwDataInitStatus.Designer.cs | 1425 +++++++++++++++++ ...261109090102_AddProjectFwDataInitStatus.cs | 29 + .../LexBoxDbContextModelSnapshot.cs | 3 + ...ncHostedServiceCreationReservationTests.cs | 131 ++ .../LexBoxApi/InitFwDataProjectJobTests.cs | 196 +++ .../InitFwDataProjectValidationTests.cs | 176 ++ .../LexCore/InitFwDataProjectInputTests.cs | 106 ++ .../InitFwDataProjectTests.cs | 234 +++ frontend/schema.graphql | 16 + 30 files changed, 3481 insertions(+), 25 deletions(-) create mode 100644 backend/FwHeadless/Routes/ProjectRoutes.cs create mode 100644 backend/FwHeadless/Services/ProjectCreationService.cs create mode 100644 backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs create mode 100644 backend/LexCore/Entities/InitFwDataProjectInput.cs create mode 100644 backend/LexCore/Sync/ProjectCreationStatus.cs create mode 100644 backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs create mode 100644 backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs create mode 100644 backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs create mode 100644 backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs create mode 100644 backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs create mode 100644 backend/Testing/LexCore/InitFwDataProjectInputTests.cs create mode 100644 backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs diff --git a/backend/FwHeadless/FwHeadless.csproj b/backend/FwHeadless/FwHeadless.csproj index 4a6c2721e6..c54ddafba3 100644 --- a/backend/FwHeadless/FwHeadless.csproj +++ b/backend/FwHeadless/FwHeadless.csproj @@ -15,6 +15,9 @@ + + @@ -40,6 +43,11 @@ + + diff --git a/backend/FwHeadless/FwHeadlessConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs index a7edd76b72..84fe6a1863 100644 --- a/backend/FwHeadless/FwHeadlessConfig.cs +++ b/backend/FwHeadless/FwHeadlessConfig.cs @@ -23,6 +23,22 @@ public class FwHeadlessConfig public long MaxUploadFileSizeBytes => MaxUploadFileSizeKb * 1024; public string FdoDataModelVersion { get; init; } = "7000072"; + /// + /// The FLExBridge data version. FieldWorks/LfMergeBridge form the Mercurial branch a project's + /// data lives on by joining this and the FDO model version with a dot (see ). + /// Mirrors FlexBridgeConstants.FlexBridgeDataVersion in FLExBridge, which is internal so we can't + /// reference it directly; bump this if FLExBridge bumps its data version. + /// + public string FlexBridgeDataVersion { get; init; } = "7500002"; + + /// + /// The Mercurial branch FieldWorks/LfMergeBridge keep a project's data on: FlexBridgeDataVersion, + /// a dot, then the FDO model version (e.g. "7500002.7000072"). This is exactly the branch name + /// LfMergeBridge derives from , so a new project's genesis commit + /// must land on this branch for send/receive to find the data. + /// + public string SendReceiveBranchName => $"{FlexBridgeDataVersion}.{FdoDataModelVersion}"; + /// /// Project directory structure in FwHeadless: (Note that FwDataProject.ProjectsPath is the root of a SINGLE project) /// {ProjectStorageRoot}/ diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index e96157c7da..2604d58671 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -29,6 +29,7 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); // Singleton so its file lock is shared across requests and the sync worker @@ -37,6 +38,14 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) .AddLcmCrdtClientCore() .AddFwDataBridge(ServiceLifetime.Scoped) .AddFwLiteProjectSync(); + // The LCM project templates (NewLangProj.fwdata etc.) ship in our output (see FwHeadless.csproj). + // FwDataBridgeConfig's default TemplatesFolder points at the FieldWorks user-data dir, which + // doesn't exist in the container; point it at the shipped templates unless overridden in config. + services.Configure(config => + { + if (config.TemplatesFolder == new FwDataBridgeConfig().TemplatesFolder) + config.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + }); services.RemoveAll(typeof(IMediaAdapter)); services.AddScoped(); services.AddScoped(); diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index b5aa38ef19..643d731b80 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -93,6 +93,7 @@ app.MapMediaFileRoutes(); app.MapMergeRoutes(); app.MapLexiconRoutes(); +app.MapProjectRoutes(); // DELETE endpoint to delete the FieldWorks repo/project (and nothing else) app.MapDelete("/api/manage/repo/{projectId}", async (Guid projectId, ProjectDeletionService deletionService) => diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index a54199a122..010a3ceec3 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -74,6 +74,10 @@ static async Task> ExecuteMergeRequest( switch (syncHostedService.QueueJob(projectId)) { + case QueueJobResult.ProjectBeingCreated: + activity?.SetStatus(ActivityStatusCode.Ok, "Project is still being created"); + return TypedResults.Problem("Project is still being created, try again once creation has finished", + statusCode: StatusCodes.Status409Conflict); case QueueJobResult.SyncAlreadyRunning: activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running"); return TypedResults.Problem("A sync is already running for this project, try again once it has finished", diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs new file mode 100644 index 0000000000..dd3e540c4f --- /dev/null +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -0,0 +1,145 @@ +using System.Diagnostics; +using FwHeadless.Services; +using LexCore.Entities; +using LexCore.Sync; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.Extensions.Options; +using SIL.WritingSystems; + +namespace FwHeadless.Routes; + +public static class ProjectRoutes +{ + public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app) + { + var group = app.MapGroup("/api/project"); + group.MapPost("/initFwDataProject", InitFwDataProject); + group.MapGet("/creation-status", GetCreationStatus); + group.MapGet("/await-creation-finished", AwaitCreationFinished); + return group; + } + + // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the + // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the + // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates. + private static async Task> InitFwDataProject( + Guid projectId, + InitFwDataProjectInput input, + IProjectLookupService projectLookupService, + ProjectCreationService projectCreationService, + ILogger logger) + { + // Null-safe against a malformed body (System.Text.Json leaves an absent list null). + var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", input.WsVernacular) + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", input.WsAnalysis) + ?? InitFwDataProjectInput.ValidateUiWritingSystem(input.WsUi); + if (wsError is not null) + return TypedResults.Problem(wsError, statusCode: StatusCodes.Status400BadRequest); + var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); + if (invalidWs is not null) + return TypedResults.Problem($"Invalid writing system code: {invalidWs}", + statusCode: StatusCodes.Status400BadRequest); + if (!IetfLanguageTag.IsValid(input.WsUi)) + return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", + statusCode: StatusCodes.Status400BadRequest); + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) + { + logger.LogError("initFwDataProject request for non-existent project {ProjectId}", projectId); + return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound); + } + + await projectCreationService.InitFwDataProject( + projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi); + return TypedResults.Ok(); + } + + // Instantaneous creation status, for a caller that wants to poll. Mirrors /api/merge/status. + private static async Task, NotFound>> GetCreationStatus( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.project_id", projectId); + + var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config); + if (status is null) + { + activity?.SetStatus(ActivityStatusCode.Error, "Project not found"); + return TypedResults.NotFound(); + } + activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString()); + return TypedResults.Ok(status); + } + + // Long-poll until an in-flight creation finishes. Mirrors /api/merge/await-finished: giving up on the + // wait is reported as a status, not an error, because the creation itself keeps running either way. + private static async Task, NotFound>> AwaitCreationFinished( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config, + CancellationToken cancellationToken) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.project_id", projectId); + + try + { + var result = await syncHostedService.AwaitCreationFinished(projectId, cancellationToken); + if (result is not null) + { + activity?.SetStatus(ActivityStatusCode.Ok, result.Status.ToString()); + return TypedResults.Ok(result); + } + } + catch (OperationCanceledException) + { + // The caller's wait was cancelled; the creation was not (it takes no cancellation token). + // Report that distinctly so a caller can't read a timeout as a failed creation. + activity?.SetStatus(ActivityStatusCode.Unset, "Timed out awaiting creation"); + return TypedResults.Ok(ProjectCreationStatus.TimedOutAwaitingCreation); + } + + // Nothing in flight and nothing remembered, so fall back to what the project itself tells us. + var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config); + if (status is null) + { + activity?.SetStatus(ActivityStatusCode.Error, "Project not found"); + return TypedResults.NotFound(); + } + activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString()); + return TypedResults.Ok(status); + } + + /// + /// Creation status for a project, or null when LexBox has no such project. In-memory state wins + /// when we have it; otherwise the status is derived from durable state, so it survives a FwHeadless + /// restart and a result that has aged out of the cache. + /// + private static async Task DetermineCreationStatus( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config) + { + if (syncHostedService.IsProjectBeingCreated(projectId)) return ProjectCreationStatus.Creating; + var recent = syncHostedService.TryGetRecentCreationResult(projectId); + if (recent is not null) return recent; + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) return null; + + // A successful creation leaves the .fwdata on disk; a failed one deletes the project folder + // (ProjectCreationService.CleanupLocalProject). Note this says "project data is here", not "this + // process created it" -- a project cloned by an ordinary sync looks the same, which is the right + // answer for a caller asking whether the project is ready to use. + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + return File.Exists(fwDataProject.FilePath) + ? ProjectCreationStatus.Created + : ProjectCreationStatus.NotCreated; + } +} diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs index 4755ff1ac8..b1835ec7f5 100644 --- a/backend/FwHeadless/Services/ISendReceiveService.cs +++ b/backend/FwHeadless/Services/ISendReceiveService.cs @@ -10,4 +10,6 @@ public interface ISendReceiveService Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode); Task PendingCommitCountBothWays(FwDataProject project, string? projectCode); Task CommitFile(string filePath, string commitMessage); + Task InitRepo(string folder); + Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = ""); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs new file mode 100644 index 0000000000..2f455f8b57 --- /dev/null +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -0,0 +1,216 @@ +using System.Xml; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.LcmUtils; +using LexCore.Exceptions; +using LexCore.Sync; +using Microsoft.Extensions.Options; + +namespace FwHeadless.Services; + +/// +/// Creates a brand-new FieldWorks project from the SIL.LCModel template and pushes it into the +/// (empty) Mercurial repo that LexBox has already initialised for the project. +/// +public class ProjectCreationService( + IOptions config, + ISendReceiveService srService, + IProjectLoader projectLoader, + SyncHostedService syncHostedService, + ILogger logger) +{ + private const string InitialCommitMessage = "Initial project creation"; + + // The one file FieldWorks/FLExBridge put in a brand-new FLEx repo's first commit: a minimal custom + // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge). + private const string CustomPropertiesFilename = "FLExProject.CustomProperties"; + + public async Task InitFwDataProject( + Guid projectId, + string projectCode, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems, + string uiWritingSystem) + { + if (vernacularWritingSystems.Count == 0) + throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); + if (analysisWritingSystems.Count == 0) + throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); + if (string.IsNullOrEmpty(uiWritingSystem)) + throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem)); + + // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. + if (!syncHostedService.TryStartProjectCreation(projectId)) + throw new ProjectSyncInProgressException(projectId); + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + // Published to status pollers in the finally below. Pre-set to a failure so an exit path that + // somehow reaches neither the success line nor the catch still releases pollers with an answer. + var outcome = ProjectCreationStatus.Failed("Project creation ended without reporting a result"); + try + { + // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/ + // FLExBridge create a repo, since Clone can't establish an empty remote and + // Language_Forge_Send_Receive refuses to make the *first* commit itself: + // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). + // 2. Build fw.fwdata into that folder from the SIL.LCModel NewLangProj template, + // configured with all of the requested writing systems, but do not commit it yet. + // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch to be + // revision 0 -- Chorus expects revision 0 to be on the default branch, because it uses + // that rev 0 commit to track a repo's history and catch when someone would try to push + // an unrelated repo (which would otherwise create two Mercurial heads). This is the + // same initial commit, with the same file, that FieldWorks creates. The fwdata itself + // is NOT committed; Send/Receive splits it into the nested files Mercurial actually + // tracks (fwdata is excluded from tracking). + // 4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, + // e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on + // that branch, so the split data must land there while the genesis stays on 'default'. + // 5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push. + await srService.InitRepo(fwDataProject.ProjectFolder); + + BuildFwDataFile(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); + FixLinkedFilesRootDirSeparator(fwDataProject); + AssertModelVersionMatches(fwDataProject); + + var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject); + await srService.CommitFile(customPropertiesFile, InitialCommitMessage); + + await srService.SwitchBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); + var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage); + if (!pushResult.Success) + throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); + + logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId); + outcome = ProjectCreationStatus.Created; + } + catch (Exception ex) + { + outcome = ProjectCreationStatus.Failed(ex.Message); + CleanupLocalProject(fwDataProject); + throw; + } + finally + { + syncHostedService.EndProjectCreation(projectId, outcome); + } + } + + private void BuildFwDataFile( + FwDataProject fwDataProject, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems, + string uiWs) + { + // NewProject copies the SIL.LCModel template, configured with all of the requested writing + // systems (the first of each list is the default of that type), and returns a loaded cache; + // dispose it right away so its file locks are released before the following hg add/commit. + using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs); + } + + /// + /// FieldWorks (Windows) compares LangProject.LinkedFilesRootDir with an exact string, expecting the + /// Windows-style default %proj%\LinkedFiles. liblcm builds that path with Path.Combine, which on + /// this Linux host yields %proj%/LinkedFiles (forward slash), so FieldWorks warns on its first + /// Send/Receive. Rewrite just that one value to the backslash form. + /// + /// We deliberately do NOT parse the (potentially multi-MB) fwdata as XML -- that would be a needless + /// cost. The value always sits on the line immediately after the <LinkedFilesRootDir> open tag, + /// so stream the file once, fix the <Uni> on the line following that tag, and copy every other + /// line through untouched. + /// + private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) + { + const string markerLinePrefix = "%proj%\LinkedFiles"; + + var sourcePath = fwDataProject.FilePath; + var tempPath = sourcePath + ".tmp"; + var madeChanges = false; + using (var reader = new StreamReader(sourcePath)) + using (var writer = new StreamWriter(tempPath)) + { + var nextLineIsValue = false; + var done = false; + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (nextLineIsValue) + { + var changedLine = line.Replace(forwardSlashValue, backslashValue); + if (string.Equals(changedLine, line, StringComparison.InvariantCulture)) + { + // Exit early and don't overwrite since we didn't need this change + break; + } + madeChanges = true; + line = changedLine; + nextLineIsValue = false; + done = true; + } + else if (!done && line.StartsWith(markerLinePrefix, StringComparison.Ordinal)) + { + nextLineIsValue = true; + } + writer.WriteLine(line); + } + } + if (madeChanges) + File.Move(tempPath, sourcePath, overwrite: true); + else + File.Delete(tempPath); + } + + private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) + { + var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename); + File.WriteAllText(path, """ + + + """); + return path; + } + + /// + /// The freshly-created project is stamped with the SIL.LCModel package's data-model version. + /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data + /// (see FwHeadless AGENTS.md), so fail before the push rather than after. + /// + private void AssertModelVersionMatches(FwDataProject fwDataProject) + { + var expected = config.Value.FdoDataModelVersion; + var actual = ReadModelVersion(fwDataProject.FilePath); + // Fail closed: an unreadable version ('actual' == null) is treated as a mismatch, since this is + // the last safety net before an irreversible push in a data-loss-risk area. + if (actual != expected) + { + throw new InvalidOperationException( + $"New project's data-model version '{actual ?? "(unreadable)"}' does not match FwHeadless " + + $"FdoDataModelVersion '{expected}'; refusing to push to avoid data corruption."); + } + } + + private static string? ReadModelVersion(string fwDataFilePath) + { + using var reader = XmlReader.Create(fwDataFilePath, new XmlReaderSettings { IgnoreComments = true, IgnoreWhitespace = true }); + // The .fwdata root element carries the FDO model version, e.g. . + return reader.MoveToContent() == XmlNodeType.Element ? reader.GetAttribute("version") : null; + } + + private void CleanupLocalProject(FwDataProject fwDataProject) + { + // Mirror SyncWorker.SetupFwData's failure cleanup so a retry starts clean. The remote repo + // (created by LexBox) is torn down by the caller on the LexBox side. + try + { + if (Directory.Exists(fwDataProject.ProjectFolder)) + Directory.Delete(fwDataProject.ProjectFolder, true); + if (Directory.Exists(fwDataProject.ProjectsPath) && + !Directory.EnumerateFileSystemEntries(fwDataProject.ProjectsPath).Any()) + Directory.Delete(fwDataProject.ProjectsPath); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to clean up local project folder after a failed creation: {ProjectFolder}", + fwDataProject.ProjectFolder); + } + } +} diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 509644a88a..a5066a58fd 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -166,6 +166,46 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)} {EscapeShellArg(filePath)}", fileDir, progress, allowedExitCodes: [0, 1]); } + public static async Task InitRepo(string folder, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + progress ??= new NullProgress(); + Directory.CreateDirectory(folder); + // Use Chorus rather than a bare `hg init`: CreateOrUseExisting also wires up the custom hg + // extensions FwHeadless relies on (e.g. fixutf8) so later send/receive works correctly. + await Task.Run(() => HgRepository.CreateOrUseExisting(folder, progress)); + } + + public static async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "", IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + activity?.SetTag("app.branch", branchName); + progress ??= new NullProgress(); + // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version + // (e.g. 7500002.7000072); the initial commit of a repo needs to be on the 'default' branch + // so that Chorus can use rev 0 to compare project origin, but then we need to switch branches + // to the numbered branch. The `hg branch` command records the branch that will be used for + // the next commit, but we need to add an empty commit in order to actually create the branch, + // otherwise LfMergeBridge will refuse to create a new branch (since in the Language Forge code + // that LfMergeBridge targets, creating a new branch would be an error). This results in a new + // project being created by this process having three initial commits, rather than the two + // initial commits that it would have if created by FieldWorks Classic, but that's an acceptable + // price to pay for such a simple workaround. If LfMergeBridge is updated to be okay with creating + // new branches on its own, then this method can be passed "createBranch: false" to skip running + // the `hg commit` step that creates that empty commit. + // + // Commit message defaults to "Switching to branch {branchName}", pass `commitMessage` param to change + await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress); + if (createBranch) + { + // Could run "hg log -r . --template {desc}" to reuse previous commit message if we wanted to + if (string.IsNullOrEmpty(commitMessage)) commitMessage = $"Switching to branch {branchName}"; + await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress); + } + } + private static string EscapeShellArg(string arg) { var quote = """ diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs index 735e90b8ae..0c410ba1f9 100644 --- a/backend/FwHeadless/Services/SendReceiveService.cs +++ b/backend/FwHeadless/Services/SendReceiveService.cs @@ -70,4 +70,14 @@ public async Task CommitFile(string filePath, string commitMessage) { await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress); } + + public async Task InitRepo(string folder) + { + await SendReceiveHelpers.InitRepo(folder, progress); + } + + public async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "") + { + await SendReceiveHelpers.SwitchBranch(folder, branchName, createBranch, commitMessage, progress); + } } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 33bc51334a..e7663c109c 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -20,6 +20,8 @@ public enum QueueJobResult { Queued, AlreadyQueued, + // A sync must not race a project that's still being created; the caller should retry later + ProjectBeingCreated, // The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes SyncAlreadyRunning, QueueFull, @@ -29,22 +31,41 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsToSync = Channel.CreateUnbounded(); - // Projects with a sync queued or running. A single atomic GetOrAdd decides whether a request queues a new sync, - // so no lock is needed (this requires that SyncHostedService be a singleton). - private readonly ConcurrentDictionary _syncJobs = new(); + // Projects with a sync queued or running, or currently being created and populated with an initial .fwdata file. + // Creations are tracked here (not just in ProjectCreationService) so a sync job can't be queued for a project + // mid-creation and race on the same fw/ folder and repo. Keeping both in one dictionary means a single atomic + // TryAdd/GetOrAdd decides who gets the project, so no lock is needed (this requires that SyncHostedService be a singleton). + private readonly ConcurrentDictionary _projectJobs = new(); /// - /// The TaskCompletionSource carries the outcome to anyone long-polling AwaitSyncFinished. - /// distinguishes a queued sync (which will pick up any changes made before it starts) + /// Exactly one of or is set. The TaskCompletionSource carries the + /// outcome to anyone long-polling AwaitSyncFinished or AwaitCreationFinished. + /// distinguishes a queued sync (which will pick up any changes made before it starts) /// from a running one (which might not). /// - private record SyncJob(TaskCompletionSource Result, bool Started = false) + private record ProjectJob( + TaskCompletionSource? Sync, + TaskCompletionSource? Creation, + bool SyncStarted = false) { // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on - // the thread that finished the sync. - public static SyncJob New() => new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + // the thread that finished the sync or creation. + public static ProjectJob ForSync() => + new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), null); + + public static ProjectJob ForCreation() => + new(null, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); } + private TaskCompletionSource? GetSyncJob(Guid projectId) => + _projectJobs.TryGetValue(projectId, out var job) ? job.Sync : null; + + private TaskCompletionSource? GetCreationJob(Guid projectId) => + _projectJobs.TryGetValue(projectId, out var job) ? job.Creation : null; + + // Only removes the entry if it's still the given job, so ending one kind of job can never clear the other kind + private bool TryRemoveJob(Guid projectId, ProjectJob job) => _projectJobs.TryRemove(KeyValuePair.Create(projectId, job)); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) @@ -73,38 +94,93 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Give clients a bit more time to poll the status CacheRecentSyncResult(projectId, result); - if (_syncJobs.TryRemove(projectId, out var job)) - job.Result.TrySetResult(result); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is { } tcs && TryRemoveJob(projectId, job)) + tcs.TrySetResult(result); } } - // QueueJob never modifies an existing entry, and only ExecuteAsync removes one, so nothing can race this update + // QueueJob never modifies an existing sync entry, and only ExecuteAsync removes one, so nothing can race this update internal void MarkSyncStarted(Guid projectId) { - if (_syncJobs.TryGetValue(projectId, out var job)) - _syncJobs.TryUpdate(projectId, job with { Started = true }, job); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is not null) + _projectJobs.TryUpdate(projectId, job with { SyncStarted = true }, job); } public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _syncJobs.ContainsKey(projectId); + return _projectJobs.ContainsKey(projectId); + } + + /// + /// Reserves a project for creation-and-population, blocking a concurrent creation or sync of the + /// same project. Returns false if a sync is already queued/running or another creation is in flight. + /// Pair with in a finally block. + /// + public bool TryStartProjectCreation(Guid projectId) + { + return _projectJobs.TryAdd(projectId, ProjectJob.ForCreation()); + } + + /// + /// Releases the reservation and publishes the outcome to pollers. Must be called on every exit path + /// from a creation, or an AwaitCreationFinished caller waits until its own token trips. + /// + public void EndProjectCreation(Guid projectId, ProjectCreationStatus result) + { + // Cache before removing the reservation, so a poller that sees "not in flight" always finds the + // result waiting for it rather than a gap. + CacheRecentCreationResult(projectId, result); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Creation is { } tcs && TryRemoveJob(projectId, job)) + tcs.TrySetResult(result); } + public bool IsProjectBeingCreated(Guid projectId) => GetCreationJob(projectId) is not null; + public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) { - if (_syncJobs.TryGetValue(projectId, out var job)) - return await job.Result.Task.WaitAsync(cancellationToken); + if (GetSyncJob(projectId) is { } tcs) + return await tcs.Task.WaitAsync(cancellationToken); return TryGetRecentSyncResult(projectId); } + /// + /// Waits for an in-flight creation to finish. Returns the recent result if one just finished, or + /// null when this FwHeadless has no record of a creation for the project (the caller should then + /// derive the status from durable state instead). + /// + public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken) + { + if (GetCreationJob(projectId) is { } tcs) + return await tcs.Task.WaitAsync(cancellationToken); + return TryGetRecentCreationResult(projectId); + } + + public ProjectCreationStatus? TryGetRecentCreationResult(Guid projectId) + { + return memoryCache.Get($"CreationResult|{projectId}"); + } + + private void CacheRecentCreationResult(Guid projectId, ProjectCreationStatus result) + { + // Longer than the sync equivalent's 30s: a caller whose own request timed out partway through a + // creation should still be able to come back and learn how it ended. + memoryCache.Set($"CreationResult|{projectId}", result, TimeSpan.FromMinutes(5)); + } + public QueueJobResult QueueJob(Guid projectId) { - var newJob = SyncJob.New(); - var job = _syncJobs.GetOrAdd(projectId, newJob); + var newJob = ProjectJob.ForSync(); + var job = _projectJobs.GetOrAdd(projectId, newJob); + //don't sync a project while it's still being created and populated with initial .fwdata (would race on the repo) + if (job.Creation is not null) + { + logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); + return QueueJobResult.ProjectBeingCreated; + } //will only queue job if it's not already queued if (!ReferenceEquals(job, newJob)) { - if (job.Started) + if (job.SyncStarted) { logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId); return QueueJobResult.SyncAlreadyRunning; @@ -116,7 +192,7 @@ public QueueJobResult QueueJob(Guid projectId) if (!_projectsToSync.Writer.TryWrite(projectId)) { logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); - _syncJobs.TryRemove(KeyValuePair.Create(projectId, newJob)); + TryRemoveJob(projectId, newJob); return QueueJobResult.QueueFull; } diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index b05e8b058c..739a4001ff 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,13 +1,17 @@ +using System.Text.RegularExpressions; using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; using LexBoxApi.Jobs; +using LexBoxApi.Models.Project; using LexBoxApi.Services; using LexCore.Entities; using LexCore.Exceptions; using LexCore.ServiceInterfaces; +using LexCore.Sync; using LexData; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; using Quartz; using Path = System.IO.Path; @@ -20,9 +24,157 @@ public class ProjectController( IHgService hgService, LexBoxDbContext lexBoxDbContext, IPermissionService permissionService, - ISchedulerFactory scheduler) + ISchedulerFactory scheduler, + ILogger logger) : ControllerBase { + /// + /// Admin-only: create a new FLEx project whose repo is populated with a template .fwdata + /// (from the SIL.LCModel package) configured for the requested writing systems. + /// + /// At some future date this API may be opened up to trusted users, but the projectOrigin + /// parameter will remain admin-only. + /// + /// Building and pushing the project takes too long to hold a request open for, so this creates the + /// project row and empty repo, queues the rest as a background job, and returns 202 Accepted with the + /// new project's ID. Poll to learn when it's ready. + /// + /// Project code for the new project. + /// Display name for the new project. + /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. + /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. + /// Optional user interface writing system id; defaults to "en" if not given. + /// Optional to record as the project's origin; admin-only. + [HttpPost("initFwDataProject")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status202Accepted)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status409Conflict)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public async Task> InitFwDataProject( + string code, + string name, + [FromQuery] string[] wsVernacular, + [FromQuery] string[]? wsAnalysis = null, + string wsUi = "en", + string? projectOrigin = null) + { + // Validate before anything is created, so a bad request never leaves a project to tear down. + var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); + var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", wsVernacular) + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", wsAnalysisOrDefault) + ?? InitFwDataProjectInput.ValidateUiWritingSystem(wsUi); + if (wsError is not null) + return Problem(wsError, statusCode: StatusCodes.Status400BadRequest); + if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) + return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); + if (string.IsNullOrWhiteSpace(name)) + return Problem("A project name is required", statusCode: StatusCodes.Status400BadRequest); + + ProjectMigrationStatus? origin = null; + if (!string.IsNullOrEmpty(projectOrigin)) + { + // Only admins may say where a project came from. Currently redundant with [AdminRequired], + // but this endpoint is expected to open up to non-admins later; this check must stay. + permissionService.AssertIsAdmin(); + if (!TryParseProjectOrigin(projectOrigin, out var parsedOrigin)) + return Problem($"Invalid project origin '{projectOrigin}'", statusCode: StatusCodes.Status400BadRequest); + origin = parsedOrigin; + } + if (await projectService.ProjectExists(code)) + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + + Guid projectId; + try + { + projectId = await projectService.CreateProject(new CreateProjectInput( + Id: null, + Name: name, + Description: string.Empty, + Code: code, + Type: ProjectType.FLEx, + RetentionPolicy: RetentionPolicy.Verified, + IsConfidential: false, + ProjectManagerId: null, + OrgId: null), + projectOrigin: origin, + // In the same insert as the row itself, so the project is never visible without it + fwDataInitStatus: FwDataInitStatus.InProgress); + } + catch (DbUpdateException e) when (DbError.CreateErrorFrom(e).Code == DbErrorCode.DuplicateProjectCode) + { + // ProjectExists above is only a fast path: a concurrent request can take the code after that check, + // and the unique index on Project.Code is what actually catches it. Nothing was created by this request. + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + } + + // The project row (marked as being initialised) and its empty repo now exist. Populating the repo + // happens in the job, which also marks the project ready or rolls it back. + try + { + await InitFwDataProjectJob.Queue(scheduler, projectId, + new InitFwDataProjectInput(wsVernacular, wsAnalysisOrDefault, wsUi)); + } + catch (Exception ex) + { + // Nothing has been sent to FwHeadless yet, so tearing the project down is unambiguous + logger.LogError(ex, "Failed to queue initialising project {ProjectId} ({Code}), rolling it back", projectId, code); + await CleanupFailedCreation(projectId, code); + throw; + } + return AcceptedAtAction(nameof(FwDataProjectCreationStatus), new { projectId }, projectId); + } + + /// + /// Admin-only: how the creation started by is going. + /// Creating while the job is still running; Created once the project exists and isn't being + /// initialised (so also for any project that wasn't created that way). A failed creation is rolled + /// back, deleting the project, after which this reports CreationFailed with the reason for an hour, + /// as long as LexBox hasn't restarted, and 404 after that. + /// + [HttpGet("fwDataProjectCreationStatus/{projectId}")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task> FwDataProjectCreationStatus( + Guid projectId, + [FromServices] IMemoryCache memoryCache) + { + var initStatus = await lexBoxDbContext.Projects + .Where(p => p.Id == projectId) + .Select(p => (FwDataInitStatus?)p.FwDataInitStatus) + .SingleOrDefaultAsync(); + if (initStatus is not null) + return initStatus == FwDataInitStatus.InProgress ? ProjectCreationStatus.Creating : ProjectCreationStatus.Created; + var failure = InitFwDataProjectJob.TryGetRecentFailure(memoryCache, projectId); + return failure is null ? NotFound() : ProjectCreationStatus.Failed(failure); + } + + /// + /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists + /// ("Migrated,Migrating"), so the parsed value must round-trip to the name that was passed in. + /// + public static bool TryParseProjectOrigin(string projectOrigin, out ProjectMigrationStatus origin) => + Enum.TryParse(projectOrigin, ignoreCase: true, out origin) + && Enum.IsDefined(origin) + && string.Equals(origin.ToString(), projectOrigin, StringComparison.OrdinalIgnoreCase); + + /// Analysis writing systems to use, defaulting to English when none are supplied. + public static string[] AnalysisWritingSystemsOrDefault(string[]? wsAnalysis) => + wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; + + private async Task CleanupFailedCreation(Guid projectId, string code) + { + try + { + await projectService.CleanupFailedProjectCreation(projectId, code); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}); it may need manual cleanup", projectId, code); + } + } + [HttpPost("refreshProjectLastChanged")] public async Task RefreshProjectLastChanged(string projectCode) { diff --git a/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs new file mode 100644 index 0000000000..38c90de1de --- /dev/null +++ b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs @@ -0,0 +1,211 @@ +using System.Net; +using System.Text.Json; +using LexBoxApi.Services; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexCore.Sync; +using LexData; +using Microsoft.Extensions.Caching.Memory; +using Quartz; + +namespace LexBoxApi.Jobs; + +/// +/// Has FwHeadless populate the empty repo of a project created by ProjectController.InitFwDataProject, +/// then marks the project ready, or rolls it back if FwHeadless reports that the creation failed. +/// +/// +/// Quartz keeps this job in the database and reruns it if LexBox stops partway through (RequestRecovery), +/// so every run must be safe to repeat. Each run therefore starts by finding out how far an earlier run +/// got, using LexBox's own repo as the source of truth. When the outcome is unknown (FwHeadless is +/// unreachable, busy, or still running a creation), the job schedules another attempt rather than guess. +/// +public class InitFwDataProjectJob( + LexBoxDbContext dbContext, + ProjectService projectService, + FwHeadlessClient fwHeadlessClient, + IHgService hgService, + ISchedulerFactory schedulerFactory, + IMemoryCache memoryCache, + ILogger logger) : LexJob +{ + public const int MaxAttempts = 10; + public static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + public static JobKey Key { get; } = new("InitFwDataProjectJob", "ProjectCreation"); + + public static async Task Queue(ISchedulerFactory schedulerFactory, + Guid projectId, + InitFwDataProjectInput input, + int attempt = 1, + TimeSpan? delay = null, + CancellationToken cancellationToken = default) + { + await QueueJob(schedulerFactory, + Key, + new JobDataMap + { + { nameof(ProjectId), projectId.ToString() }, + // The job store only takes strings, so the writing systems travel as JSON + { nameof(Input), JsonSerializer.Serialize(input) }, + { nameof(Attempt), attempt.ToString() }, + }, + delay, + cancellationToken); + } + + public string? ProjectId { get; set; } + public string? Input { get; set; } + public string? Attempt { get; set; } + + protected override async Task ExecuteJob(IJobExecutionContext context) + { + ArgumentException.ThrowIfNullOrEmpty(ProjectId); + ArgumentException.ThrowIfNullOrEmpty(Input); + ArgumentException.ThrowIfNullOrEmpty(Attempt); + if (!Guid.TryParse(ProjectId, out var projectId)) throw new ArgumentException($"Invalid project ID {ProjectId}", nameof(ProjectId)); + if (!int.TryParse(Attempt, out var attempt)) throw new ArgumentException($"Invalid number {Attempt}", nameof(Attempt)); + var input = JsonSerializer.Deserialize(Input) + ?? throw new ArgumentException("Missing creation input", nameof(Input)); + + var project = await dbContext.Projects.FindAsync(projectId); + if (project is null) + { + logger.LogInformation("Project {ProjectId} no longer exists, nothing to initialise", projectId); + return; + } + if (project.FwDataInitStatus != FwDataInitStatus.InProgress) + { + // An earlier run finished the job and this is a duplicate (e.g. a recovered run) + logger.LogInformation("Project {ProjectId} ({Code}) has already been initialised", projectId, project.Code); + return; + } + + var outcome = await AttemptInit(fwHeadlessClient, hgService, logger, projectId, project.Code, input); + switch (outcome.Result) + { + case InitAttemptResult.Created: + await projectService.FinishFwDataInit(projectId); + logger.LogInformation("Initialised project {ProjectId} ({Code}) from the template", projectId, project.Code); + return; + + case InitAttemptResult.Failed: + logger.LogError("Creating project {ProjectId} ({Code}) from the template failed, rolling it back: {Error}", + projectId, project.Code, outcome.Detail); + // Before the rollback, so a poller that finds the project gone always finds the reason too + RecordFailure(memoryCache, projectId, outcome.Detail); + try + { + await projectService.CleanupFailedProjectCreation(projectId, project.Code); + return; + } + catch (Exception ex) + { + // The project is still marked as being initialised, so a later attempt starts over: it + // either creates the project after all, or fails again and retries the rollback + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code})", projectId, project.Code); + } + break; + + case InitAttemptResult.RetryLater: + logger.LogWarning("Couldn't tell whether project {ProjectId} ({Code}) was created (attempt {Attempt} of {MaxAttempts}): {Reason}", + projectId, project.Code, attempt, MaxAttempts, outcome.Detail); + break; + } + + if (attempt >= MaxAttempts) + { + logger.LogError( + "Giving up on initialising project {ProjectId} ({Code}) after {Attempts} attempts; it is still marked as being initialised and needs manual cleanup", + projectId, project.Code, attempt); + return; + } + await Queue(schedulerFactory, projectId, input, attempt + 1, RetryDelay); + } + + /// + /// One attempt at getting the project populated, picking up wherever an earlier attempt got to. + /// Never throws: anything unexpected means the outcome is unknown, so it's reported as RetryLater. + /// + /// + /// None of these calls take a cancellation token, for the same reason ProjectController used not to + /// pass one: FwHeadless finishes a creation whatever happens to our side of the connection, so + /// abandoning the wait would only stop us learning the outcome. If LexBox shuts down mid-attempt, + /// Quartz recovers the job and the next run picks the creation up again. + /// + public static async Task AttemptInit(FwHeadlessClient fwHeadlessClient, + IHgService hgService, + ILogger logger, + Guid projectId, + string code, + InitFwDataProjectInput input) + { + try + { + // FwHeadless's push is the last step of a creation, so a non-empty repo means an earlier attempt + // succeeded, even if we never heard back. Trust this over FwHeadless's own status, which says + // Created as soon as the .fwdata exists on its disk, even if FwHeadless died before pushing it. + if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created; + + var status = await fwHeadlessClient.CreationStatus(projectId); + if (status?.Status == ProjectCreationStatusEnum.Creating) + { + // An earlier attempt is still running in FwHeadless (e.g. this job was interrupted and + // recovered); starting another would be refused, so wait for that one instead + status = await fwHeadlessClient.AwaitCreationFinished(projectId); + if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created; + return status?.Status == ProjectCreationStatusEnum.CreationFailed + ? InitAttemptOutcome.Failed(status.Error ?? "FwHeadless reported that the creation failed") + : InitAttemptOutcome.RetryLater($"Waiting for FwHeadless's creation ended with {status?.Status.ToString() ?? "no status"}"); + } + + // Clear anything an interrupted attempt left in FwHeadless's local project folder, which would + // otherwise break the new attempt. Throws ProjectSyncInProgressException if FwHeadless is busy + // with this project. + await fwHeadlessClient.DeleteProject(projectId); + var (statusCode, error) = await fwHeadlessClient.InitFwDataProject(projectId, input.WsVernacular, input.WsAnalysis, input.WsUi); + if (error is null) return InitAttemptOutcome.Created; + // FwHeadless refused to start because something else holds the project, so nothing was built + if (statusCode == HttpStatusCode.Conflict) return InitAttemptOutcome.RetryLater(error); + // FwHeadless answered with a failure, so the creation isn't running and FwHeadless has already + // cleaned up its own local state + return InitAttemptOutcome.Failed(error); + } + catch (Exception ex) + { + // Most likely FwHeadless is unreachable (e.g. restarting) or our request timed out. FwHeadless + // may still be creating the project, so only a later attempt can tell how this ended. + logger.LogWarning(ex, "Attempt to initialise project {ProjectId} ({Code}) ended without an answer", projectId, code); + return InitAttemptOutcome.RetryLater(ex.Message); + } + } + + private static async Task RepoHasCommits(IHgService hgService, string code) => + await hgService.GetTipHash(code) != HgService.AllZeroHash; + + // In memory only: once the rollback has deleted the project row, this is all that's left to tell a + // poller why. It doesn't survive a LexBox restart, after which the poller sees "not found". + private static string FailureCacheKey(Guid projectId) => $"InitFwDataProjectFailure|{projectId}"; + + private static void RecordFailure(IMemoryCache memoryCache, Guid projectId, string? error) => + memoryCache.Set(FailureCacheKey(projectId), error ?? "Creating the project from the template failed", TimeSpan.FromHours(1)); + + public static string? TryGetRecentFailure(IMemoryCache memoryCache, Guid projectId) => + memoryCache.Get(FailureCacheKey(projectId)); +} + +public enum InitAttemptResult +{ + Created, + // FwHeadless definitely isn't creating the project and didn't create it + Failed, + // The outcome is unknown, e.g. FwHeadless was unreachable or is still busy + RetryLater, +} + +public record InitAttemptOutcome(InitAttemptResult Result, string? Detail = null) +{ + public static InitAttemptOutcome Created { get; } = new(InitAttemptResult.Created); + public static InitAttemptOutcome Failed(string error) => new(InitAttemptResult.Failed, error); + public static InitAttemptOutcome RetryLater(string reason) => new(InitAttemptResult.RetryLater, reason); +} diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 209e2fe849..2db9c3b5c6 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -69,7 +69,13 @@ public static void AddLexBoxApi(this IServiceCollection services, config.Enabled = false; }); services.AddServiceDiscovery(); - services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless")) + services.AddHttpClient(client => + { + client.BaseAddress = new ("http://fwHeadless"); + // initFwDataProject runs inline (hg init + build the project from the template + push), + // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom. + client.Timeout = TimeSpan.FromMinutes(5); + }) .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config services.AddHttpContextAccessor(); services.AddMemoryCache(); diff --git a/backend/LexBoxApi/ScheduledTasksKernel.cs b/backend/LexBoxApi/ScheduledTasksKernel.cs index 5293d20502..5345071d68 100644 --- a/backend/LexBoxApi/ScheduledTasksKernel.cs +++ b/backend/LexBoxApi/ScheduledTasksKernel.cs @@ -37,6 +37,9 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur q.AddJob(DeleteTempDirectoryJob.Key, j => j.StoreDurably()); q.AddJob(UpdateProjectMetadataJob.Key, j => j.StoreDurably()); q.AddJob(RetryEmailJob.Key, j => j.StoreDurably()); + // RequestRecovery: rerun a creation that was interrupted by LexBox stopping, rather than leave the + // project marked as being initialised forever. The job is written to be safe to rerun. + q.AddJob(InitFwDataProjectJob.Key, j => j.StoreDurably().RequestRecovery()); q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key) .WithIdentity("WeeklyCleanupTrigger") //every sunday at 2am diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 4b2d8d7cde..549b9e9a02 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -1,5 +1,7 @@ using System.Net; +using System.Net.Http.Json; using System.Text.Json; +using LexCore.Entities; using LexCore.Exceptions; using LexCore.Sync; @@ -209,4 +211,91 @@ private static async Task FailedResponse(HttpResponseMessag } return body; } + + /// + /// Initializes a newly-created (empty) project's repo with a new .fwdata configured for the + /// given writing systems. Runs inline (a new empty project is small). Returns a null Error on + /// success; on failure returns the FwHeadless status code and error body so the caller can + /// propagate the right status (e.g. surface a 400 rather than flattening it to 500). + /// + public async Task<(HttpStatusCode StatusCode, string? Error)> InitFwDataProject(Guid projectId, + IReadOnlyList wsVernacular, + IReadOnlyList wsAnalysis, + string wsUi, + CancellationToken cancellationToken = default) + { + var input = new InitFwDataProjectInput(wsVernacular, wsAnalysis, wsUi); + var response = await httpClient.PostAsJsonAsync($"/api/project/initFwDataProject?projectId={projectId}", + input, + cancellationToken); + if (response.IsSuccessStatusCode) return (response.StatusCode, null); + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + logger.LogError("Failed to init fwdata project: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + responseBody); + return (response.StatusCode, responseBody); + } + + /// + /// Current creation status for a project, or null if FwHeadless doesn't know the project or the + /// call failed (both are logged). Cheap and safe to poll. + /// + /// + /// Unlike , this one does take a cancellation token, and callers may + /// safely pass HttpContext.RequestAborted. Abandoning a status read costs nothing: it starts no + /// durable work and tells the caller nothing it has to act on. Abandoning the creation call is what + /// is dangerous, because the creation keeps running on the FwHeadless side either way. + /// + public async Task CreationStatus(Guid projectId, CancellationToken cancellationToken = default) + { + var response = await httpClient.GetAsync($"/api/project/creation-status?projectId={projectId}", cancellationToken); + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(cancellationToken); + // 404 is an ordinary answer ("no such project"), not a fault worth an error log. + if (response.StatusCode != HttpStatusCode.NotFound) + { + logger.LogError("Failed to get project creation status: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync(cancellationToken)); + } + return null; + } + + /// + /// Waits for an in-flight creation to finish and returns how it ended, or null if FwHeadless doesn't + /// know the project or the call failed. Returns TimedOutAwaitingCreation if the wait is given up on + /// -- by either side -- which means "still running, ask again", never "creation failed". + /// + public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken = default) + { + HttpResponseMessage response; + try + { + response = await httpClient.GetAsync($"/api/project/await-creation-finished?projectId={projectId}", cancellationToken); + } + catch (OperationCanceledException) + { + // We gave up first, so FwHeadless's own TimedOutAwaitingCreation response is on a connection + // we already dropped. Report the same thing ourselves, so a caller sees one contract no + // matter which side stopped waiting. The creation itself is untouched and still running. + logger.LogInformation("Gave up awaiting creation of project {ProjectId}; it is still running", projectId); + return ProjectCreationStatus.TimedOutAwaitingCreation; + } + + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(cancellationToken); + if (response.StatusCode != HttpStatusCode.NotFound) + { + logger.LogError("Failed to await project creation: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync(cancellationToken)); + } + return null; + } } diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 8aac13b26b..55d7fd3e2b 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -20,7 +20,7 @@ public partial class HgService : IHgService { public const string ManageClientName = "HgResumeManage"; private const string ManageSecretHeader = "X-Manage-Secret"; - private const string AllZeroHash = "0000000000000000000000000000000000000000"; + public const string AllZeroHash = "0000000000000000000000000000000000000000"; private readonly IOptions _options; private readonly Lazy _hgClient; diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 459f14b138..a050169c49 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -25,7 +25,9 @@ public class ProjectService( IEmailService emailService, FwHeadlessClient fwHeadless) { - public async Task CreateProject(CreateProjectInput input) + public async Task CreateProject(CreateProjectInput input, + ProjectMigrationStatus? projectOrigin = null, + FwDataInitStatus fwDataInitStatus = FwDataInitStatus.None) { await using var transaction = await dbContext.Database.BeginTransactionAsync(); var projectId = input.Id ?? Guid.NewGuid(); @@ -41,7 +43,8 @@ public async Task CreateProject(CreateProjectInput input) Id = projectId, Code = input.Code, Name = input.Name, - ProjectOrigin = ProjectMigrationStatus.Migrated, + ProjectOrigin = projectOrigin ?? ProjectMigrationStatus.Migrated, + FwDataInitStatus = fwDataInitStatus, Description = input.Description, Type = input.Type, LastCommit = null, @@ -232,6 +235,52 @@ public async Task FinishReset(string code, Stream? zipFile = null) return project; } + /// + /// Marks a project whose repo FwHeadless has populated from the template as ready, recording the + /// commit it pushed. + /// + public async Task FinishFwDataInit(Guid projectId) + { + var project = await dbContext.Projects.FindAsync(projectId); + if (project is null) return; + project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code); + project.FwDataInitStatus = FwDataInitStatus.None; + await dbContext.SaveChangesAsync(); + } + + /// + /// Compensating cleanup for a project whose template population failed after CreateProject: + /// deletes FwHeadless's local project state, then removes the repo and the project row and + /// invalidates the same caches as a permanent delete. + /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a + /// project that InitFwDataProject just created. + /// + public async Task CleanupFailedProjectCreation(Guid projectId, string code) + { + // do this first, because it throws if a creation or sync is in progress, which stops us from + // tearing down an active or already-completed creation (mirrors DeleteProjectPermanently) + await fwHeadless.DeleteProject(projectId); + try + { + // Repo before row: if deleting the repo fails, the project row survives for an admin to see and + // delete, instead of an orphaned repo with no row pointing at it + await hgService.DeleteRepoIfExists(code); + var project = await dbContext.Projects.FindAsync(projectId); + if (project is not null) + { + dbContext.Projects.Remove(project); + await dbContext.SaveChangesAsync(); + } + } + finally + { + // Don't forget to add more Invalidate calls here if we add new caches + InvalidateProjectCodeCache(code); + InvalidateProjectConfidentialityCache(projectId); + InvalidateProjectOrgIdsCache(projectId); + } + } + public async ValueTask LookupProjectOrgIds(Guid projectId) { var cacheKey = $"ProjectOrgsForId:{projectId}"; diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs new file mode 100644 index 0000000000..1785126854 --- /dev/null +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -0,0 +1,51 @@ +namespace LexCore.Entities; + +/// +/// Body of the internal FwHeadless "init fwdata project" call. The vernacular/analysis +/// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one +/// vernacular; analysis and UI default to "en"); FwHeadless validates them again before using them. +/// +public record InitFwDataProjectInput( + IReadOnlyList WsVernacular, + IReadOnlyList WsAnalysis, + string WsUi) +{ + // Deliberately generous: real projects use far fewer writing systems than this, and real tags are far + // shorter, but private-use subtags after "-x-" may carry arbitrary metadata and we don't want to forbid + // any reasonable use of it. + public const int MaxWritingSystemsPerList = 100; + public const int MaxWritingSystemTagLength = 128; + + /// + /// Shape checks shared by LexBoxApi (before it creates anything) and FwHeadless. Returns an error + /// message, or null if the list is acceptable. FwHeadless additionally checks that each tag is valid BCP 47. + /// + public static string? ValidateWritingSystemList(string kind, IReadOnlyList? tags) + { + if (tags is not { Count: > 0 }) + return $"At least one {kind} writing system is required"; + if (tags.Count > MaxWritingSystemsPerList) + return $"At most {MaxWritingSystemsPerList} {kind} writing systems are allowed"; + if (tags.Any(string.IsNullOrWhiteSpace)) + return $"Empty {kind} writing system code"; + if (tags.FirstOrDefault(t => t.Length > MaxWritingSystemTagLength) is { } tooLong) + return $"The {kind} writing system code is longer than {MaxWritingSystemTagLength} characters: {tooLong[..32]}…"; + // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system + if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate) + return $"Duplicate {kind} writing system: {duplicate.Key}"; + return null; + } + + /// + /// Shape checks for the UI writing system, shared like . Returns an + /// error message, or null if the tag is acceptable. FwHeadless additionally checks that it is valid BCP 47. + /// + public static string? ValidateUiWritingSystem(string? tag) + { + if (string.IsNullOrWhiteSpace(tag)) + return "A UI writing system is required"; + if (tag.Length > MaxWritingSystemTagLength) + return $"The UI writing system code is longer than {MaxWritingSystemTagLength} characters: {tag[..32]}…"; + return null; + } +} diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index 85d795c720..d3a5406a28 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -25,6 +25,7 @@ public class Project : EntityBase public int? RepoSizeInKb { get; set; } public DateTimeOffset? DeletedDate { get; set; } public ResetStatus ResetStatus { get; set; } = ResetStatus.None; + public FwDataInitStatus FwDataInitStatus { get; set; } = FwDataInitStatus.None; //historical reference for if this project originated here (migrated), or came from redmine, public or private public ProjectMigrationStatus ProjectOrigin { get; set; } = ProjectMigrationStatus.Migrated; @@ -97,6 +98,7 @@ public enum ProjectMigrationStatus Migrating, PrivateRedmine, PublicRedmine, + LanguageForgeNonSR, } public enum ResetStatus @@ -105,6 +107,13 @@ public enum ResetStatus InProgress = 1 } +public enum FwDataInitStatus +{ + None = 0, + // Created by InitFwDataProject, and FwHeadless hasn't yet finished populating its repo from the template + InProgress = 1 +} + public enum ProjectType { Unknown = 0, diff --git a/backend/LexCore/Sync/ProjectCreationStatus.cs b/backend/LexCore/Sync/ProjectCreationStatus.cs new file mode 100644 index 0000000000..67260ec19b --- /dev/null +++ b/backend/LexCore/Sync/ProjectCreationStatus.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; + +namespace LexCore.Sync; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ProjectCreationStatusEnum +{ + /// No creation is in flight and no project data exists in FwHeadless for this project. + NotCreated, + /// A creation is running right now. + Creating, + /// Project data exists in FwHeadless, i.e. a creation finished successfully. + Created, + /// A creation was attempted and failed. says why. + CreationFailed, + /// + /// The caller's await gave up before creation finished. The creation itself is unaffected and is + /// still running -- poll again rather than treating this as a failure. + /// + TimedOutAwaitingCreation, +} + +/// +/// Status of a FwHeadless project creation, for callers that want to poll (or long-poll) instead of +/// holding a request open for the whole operation. Mirrors , which does +/// the same job for merges. +/// +public record ProjectCreationStatus( + [property: JsonConverter(typeof(JsonStringEnumConverter))] + ProjectCreationStatusEnum Status, + string? Error = null) +{ + public static ProjectCreationStatus NotCreated => new(ProjectCreationStatusEnum.NotCreated); + public static ProjectCreationStatus Creating => new(ProjectCreationStatusEnum.Creating); + public static ProjectCreationStatus Created => new(ProjectCreationStatusEnum.Created); + public static ProjectCreationStatus TimedOutAwaitingCreation => new(ProjectCreationStatusEnum.TimedOutAwaitingCreation); + + public static ProjectCreationStatus Failed(string error) => new(ProjectCreationStatusEnum.CreationFailed, error); + + /// True once creation has stopped running, whether it succeeded or failed. + [JsonIgnore] + public bool IsFinished => Status is ProjectCreationStatusEnum.Created or ProjectCreationStatusEnum.CreationFailed; +} diff --git a/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs new file mode 100644 index 0000000000..9795c46fde --- /dev/null +++ b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs @@ -0,0 +1,1425 @@ +// +using System; +using System.Collections.Generic; +using LexData; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace LexData.Migrations +{ + [DbContext(typeof(LexBoxDbContext))] + [Migration("20261109090102_AddProjectFwDataInitStatus")] + partial class AddProjectFwDataInitStatus + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("bytea") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_blob_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("qrtz_calendars", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_cron_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("text") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_qrtz_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_qrtz_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_qrtz_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_qrtz_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_nm_gp"); + + b.ToTable("qrtz_fired_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bool") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bool") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_j_req_recovery"); + + b.ToTable("qrtz_job_details", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("text") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("qrtz_locks", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("qrtz_paused_trigger_grps", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("qrtz_scheduler_state", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bool") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bool") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("integer") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("integer") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("text") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("text") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("text") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_simprop_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_simple_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("smallint") + .HasColumnName("misfire_instr"); + + b.Property("MisfireOriginalFireTime") + .HasColumnType("bigint") + .HasColumnName("misfire_orig_fire_time"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_qrtz_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_qrtz_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_qrtz_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("qrtz_triggers", "quartz"); + }); + + modelBuilder.Entity("LexCore.Entities.DraftProject", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsConfidential") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("ProjectManagerId") + .HasColumnType("uuid"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ProjectManagerId"); + + b.ToTable("DraftProjects"); + }); + + modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b => + { + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("FlexModelVersion") + .HasColumnType("integer"); + + b.Property("LangProjectId") + .HasColumnType("uuid"); + + b.Property("LexEntryCount") + .HasColumnType("integer"); + + b.HasKey("ProjectId"); + + b.ToTable("FlexProjectMetadata"); + }); + + modelBuilder.Entity("LexCore.Entities.MediaFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("text"); + + b.Property("Metadata") + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValueSql("'{}'"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Revision") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("LexCore.Entities.OrgMember", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("OrgId"); + + b.HasIndex("UserId", "OrgId") + .IsUnique(); + + b.ToTable("OrgMembers", (string)null); + }); + + modelBuilder.Entity("LexCore.Entities.OrgProjects", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("OrgId", "ProjectId") + .IsUnique(); + + b.ToTable("OrgProjects"); + }); + + modelBuilder.Entity("LexCore.Entities.Organization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Orgs", (string)null); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("DeletedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("FwDataInitStatus") + .HasColumnType("integer"); + + b.Property("IsConfidential") + .HasColumnType("boolean"); + + b.Property("LastCommit") + .HasColumnType("timestamp with time zone"); + + b.Property("MigratedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("ProjectOrigin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("RepoSizeInKb") + .HasColumnType("integer"); + + b.Property("ResetStatus") + .HasColumnType("integer"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ParentId"); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("UserId", "ProjectId") + .IsUnique(); + + b.ToTable("ProjectUsers"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CanCreateProjects") + .HasColumnType("boolean"); + + b.Property("CreatedById") + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Email") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("EmailVerified") + .HasColumnType("boolean"); + + b.Property>("FeatureFlags") + .HasColumnType("text[]"); + + b.Property("GoogleId") + .HasColumnType("text"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("LastActive") + .HasColumnType("timestamp with time zone"); + + b.Property("LocalizationCode") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("en"); + + b.Property("Locked") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OptedOutOfAnalytics") + .HasColumnType("boolean"); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("PasswordStrength") + .HasColumnType("integer"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Username") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("Email") + .IsUnique(); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ClientId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ClientSecret") + .HasColumnType("text"); + + b.Property("ClientType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ConsentType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("DisplayNames") + .HasColumnType("text"); + + b.Property("JsonWebKeySet") + .HasColumnType("text"); + + b.Property("Permissions") + .HasColumnType("text"); + + b.Property("PostLogoutRedirectUris") + .HasColumnType("text"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RedirectUris") + .HasColumnType("text"); + + b.Property("Requirements") + .HasColumnType("text"); + + b.Property("Settings") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ClientId") + .IsUnique(); + + b.ToTable("OpenIddictApplications", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("Scopes") + .HasColumnType("text"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)"); + + b.Property("Type") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("Id"); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type"); + + b.ToTable("OpenIddictAuthorizations", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreScope", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Descriptions") + .HasColumnType("text"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("DisplayNames") + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("Resources") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("OpenIddictScopes", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("AuthorizationId") + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Payload") + .HasColumnType("text"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RedemptionDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ReferenceId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)"); + + b.Property("Type") + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.HasKey("Id"); + + b.HasIndex("AuthorizationId"); + + b.HasIndex("ReferenceId") + .IsUnique(); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type"); + + b.ToTable("OpenIddictTokens", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChangeEntities") + .HasColumnType("jsonb"); + + b.Property("ClientId") + .HasColumnType("uuid"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 => + { + b1.IsRequired(); + + b1.Property("Counter") + .HasColumnType("bigint"); + + b1.Property("DateTime") + .HasColumnType("timestamp with time zone"); + }); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("CrdtCommits", (string)null); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("LexCore.Entities.DraftProject", b => + { + b.HasOne("LexCore.Entities.User", "ProjectManager") + .WithMany() + .HasForeignKey("ProjectManagerId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ProjectManager"); + }); + + modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithOne("FlexProjectMetadata") + .HasForeignKey("LexCore.Entities.FlexProjectMetadata", "ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("LexCore.Entities.ProjectWritingSystems", "WritingSystems", b1 => + { + b1.Property("FlexProjectMetadataProjectId"); + + b1.HasKey("FlexProjectMetadataProjectId"); + + b1.ToTable("FlexProjectMetadata"); + + b1 + .ToJson("WritingSystems") + .HasColumnType("jsonb"); + + b1.WithOwner() + .HasForeignKey("FlexProjectMetadataProjectId"); + + b1.OwnsMany("LexCore.Entities.FLExWsId", "AnalysisWss", b2 => + { + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); + + b2.Property("__synthesizedOrdinal") + .ValueGeneratedOnAdd(); + + b2.Property("IsActive"); + + b2.Property("IsDefault"); + + b2.Property("Tag") + .IsRequired(); + + b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); + + b2.ToTable("FlexProjectMetadata"); + + b2.WithOwner() + .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId"); + }); + + b1.OwnsMany("LexCore.Entities.FLExWsId", "VernacularWss", b2 => + { + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); + + b2.Property("__synthesizedOrdinal") + .ValueGeneratedOnAdd(); + + b2.Property("IsActive"); + + b2.Property("IsDefault"); + + b2.Property("Tag") + .IsRequired(); + + b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); + + b2.ToTable("FlexProjectMetadata"); + + b2.WithOwner() + .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId"); + }); + + b1.Navigation("AnalysisWss"); + + b1.Navigation("VernacularWss"); + }); + + b.Navigation("WritingSystems"); + }); + + modelBuilder.Entity("LexCore.Entities.MediaFile", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("LexCore.Entities.OrgMember", b => + { + b.HasOne("LexCore.Entities.Organization", "Organization") + .WithMany("Members") + .HasForeignKey("OrgId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Organizations") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Organization"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.OrgProjects", b => + { + b.HasOne("LexCore.Entities.Organization", "Org") + .WithMany() + .HasForeignKey("OrgId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Org"); + + b.Navigation("Project"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ParentId"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany("Users") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Projects") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Project"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.HasOne("LexCore.Entities.User", "CreatedBy") + .WithMany("UsersICreated") + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Authorizations") + .HasForeignKey("ApplicationId"); + + b.Navigation("Application"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Tokens") + .HasForeignKey("ApplicationId"); + + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", "Authorization") + .WithMany("Tokens") + .HasForeignKey("AuthorizationId"); + + b.Navigation("Application"); + + b.Navigation("Authorization"); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany("HarmonyCommits") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); + + modelBuilder.Entity("LexCore.Entities.Organization", b => + { + b.Navigation("Members"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Navigation("FlexProjectMetadata"); + + b.Navigation("HarmonyCommits"); + + b.Navigation("Users"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Navigation("Organizations"); + + b.Navigation("Projects"); + + b.Navigation("UsersICreated"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Navigation("Authorizations"); + + b.Navigation("Tokens"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Navigation("Tokens"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs new file mode 100644 index 0000000000..7fd0ffdad8 --- /dev/null +++ b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace LexData.Migrations +{ + /// + public partial class AddProjectFwDataInitStatus : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "FwDataInitStatus", + table: "Projects", + type: "integer", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "FwDataInitStatus", + table: "Projects"); + } + } +} diff --git a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs index a8a113dd7c..6678f08f20 100644 --- a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs +++ b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs @@ -694,6 +694,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Description") .HasColumnType("text"); + b.Property("FwDataInitStatus") + .HasColumnType("integer"); + b.Property("IsConfidential") .HasColumnType("boolean"); diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs new file mode 100644 index 0000000000..bb8b796ec8 --- /dev/null +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -0,0 +1,131 @@ +using FwHeadless.Services; +using LexCore.Sync; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.FwHeadless; + +/// +/// Unit tests for the per-project creation reservation that stops a sync from racing a project that +/// is still being created from a template (and stops two concurrent creations of the same project). +/// +public class SyncHostedServiceCreationReservationTests +{ + /// Bounded so a regression that never completes the wait fails the test instead of hanging it. + private static CancellationToken TestTimeout => new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token; + + private static SyncHostedService NewService() => + new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + + [Fact] + public void TryStartProjectCreation_blocks_a_second_concurrent_creation_and_is_reusable_after_release() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a creation is already in flight for this project"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); + svc.TryStartProjectCreation(projectId).Should().BeTrue("the reservation is released and reusable"); + } + + [Fact] + public void QueueJob_is_refused_while_a_project_is_being_created() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.QueueJob(projectId).Should().Be(QueueJobResult.ProjectBeingCreated, "a sync must not race a project that's still being created"); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued, "syncing is allowed once creation has finished"); + } + + [Fact] + public void A_queued_sync_blocks_a_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); + } + + [Fact] + public void QueueJob_is_refused_while_a_sync_is_running() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.MarkSyncStarted(projectId); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning, + "the running sync may have missed the caller's changes, so it must not be reported as covering them"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is running for this project"); + } + + [Fact] + public async Task AwaitCreationFinished_returns_the_outcome_the_creation_reported() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + + var awaiting = svc.AwaitCreationFinished(projectId, TestTimeout); + awaiting.IsCompleted.Should().BeFalse("the creation is still in flight"); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Failed("the push failed")); + + var result = await awaiting; + result.Should().NotBeNull(); + result!.Status.Should().Be(ProjectCreationStatusEnum.CreationFailed); + result.Error.Should().Be("the push failed"); + } + + [Fact] + public async Task AwaitCreationFinished_returns_a_result_that_finished_before_the_caller_asked() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + + // The caller shows up late (e.g. its own request timed out and it came back to ask). It must + // still learn the outcome rather than being told nothing ever happened. + var result = await svc.AwaitCreationFinished(projectId, TestTimeout); + result.Should().NotBeNull(); + result!.Status.Should().Be(ProjectCreationStatusEnum.Created); + } + + [Fact] + public async Task AwaitCreationFinished_returns_null_when_no_creation_is_known() + { + var svc = NewService(); + + var result = await svc.AwaitCreationFinished(Guid.NewGuid(), TestTimeout); + result.Should().BeNull("the caller should fall back to deriving the status from the project itself"); + } + + [Fact] + public async Task AwaitCreationFinished_honours_the_caller_token_without_ending_the_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + + using var cts = new CancellationTokenSource(); + var awaiting = svc.AwaitCreationFinished(projectId, cts.Token); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => awaiting); + svc.IsProjectBeingCreated(projectId).Should().BeTrue("giving up on the wait must not cancel the creation"); + } +} diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs new file mode 100644 index 0000000000..0029e420fb --- /dev/null +++ b/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs @@ -0,0 +1,196 @@ +using System.Net; +using System.Net.Http.Json; +using LexBoxApi.Jobs; +using LexBoxApi.Services; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexCore.Sync; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Testing.LexBoxApi; + +/// +/// Unit tests for how one run of InitFwDataProjectJob decides what happened to a creation. Every run must be +/// safe to repeat (Quartz reruns the job after an interruption), so these cover picking up after each way an +/// earlier run could have stopped. FwHeadless is a fake HTTP handler and the repo is a mocked IHgService. +/// +public class InitFwDataProjectJobTests +{ + private const string Code = "test-proj"; + private const string PushedTip = "1234567890abcdef1234567890abcdef12345678"; + private static readonly Guid ProjectId = Guid.NewGuid(); + private static readonly InitFwDataProjectInput Input = new(["fr"], ["en"], "en"); + + private string _tip = HgService.AllZeroHash; + private readonly List _requests = []; + private readonly Mock _hgService = new(); + + // Responses by request, as "METHOD path". An unlisted request throws, which AttemptInit reports as + // RetryLater like any other exception, so the tests also check exactly which requests were made. + private readonly Dictionary> _fwHeadless = []; + + private const string CreationStatus = "GET /api/project/creation-status"; + private const string AwaitCreation = "GET /api/project/await-creation-finished"; + private static readonly string DeleteLocalProject = $"DELETE /api/manage/project/{ProjectId}"; + private const string InitProject = "POST /api/project/initFwDataProject"; + + public InitFwDataProjectJobTests() + { + _hgService.Setup(h => h.GetTipHash(Code, It.IsAny())).ReturnsAsync(() => _tip); + } + + private Task Attempt() + { + var client = new FwHeadlessClient(new HttpClient(new FakeHandler(this)) { BaseAddress = new Uri("http://fwheadless/") }, + NullLogger.Instance); + return InitFwDataProjectJob.AttemptInit(client, _hgService.Object, NullLogger.Instance, ProjectId, Code, Input); + } + + private static HttpResponseMessage Status(ProjectCreationStatus status) => + new(HttpStatusCode.OK) { Content = JsonContent.Create(status) }; + + private static HttpResponseMessage Response(HttpStatusCode code, string body = "") => + new(code) { Content = new StringContent(body) }; + + [Fact] + public async Task A_project_whose_push_already_landed_is_created_without_asking_FwHeadless() + { + _tip = PushedTip; + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().BeEmpty("the repo already shows that an earlier run's creation succeeded"); + } + + [Fact] + public async Task Waits_for_a_creation_still_running_in_FwHeadless_instead_of_starting_another() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => + { + _tip = PushedTip; + return Status(ProjectCreationStatus.Created); + }; + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().Equal(CreationStatus, AwaitCreation); + } + + [Fact] + public async Task A_creation_FwHeadless_reports_as_failed_while_waiting_is_a_failure() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => Status(ProjectCreationStatus.Failed("the push failed")); + + var outcome = await Attempt(); + + outcome.Should().Be(InitAttemptOutcome.Failed("the push failed")); + } + + [Fact] + public async Task A_creation_still_running_after_the_wait_is_retried_later() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => Status(ProjectCreationStatus.TimedOutAwaitingCreation); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, AwaitCreation); + } + + [Fact] + public async Task Clears_FwHeadless_local_state_before_starting_a_creation() + { + _fwHeadless[CreationStatus] = () => Response(HttpStatusCode.NotFound); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => Response(HttpStatusCode.OK); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task FwHeadless_being_busy_with_the_project_is_retried_later_without_starting_a_creation() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.Conflict); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, DeleteLocalProject); + } + + [Theory] + [InlineData(HttpStatusCode.Conflict, InitAttemptResult.RetryLater)] // something else holds the project, so nothing was built + [InlineData(HttpStatusCode.BadRequest, InitAttemptResult.Failed)] + [InlineData(HttpStatusCode.InternalServerError, InitAttemptResult.Failed)] + public async Task An_answer_from_FwHeadless_decides_the_outcome(HttpStatusCode statusCode, InitAttemptResult expected) + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => Response(statusCode, "what went wrong"); + + var outcome = await Attempt(); + + outcome.Should().Be(new InitAttemptOutcome(expected, "what went wrong")); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task No_answer_from_FwHeadless_is_retried_later_because_it_may_still_be_creating() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => throw new HttpRequestException("Connection refused"); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task Not_being_able_to_read_the_repo_is_retried_later() + { + _hgService.Setup(h => h.GetTipHash(Code, It.IsAny())).ThrowsAsync(new HttpRequestException("hg is down")); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().BeEmpty(); + } + + [Fact] + public void The_creation_input_survives_the_job_store() + { + // The job store only takes strings, so the input travels as JSON; it's a positional record with + // IReadOnlyList properties, which System.Text.Json has to bind through the constructor + var roundTripped = System.Text.Json.JsonSerializer.Deserialize( + System.Text.Json.JsonSerializer.Serialize(new InitFwDataProjectInput(["fr", "es"], ["en", "de"], "pt"))); + + roundTripped.Should().NotBeNull(); + roundTripped!.WsVernacular.Should().Equal("fr", "es"); + roundTripped.WsAnalysis.Should().Equal("en", "de"); + roundTripped.WsUi.Should().Be("pt"); + } + + private class FakeHandler(InitFwDataProjectJobTests test) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var key = $"{request.Method} {request.RequestUri!.AbsolutePath}"; + test._requests.Add(key); + if (!test._fwHeadless.TryGetValue(key, out var respond)) + throw new InvalidOperationException($"Unexpected request to FwHeadless: {key}"); + return Task.FromResult(respond()); + } + } +} diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs new file mode 100644 index 0000000000..20256dd027 --- /dev/null +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -0,0 +1,176 @@ +using LexBoxApi.Controllers; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Testing.LexBoxApi; + +/// +/// Unit tests for ProjectController.InitFwDataProject input validation. The 400 branches run before +/// any injected service is touched, so the controller can be built with null dependencies; only a +/// ProblemDetailsFactory (from AddControllers) is needed for Problem() to render. +/// +public class InitFwDataProjectValidationTests +{ + private static ProjectController NewController(IPermissionService? permissionService = null) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves + return new ProjectController(null!, null!, null!, permissionService!, null!, NullLogger.Instance) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() } + } + }; + } + + [Fact] + public async Task Rejects_when_no_vernacular_writing_system_is_supplied() + { + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: []); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("vernacular"); + } + + // The controller's services are all null, so reaching the 400 proves nothing was created first + [Fact] + public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_anything() + { + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr", "fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Duplicate vernacular writing system"); + } + + [Fact] + public async Task Rejects_too_many_analysis_writing_systems_before_creating_anything() + { + var analysis = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsAnalysis: analysis); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("analysis writing systems are allowed"); + } + + [Fact] + public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything() + { + var wsUi = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsUi: wsUi); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("UI writing system code is longer than"); + } + + [Fact] + public async Task Rejects_an_invalid_project_code() + { + var result = await NewController().InitFwDataProject("Bad Code!", name: "My project", wsVernacular: ["fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project code"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task Rejects_a_missing_project_name(string? name) + { + var result = await NewController().InitFwDataProject("myproj", name: name!, wsVernacular: ["fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("A project name is required"); + } + + /// A permission service that either passes or fails the admin assertion. + private static IPermissionService PermissionService(bool isAdmin) + { + var mock = new Mock(); + if (!isAdmin) mock.Setup(p => p.AssertIsAdmin()).Throws(); + return mock.Object; + } + + [Fact] + public async Task Rejects_a_project_origin_that_is_not_a_migration_status() + { + var result = await NewController(PermissionService(isAdmin: true)) + .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin"); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project origin"); + } + + [Fact] + public async Task Rejects_a_project_origin_from_a_non_admin() + { + // The admin check must happen before the value is even parsed, and must outlive [AdminRequired], + // which is expected to be relaxed for this endpoint later. + var act = () => NewController(PermissionService(isAdmin: false)) + .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "Migrated"); + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task Ignores_an_empty_project_origin_without_requiring_admin() + { + // An empty value isn't "specified", so it needs no admin check; a null permission service + // proves AssertIsAdmin() was never called. Fails later (null project service) than validation. + var act = () => NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: ""); + await act.Should().ThrowAsync(); + } + + [Theory] + [InlineData("Migrated", ProjectMigrationStatus.Migrated)] + [InlineData("migrating", ProjectMigrationStatus.Migrating)] + [InlineData("PUBLICREDMINE", ProjectMigrationStatus.PublicRedmine)] + [InlineData("languageforgenonsr", ProjectMigrationStatus.LanguageForgeNonSR)] + public void Project_origin_parses_case_insensitively(string input, ProjectMigrationStatus expected) + { + ProjectController.TryParseProjectOrigin(input, out var origin).Should().BeTrue(); + origin.Should().Be(expected); + } + + [Theory] + [InlineData("NotAnOrigin")] + [InlineData("1")] // Enum.TryParse accepts raw numbers; the API takes names only + [InlineData("42")] + [InlineData("Migrated,Migrating")] // ...and comma-separated lists, which OR into a defined value here + [InlineData("")] + public void Project_origin_rejects_values_that_are_not_named_members(string input) + { + ProjectController.TryParseProjectOrigin(input, out _).Should().BeFalse(); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_null() + { + ProjectController.AnalysisWritingSystemsOrDefault(null).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_empty() + { + ProjectController.AnalysisWritingSystemsOrDefault([]).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_are_kept_when_supplied() + { + ProjectController.AnalysisWritingSystemsOrDefault(["fr", "es"]).Should().Equal("fr", "es"); + } +} diff --git a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs new file mode 100644 index 0000000000..7940d6b65f --- /dev/null +++ b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs @@ -0,0 +1,106 @@ +using LexCore.Entities; +using FluentAssertions; + +namespace Testing.LexCore; + +public class InitFwDataProjectInputTests +{ + private static string? Validate(params string[] tags) => + InitFwDataProjectInput.ValidateWritingSystemList("vernacular", tags); + + [Fact] + public void Accepts_a_typical_list() + { + Validate("fr", "en-US", "qaa-Latn-x-mycustom").Should().BeNull(); + } + + [Fact] + public void Rejects_a_null_list() + { + InitFwDataProjectInput.ValidateWritingSystemList("vernacular", null) + .Should().Contain("At least one vernacular writing system is required"); + } + + [Fact] + public void Rejects_an_empty_list() + { + Validate().Should().Contain("At least one vernacular writing system is required"); + } + + [Fact] + public void Accepts_the_maximum_number_of_writing_systems() + { + var tags = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList).Select(i => $"qaa-x-ws{i}").ToArray(); + Validate(tags).Should().BeNull(); + } + + [Fact] + public void Rejects_more_than_the_maximum_number_of_writing_systems() + { + var tags = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); + Validate(tags).Should().Contain($"At most {InitFwDataProjectInput.MaxWritingSystemsPerList}"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Rejects_an_empty_tag(string tag) + { + Validate("fr", tag).Should().Contain("Empty vernacular writing system code"); + } + + [Fact] + public void Accepts_a_tag_of_the_maximum_length() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength - "qaa-x-".Length); + Validate(tag).Should().BeNull(); + } + + [Fact] + public void Rejects_a_tag_longer_than_the_maximum_and_shows_its_start() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var error = Validate("fr", tag); + error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); + error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); + } + + [Theory] + [InlineData("fr", "fr")] + [InlineData("fr", "FR")] // BCP 47 tags are case-insensitive + public void Rejects_a_duplicate_tag(string first, string second) + { + Validate(first, "en", second).Should().Contain($"Duplicate vernacular writing system: {first}"); + } + + [Fact] + public void Accepts_a_typical_UI_writing_system() + { + InitFwDataProjectInput.ValidateUiWritingSystem("en").Should().BeNull(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Rejects_a_missing_UI_writing_system(string? tag) + { + InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().Contain("A UI writing system is required"); + } + + [Fact] + public void Accepts_a_UI_writing_system_of_the_maximum_length() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength - "qaa-x-".Length); + InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().BeNull(); + } + + [Fact] + public void Rejects_a_UI_writing_system_longer_than_the_maximum_and_shows_its_start() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var error = InitFwDataProjectInput.ValidateUiWritingSystem(tag); + error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); + error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); + } +} diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs new file mode 100644 index 0000000000..aac5a48b53 --- /dev/null +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -0,0 +1,234 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Xml.Linq; +using FluentAssertions; +using LexCore.Entities; +using LexCore.Sync; +using Testing.ApiTests; +using Testing.Fixtures; +using Testing.Services; +using Xunit.Abstractions; +using static Testing.Services.Constants; + +namespace Testing.SyncReverseProxy; + +/// +/// End-to-end coverage for the admin "init fwdata project" API. This is the one path that +/// can't be unit-tested: LexBox creates an empty hg repo and FwHeadless does the first push of a +/// template-built .fwdata into it (via LfMergeBridge/Chorus against the real hgweb). Requires the +/// lexbox stack to be up and running, so only run it in CI or with `task test:integration`. +/// +[Trait("Category", "Integration")] +public class InitFwDataProjectTests : IClassFixture +{ + private readonly ITestOutputHelper _output; + private readonly ApiTestBase _adminApiTester; + + public InitFwDataProjectTests(ITestOutputHelper output, IntegrationFixture fixture) + { + _output = output; + _adminApiTester = fixture.AdminApiTester; + } + + [Fact] + public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWritingSystems() + { + // Valid code (lowercase/digits/hyphen, doesn't start with a hyphen), unique per run. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var vernacular = new[] { "fr", "es", "ko", "ru", "my" }; + var analysis = new[] { "de", "pt", "th", "id", "he", "el" }; + var query = $"?code={code}&name={code}" + + string.Concat(vernacular.Select(ws => $"&wsVernacular={ws}")) + + string.Concat(analysis.Select(ws => $"&wsAnalysis={ws}")); + + Guid projectId = default; + try + { + // 1. Create the project via the admin endpoint. It creates the DB row + empty repo and returns + // 202 at once; a background job then has FwHeadless build the template .fwdata and push it + // into the empty repo, and we poll until that has finished. + // The POST isn't idempotent. The shared tester's HttpClient retries transient failures, and a + // retried attempt (after the first already created the project row) would return 409 "already + // exists", so use a client that doesn't retry. Log it in itself (cookie auth) rather than + // reusing the shared tester's captured JWT, which may be stale. + using var createClient = ApiTestBase.NewHttpClient(_adminApiTester.BaseUrl, retry: false).Client; + await JwtHelper.ExecuteLogin(AdminAuth, includeDefaultScope: true, createClient); + var response = await createClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject{query}", null); + response.StatusCode.Should().Be(HttpStatusCode.Accepted, + "creation should be accepted; body: {0}", await response.Content.ReadAsStringAsync()); + projectId = await response.Content.ReadFromJsonAsync(); + projectId.Should().NotBe(Guid.Empty); + response.Headers.Location?.ToString().Should().EndWith($"/api/project/fwDataProjectCreationStatus/{projectId}"); + + var status = await AwaitCreationFinished(createClient, projectId); + status.Status.Should().Be(ProjectCreationStatusEnum.Created, "creation should succeed; error: {0}", status.Error); + + // 2. The first push landed a commit server-side (the empty repo is no longer empty). + var lastCommit = await _adminApiTester.GetProjectLastCommit(code); + lastCommit.Should().NotBeNull("the template project should have been pushed to the repo"); + + var tagsResponse = await _adminApiTester.HttpClient.GetAsync($"{_adminApiTester.BaseUrl}/hg/{code}/tags?style=json"); + tagsResponse.EnsureSuccessStatusCode(); + var tip = (await tagsResponse.Content.ReadFromJsonAsync())?["node"]?.ToString(); + tip.Should().NotBeNullOrEmpty(); + tip!.Replace("0", "").Should().NotBeEmpty("the repo tip should not be the all-zero empty-repo hash"); + + // 3. The pushed project carries the requested writing systems. Send/Receive split the + // template .fwdata into the nested files hg actually tracks; the LangProject's current + // analysis/vernacular writing systems live in General/LanguageProject.langproj (XML). + var langprojResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/General/LanguageProject.langproj"); + langprojResponse.EnsureSuccessStatusCode(); + var langprojXml = await langprojResponse.Content.ReadAsStringAsync(); + langprojXml.Should().NotBeEmpty(); + + // de en pt + // fr es ... + var langProject = XDocument.Parse(langprojXml).Root?.Element("LangProject"); + langProject.Should().NotBeNull("LanguageProject.langproj should contain a LangProject element"); + var allAnalysisWss = SpaceSeparatedUni(langProject!, "AnalysisWss"); + var allVernWss = SpaceSeparatedUni(langProject!, "VernWss"); + var curAnalysisWss = SpaceSeparatedUni(langProject!, "CurAnalysisWss"); + var curVernWss = SpaceSeparatedUni(langProject!, "CurVernWss"); + + // 4. The largest file survived the push. Chorus's LargeFileFilter caps a file at 1 MB unless + // some loaded file-type handler raises the cap for that extension, and the handler that knows + // about .list ships in LibFLExBridge-ChorusPlugin.dll. If plugin discovery ever regresses + // (sillsdev/chorus#393), .list silently falls back to the 1 MB default and Chorus *forgets* + // SemanticDomainList.list instead of pushing it -- no error, no failed send/receive, just a + // project that looks fine here and is missing its semantic domains when FLEx opens it. + // Every other assertion in this test reads a small file, so they all stay green through + // that failure; this is the only one that catches it. + var semDomResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/Linguistics/Lexicon/SemanticDomainList.list"); + semDomResponse.StatusCode.Should().Be(HttpStatusCode.OK, + "SemanticDomainList.list should have been pushed; anything else (404 in particular) means " + + "Chorus filtered it out as oversized, i.e. the FieldWorks file-type handlers never loaded"); + + // The requested writing systems should be current. FieldWorks may add its own defaults (e.g. + // "en" as an analysis WS), so assert each requested code is present rather than exact equality. + foreach (var ws in analysis) + curAnalysisWss.Should().Contain(ws, "analysis writing system {0} should be current in the project", ws); + foreach (var ws in vernacular) + curVernWss.Should().Contain(ws, "vernacular writing system {0} should be current in the project", ws); + + // The requested writing systems should *also* be in the correct order. Any writing systems added + // by FieldWorks should come last. + allAnalysisWss.Take(analysis.Length).Should().Equal(analysis, "analysis writing systems were in the wrong order"); + allVernWss.Take(vernacular.Length).Should().Equal(vernacular, "vernacular writing systems were in the wrong order"); + curAnalysisWss.Should().Equal(analysis, "current analysis writing systems should not contain any extras"); + curVernWss.Should().Equal(vernacular, "current vernacular writing systems should not contain any extras"); + } + finally + { + if (projectId != default) await SoftDeleteProject(projectId); + } + } + + /// + /// Only admins may say where a project came from, so a non-admin passing projectOrigin + /// must be rejected. Today it's [AdminRequired] on the endpoint that rejects it (403); once this + /// endpoint opens up to non-admins and that attribute goes away, the controller's own + /// AssertIsAdmin() will reject it (401). Either way the request must fail and no project may be + /// created, which is what this test pins down across that change. + /// To check it isn't passing for the wrong reason, temporarily delete [AdminRequired] from + /// ProjectController.InitFwDataProject, rebuild, and re-run: it must still pass. + /// + [Fact] + public async Task InitFwDataProject_RejectsProjectOriginFromANonAdmin() + { + var managerApiTester = new ApiTestBase(); + await managerApiTester.LoginAs(ManagerAuth.Username, ManagerAuth.Password); + + // Everything except projectOrigin is valid, so a 400 from input validation can't be what + // rejects this, and the origin itself is a real ProjectMigrationStatus name. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await managerApiTester.HttpClient.PostAsync( + $"{managerApiTester.BaseUrl}/api/project/initFwDataProject" + + $"?code={code}&name={code}&wsVernacular=fr&projectOrigin={nameof(ProjectMigrationStatus.LanguageForgeNonSR)}", null); + + if (response.StatusCode == HttpStatusCode.InternalServerError) + { + // In developer mode, a thrown System.UnauthorizedAccessException exception results in an HTTP 500 + // instead of a 4xx code, with a JSON structure that includes `"title":"System.UnauthorizedAccessException"`. + // We check for that special case here; any non-JSON or JSON that has a different title would indicate a + // different problem and be a test failure. + JsonNode? problemDetails = null; + try + { + var body = await response.Content.ReadAsStringAsync(); + problemDetails = JsonNode.Parse(body); + } + catch (JsonException) + { + // Do nothing here, the Should().NotBeNull() check below handles this + } + var failureReasonMsg = "HTTP 500 should have been caused by developer page printing " + + "a traceback due to " + nameof(UnauthorizedAccessException); + problemDetails.Should().NotBeNull(failureReasonMsg); + problemDetails?["title"]?.ToString().Should().EndWithEquivalentOf( + nameof(UnauthorizedAccessException), failureReasonMsg); + } + else + { + response.StatusCode.Should().BeOneOf([HttpStatusCode.Forbidden, HttpStatusCode.Unauthorized], + "a non-admin may not set projectOrigin; body: {0}", await response.Content.ReadAsStringAsync()); + } + } + + [Fact] + public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() + { + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await _adminApiTester.HttpClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject?code={code}&name={code}&wsAnalysis=en", null); + response.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } + + /// Polls the creation status until the background job has finished, for up to five minutes. + private async Task AwaitCreationFinished(HttpClient client, Guid projectId) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + while (true) + { + var response = await client.GetAsync( + $"{_adminApiTester.BaseUrl}/api/project/fwDataProjectCreationStatus/{projectId}", timeout.Token); + response.StatusCode.Should().Be(HttpStatusCode.OK, + "the project's creation status should be known; body: {0}", await response.Content.ReadAsStringAsync()); + var status = await response.Content.ReadFromJsonAsync(timeout.Token); + status.Should().NotBeNull(); + _output.WriteLine($"Creation status: {status!.Status}"); + if (status.IsFinished) return status; + await Task.Delay(TimeSpan.FromSeconds(2), timeout.Token); + } + } + + // Reads a LangProject child element's text (a space-separated writing-system list) and splits it. + private static string[] SpaceSeparatedUni(XElement langProject, string elementName) + { + var uni = langProject.Element(elementName)?.Element("Uni")?.Value ?? ""; + return uni.Split(' ', StringSplitOptions.RemoveEmptyEntries); + } + + private async Task SoftDeleteProject(Guid projectId) + { + try + { + await _adminApiTester.ExecuteGql($$""" + mutation { + softDeleteProject(input: { projectId: "{{projectId}}" }) { + project { id } + errors { __typename } + } + } + """); + } + catch (Exception ex) + { + _output.WriteLine($"[InitFwDataProjectTests] Ignored cleanup exception: {ex}"); + } + } +} diff --git a/frontend/schema.graphql b/frontend/schema.graphql index bf39d4915b..1936019704 100644 --- a/frontend/schema.graphql +++ b/frontend/schema.graphql @@ -451,6 +451,7 @@ type Project { repoSizeInKb: Int deletedDate: DateTime resetStatus: ResetStatus! + fwDataInitStatus: FwDataInitStatus! projectOrigin: ProjectMigrationStatus! migratedDate: DateTime userCount: Int! @@ -980,6 +981,13 @@ input FlexProjectMetadataSortInput { flexModelVersion: SortEnumType @cost(weight: "10") } +input FwDataInitStatusOperationFilterInput { + eq: FwDataInitStatus @cost(weight: "10") + neq: FwDataInitStatus @cost(weight: "10") + in: [FwDataInitStatus!] @cost(weight: "10") + nin: [FwDataInitStatus!] @cost(weight: "10") +} + input HybridDateTimeFilterInput { and: [HybridDateTimeFilterInput!] or: [HybridDateTimeFilterInput!] @@ -1228,6 +1236,7 @@ input ProjectFilterInput { repoSizeInKb: IntOperationFilterInput deletedDate: DateTimeOperationFilterInput resetStatus: ResetStatusOperationFilterInput + fwDataInitStatus: FwDataInitStatusOperationFilterInput projectOrigin: ProjectMigrationStatusOperationFilterInput migratedDate: DateTimeOperationFilterInput harmonyCommits: ListFilterInputTypeOfServerCommitFilterInput @@ -1264,6 +1273,7 @@ input ProjectSortInput { repoSizeInKb: SortEnumType @cost(weight: "10") deletedDate: SortEnumType @cost(weight: "10") resetStatus: SortEnumType @cost(weight: "10") + fwDataInitStatus: SortEnumType @cost(weight: "10") projectOrigin: SortEnumType @cost(weight: "10") migratedDate: SortEnumType @cost(weight: "10") userCount: SortEnumType @cost(weight: "10") @@ -1501,6 +1511,11 @@ enum FeatureFlag { FW_LITE_BETA } +enum FwDataInitStatus { + NONE + IN_PROGRESS +} + enum JsonValueKind { UNDEFINED OBJECT @@ -1552,6 +1567,7 @@ enum ProjectMigrationStatus { MIGRATING PRIVATE_REDMINE PUBLIC_REDMINE + LANGUAGE_FORGE_NON_SR } enum ProjectRole { From e3fe2f148781e77760b7f1f13a34b503ccdb10b7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 12 Nov 2026 11:56:57 +0700 Subject: [PATCH 2/6] Reject invalid writing system tags before creating a project LexBox now checks that every writing system tag is valid BCP 47 before it creates the project row and repo, so a bad tag gets a 400 instead of a 202 followed by a rollback. The check lives in the validators LexBox and FwHeadless share. FwHeadlessClient.InitFwDataProject also never returns an empty error for a failed response, since the job reads a missing error as success. Co-Authored-By: Claude Opus 5.5 --- backend/FwHeadless/Routes/ProjectRoutes.cs | 8 ----- .../LexBoxApi/Services/FwHeadlessClient.cs | 5 +++- .../Entities/InitFwDataProjectInput.cs | 14 ++++++--- backend/LexCore/LexCore.csproj | 2 ++ .../InitFwDataProjectValidationTests.cs | 10 +++++++ .../Services/FwHeadlessClientTests.cs | 12 ++++++++ .../LexCore/InitFwDataProjectInputTests.cs | 30 +++++++++++++++++-- 7 files changed, 66 insertions(+), 15 deletions(-) diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs index dd3e540c4f..c54f1e67e9 100644 --- a/backend/FwHeadless/Routes/ProjectRoutes.cs +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -4,7 +4,6 @@ using LexCore.Sync; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Options; -using SIL.WritingSystems; namespace FwHeadless.Routes; @@ -35,13 +34,6 @@ private static async Task> InitFwDataProject( ?? InitFwDataProjectInput.ValidateUiWritingSystem(input.WsUi); if (wsError is not null) return TypedResults.Problem(wsError, statusCode: StatusCodes.Status400BadRequest); - var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); - if (invalidWs is not null) - return TypedResults.Problem($"Invalid writing system code: {invalidWs}", - statusCode: StatusCodes.Status400BadRequest); - if (!IetfLanguageTag.IsValid(input.WsUi)) - return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", - statusCode: StatusCodes.Status400BadRequest); var projectCode = await projectLookupService.GetProjectCode(projectId); if (projectCode is null) diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 549b9e9a02..857757fe44 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -235,7 +235,10 @@ private static async Task FailedResponse(HttpResponseMessag response.ReasonPhrase, projectId, responseBody); - return (response.StatusCode, responseBody); + // Never null on failure: AttemptInit reads a null Error as success, and a failure body can be empty + return (response.StatusCode, string.IsNullOrWhiteSpace(responseBody) + ? $"FwHeadless returned HTTP {(int)response.StatusCode}" + : responseBody); } /// diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs index 1785126854..fc117f4ab9 100644 --- a/backend/LexCore/Entities/InitFwDataProjectInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -1,3 +1,5 @@ +using SIL.WritingSystems; + namespace LexCore.Entities; /// @@ -17,8 +19,8 @@ public record InitFwDataProjectInput( public const int MaxWritingSystemTagLength = 128; /// - /// Shape checks shared by LexBoxApi (before it creates anything) and FwHeadless. Returns an error - /// message, or null if the list is acceptable. FwHeadless additionally checks that each tag is valid BCP 47. + /// Checks shared by LexBoxApi (before it creates anything) and FwHeadless. Returns an error + /// message, or null if the list is acceptable. /// public static string? ValidateWritingSystemList(string kind, IReadOnlyList? tags) { @@ -33,12 +35,14 @@ public record InitFwDataProjectInput( // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate) return $"Duplicate {kind} writing system: {duplicate.Key}"; + if (tags.FirstOrDefault(t => !IetfLanguageTag.IsValid(t)) is { } invalid) + return $"Invalid {kind} writing system code: {invalid}"; return null; } /// - /// Shape checks for the UI writing system, shared like . Returns an - /// error message, or null if the tag is acceptable. FwHeadless additionally checks that it is valid BCP 47. + /// Checks for the UI writing system, shared like . Returns an + /// error message, or null if the tag is acceptable. /// public static string? ValidateUiWritingSystem(string? tag) { @@ -46,6 +50,8 @@ public record InitFwDataProjectInput( return "A UI writing system is required"; if (tag.Length > MaxWritingSystemTagLength) return $"The UI writing system code is longer than {MaxWritingSystemTagLength} characters: {tag[..32]}…"; + if (!IetfLanguageTag.IsValid(tag)) + return $"Invalid UI writing system code: {tag}"; return null; } } diff --git a/backend/LexCore/LexCore.csproj b/backend/LexCore/LexCore.csproj index 8a1b8a72ff..cded3b2bad 100644 --- a/backend/LexCore/LexCore.csproj +++ b/backend/LexCore/LexCore.csproj @@ -10,6 +10,8 @@ LexCore is a plain library (no ASP.NET framework reference), so these come as packages. --> + + diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index 20256dd027..ba23287c37 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -51,6 +51,16 @@ public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_ problem.Detail.Should().Contain("Duplicate vernacular writing system"); } + [Fact] + public async Task Rejects_an_invalid_vernacular_writing_system_before_creating_anything() + { + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["not_a_tag"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Be("Invalid vernacular writing system code: not_a_tag"); + } + [Fact] public async Task Rejects_too_many_analysis_writing_systems_before_creating_anything() { diff --git a/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs b/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs index 36fb1f75c5..6e7041825b 100644 --- a/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs +++ b/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs @@ -81,4 +81,16 @@ public async Task DeleteRepo_ThrowsProjectSyncInProgress_OnConflict() await act.Should().ThrowAsync(); } + + [Fact] + public async Task InitFwDataProject_ReturnsAnError_ForAFailureWithAnEmptyBody() + { + // e.g. a FwHeadless that doesn't have the route yet. A null error would read as success. + var client = ClientReturning(HttpStatusCode.NotFound, "", "text/plain"); + + var (statusCode, error) = await client.InitFwDataProject(Guid.NewGuid(), ["fr"], ["en"], "en"); + + statusCode.Should().Be(HttpStatusCode.NotFound); + error.Should().Be("FwHeadless returned HTTP 404"); + } } diff --git a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs index 7940d6b65f..fa40c1aaa9 100644 --- a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs +++ b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs @@ -8,6 +8,17 @@ public class InitFwDataProjectInputTests private static string? Validate(params string[] tags) => InitFwDataProjectInput.ValidateWritingSystemList("vernacular", tags); + // A valid BCP 47 tag of the given length. Private-use subtags are at most 8 characters each, and + // libpalaso also rejects a repeated one, so each is different. + private static string ValidTagOfLength(int length) + { + var tag = "qaa-x"; + var i = 0; + while (tag.Length + 9 <= length) tag += $"-p{i++:D7}"; + if (tag.Length < length) tag += "-" + $"q{i:D7}"[..(length - tag.Length - 1)]; + return tag; + } + [Fact] public void Accepts_a_typical_list() { @@ -52,7 +63,8 @@ public void Rejects_an_empty_tag(string tag) [Fact] public void Accepts_a_tag_of_the_maximum_length() { - var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength - "qaa-x-".Length); + var tag = ValidTagOfLength(InitFwDataProjectInput.MaxWritingSystemTagLength); + tag.Length.Should().Be(InitFwDataProjectInput.MaxWritingSystemTagLength); Validate(tag).Should().BeNull(); } @@ -65,6 +77,14 @@ public void Rejects_a_tag_longer_than_the_maximum_and_shows_its_start() error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); } + [Theory] + [InlineData("not_a_tag")] + [InlineData("qaa-x-thissubtagistoolong")] + public void Rejects_a_tag_that_is_not_valid_BCP_47(string tag) + { + Validate("fr", tag).Should().Be($"Invalid vernacular writing system code: {tag}"); + } + [Theory] [InlineData("fr", "fr")] [InlineData("fr", "FR")] // BCP 47 tags are case-insensitive @@ -91,7 +111,7 @@ public void Rejects_a_missing_UI_writing_system(string? tag) [Fact] public void Accepts_a_UI_writing_system_of_the_maximum_length() { - var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength - "qaa-x-".Length); + var tag = ValidTagOfLength(InitFwDataProjectInput.MaxWritingSystemTagLength); InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().BeNull(); } @@ -103,4 +123,10 @@ public void Rejects_a_UI_writing_system_longer_than_the_maximum_and_shows_its_st error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); } + + [Fact] + public void Rejects_a_UI_writing_system_that_is_not_valid_BCP_47() + { + InitFwDataProjectInput.ValidateUiWritingSystem("not_a_tag").Should().Be("Invalid UI writing system code: not_a_tag"); + } } From 5a2b8c5afbcdef773b1748d8fcf4565ea4d82122 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 17 Nov 2026 17:46:46 +0700 Subject: [PATCH 3/6] Fix project code and writing system bugs in project creation - Reject a project code with a trailing newline. Regex $ matched before it, and the hg URL then named a different project. - Set the current writing system lists to exactly what was requested. liblcm added an unrequested "en" analysis writing system, repeated "en" when it was requested, and kept equivalent tags like fr and fr-Latn twice. - Work around a liblcm bug that made creation throw when the default analysis writing system's ICU locale differs from its tag (pt-BR). Co-Authored-By: Claude Opus 5.5 --- .../LcmUtils/ProjectLoader.cs | 93 ++++++++++++++++--- .../NewProjectWritingSystemsTests.cs | 43 +++++++-- .../Controllers/ProjectController.cs | 3 +- .../Entities/InitFwDataProjectInput.cs | 8 +- backend/LexCore/Entities/ProjectCode.cs | 13 ++- .../InitFwDataProjectValidationTests.cs | 9 +- backend/Testing/LexCore/ProjectCodeTests.cs | 2 + 7 files changed, 140 insertions(+), 31 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index db3b5b83f6..1e2b22fc3a 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using Microsoft.Extensions.Options; using SIL.LCModel; +using SIL.LCModel.Infrastructure; using SIL.LCModel.Core.WritingSystems; using SIL.LCModel.Utils; using SIL.WritingSystems; @@ -96,28 +97,98 @@ public virtual LcmCache NewProject(FwDataProject project, var progress = new LcmThreadedProgress(); var analysisDefinitions = analysisWss.Select(CreateWritingSystemDefinition).ToList(); var vernacularDefinitions = vernacularWss.Select(CreateWritingSystemDefinition).ToList(); + // A liblcm bug: when no anthropology list file is passed (FLEx passes one unless the user picks an + // empty list), CreateNewLangProj names the empty list in the default analysis WS, which it looks up + // with GetWsFromStr(IcuLocale). GetWsFromStr expects a language tag, so this only works when the + // two match: "pt-BR" is "pt_BR" in ICU, the lookup returns 0, and creation throws with the project + // already on disk. Remove this workaround once liblcm passes the tag. Until then hand liblcm "en" as the default + // and pass every requested analysis WS as additional; SetRequestedWritingSystems fixes the default. + var substituteDefaultAnalysis = !IcuLocaleMatchesTag(analysisDefinitions[0]); + var defaultAnalysis = substituteDefaultAnalysis ? CreateWritingSystemDefinition("en") : analysisDefinitions[0]; NewProject(progress, project.Name, lcmDirectories, progress.SynchronizeInvoke, - analysisDefinitions[0], + defaultAnalysis, vernacularDefinitions[0], uiWs, - AdditionalWritingSystems(analysisDefinitions), - AdditionalWritingSystems(vernacularDefinitions)); - return LoadCache(project); + AdditionalWritingSystems(analysisDefinitions, defaultAnalysis), + AdditionalWritingSystems(vernacularDefinitions, vernacularDefinitions[0])); + var cache = LoadCache(project); + try + { + SetRequestedWritingSystems(cache, analysisWss, vernacularWss, substituteDefaultAnalysis); + } + catch + { + cache.Dispose(); + throw; + } + return cache; } private static CoreWritingSystemDefinition CreateWritingSystemDefinition(string ws) => new(ws) { Id = ws }; - // The default writing system (the first of the list) is passed to CreateNewLangProj on its own, so - // the "additional" set is the whole list as a hash set with that default removed. The new project - // ends up with them in list order (NewProjectWritingSystemsTests checks this). - private static HashSet AdditionalWritingSystems(List definitions) + private static bool IcuLocaleMatchesTag(CoreWritingSystemDefinition definition) => + string.Equals(definition.IcuLocale, definition.Id, StringComparison.OrdinalIgnoreCase); + + // Everything except the default, which CreateNewLangProj takes on its own + private static HashSet AdditionalWritingSystems( + List definitions, + CoreWritingSystemDefinition defaultDefinition) => + definitions.Where(d => !string.Equals(d.Id, defaultDefinition.Id, StringComparison.OrdinalIgnoreCase)).ToHashSet(); + + /// + /// Makes the current analysis and vernacular lists exactly the requested writing systems, in the + /// requested order. CreateNewLangProj doesn't on its own: it adds "en" as a current analysis WS + /// whenever another one is the default, appends the "additional" sets without checking for one that's + /// already there (so "en" listed later, or "fr" plus "fr-Latn", end up twice), and takes them as + /// unordered sets. Writing systems it added that weren't asked for stay in the full lists, unselected. + /// + private static void SetRequestedWritingSystems(LcmCache cache, + IReadOnlyList analysisWss, + IReadOnlyList vernacularWss, + bool substitutedDefaultAnalysis) + { + var container = cache.ServiceLocator.WritingSystems; + var wsManager = cache.ServiceLocator.WritingSystemManager; + // GetOrSet resolves a tag to the WS liblcm already created for it, also when liblcm stored it under + // an equivalent tag (e.g. "fr" for "fr-Latn") + List Resolve(IReadOnlyList tags) => tags.Select(tag => + { + wsManager.GetOrSet(tag, out var ws); + return ws; + }).Distinct().ToList(); + + NonUndoableUnitOfWorkHelper.Do(cache.ActionHandlerAccessor, () => + { + var analysis = Resolve(analysisWss); + var vernacular = Resolve(vernacularWss); + // liblcm doesn't keep the full lists in the same order as the current lists, so set both + SetList(container.AnalysisWritingSystems, [.. analysis, .. container.AnalysisWritingSystems.Except(analysis)]); + SetList(container.VernacularWritingSystems, [.. vernacular, .. container.VernacularWritingSystems.Except(vernacular)]); + SetList(container.CurrentAnalysisWritingSystems, analysis); + SetList(container.CurrentVernacularWritingSystems, vernacular); + if (substitutedDefaultAnalysis) MoveReversalIndexToDefaultAnalysis(cache, analysis[0]); + }); + cache.ActionHandlerAccessor.Commit(); + } + + private static void SetList(ICollection list, List wanted) + { + if (list.SequenceEqual(wanted)) return; + list.Clear(); + foreach (var ws in wanted) list.Add(ws); + } + + // CreateNewLangProj creates a reversal index for the default analysis WS, which was "en" only because + // the requested default couldn't be passed in. Give the requested default the index instead. + private static void MoveReversalIndexToDefaultAnalysis(LcmCache cache, CoreWritingSystemDefinition defaultAnalysis) { - var additional = definitions.ToHashSet(); - additional.Remove(definitions[0]); - return additional; + cache.ServiceLocator.GetInstance().FindOrCreateIndexForWs(defaultAnalysis.Handle); + if (cache.ServiceLocator.WritingSystems.CurrentAnalysisWritingSystems.Any(ws => ws.Id == "en")) return; + var englishIndex = cache.LangProject.LexDbOA.ReversalIndexesOC.FirstOrDefault(ri => ri.WritingSystem == "en"); + if (englishIndex is not null) cache.LangProject.LexDbOA.ReversalIndexesOC.Remove(englishIndex); } private static void NewProject(IThreadedProgress progress, diff --git a/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs index aaa4e80d39..4ee17a0b19 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs @@ -37,24 +37,47 @@ public async Task DisposeAsync() await _rootServiceProvider.DisposeAsync(); } - [Fact] - public async Task NewProjectAddsEveryRequestedWritingSystemToTheCorrectListInOrder() + [Theory] + // Distinct codes per type so a swapped analysis/vernacular argument would fail, and several additional + // ones per type because CreateNewLangProj takes those as an unordered set + [InlineData("en,de,pt,ru", "fr,es,it,nl", "en,de,pt,ru", "fr,es,it,nl")] + // liblcm adds "en" as a current analysis WS whenever another one is the default + [InlineData("de,pt", "fr", "de,pt", "fr")] + // ...and appends a later "en" a second time + [InlineData("de,en,pt", "fr", "de,en,pt", "fr")] + // liblcm can't take a default analysis WS whose ICU locale differs from its tag ("pt_BR") + [InlineData("pt-BR,en", "fr", "pt-BR,en", "fr")] + [InlineData("qaa-x-foo", "fr", "qaa-x-foo", "fr")] + // Tags liblcm treats as the same WS end up once + [InlineData("en", "fr,fr-Latn", "en", "fr")] + public async Task NewProjectHasExactlyTheRequestedWritingSystemsInOrder( + string analysis, string vernacular, string expectedAnalysis, string expectedVernacular) { var fwDataBridgeConfig = Services.GetRequiredService>().Value; // The SIL.LCModel template ships next to the test assembly; point liblcm at it (this project, // unlike FwDataMiniLcmBridge.Tests, doesn't otherwise configure TemplatesFolder). fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); var fwDataProject = new FwDataProject("multi-ws-source", fwDataBridgeConfig.ProjectsFolder); - // First of each list is the default; the rest go through CreateNewLangProj's additional-WS sets. - // Distinct codes per type so a swapped analysis/vernacular argument would fail this test, and - // several additional ones per type because CreateNewLangProj takes those as an unordered set. - string[] analysis = ["en", "de", "pt", "ru"]; - string[] vernacular = ["fr", "es", "it", "nl"]; - Services.GetRequiredService().NewProject(fwDataProject, analysis, vernacular, uiWs: "en").Dispose(); + Services.GetRequiredService() + .NewProject(fwDataProject, analysis.Split(','), vernacular.Split(','), uiWs: "en").Dispose(); + // Reopened from disk, so this also checks the fixed-up lists were saved using var api = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); var writingSystems = await api.GetWritingSystems(); - writingSystems.Analysis.Select(ws => ws.WsId.Code).Should().Equal(analysis); - writingSystems.Vernacular.Select(ws => ws.WsId.Code).Should().Equal(vernacular); + writingSystems.Analysis.Select(ws => ws.WsId.Code).Should().Equal(expectedAnalysis.Split(',')); + writingSystems.Vernacular.Select(ws => ws.WsId.Code).Should().Equal(expectedVernacular.Split(',')); + } + + [Fact] + public void ADefaultAnalysisWritingSystemLiblcmCantTakeStillGetsTheReversalIndex() + { + var fwDataBridgeConfig = Services.GetRequiredService>().Value; + fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + var fwDataProject = new FwDataProject("reversal-source", fwDataBridgeConfig.ProjectsFolder); + + // "en" is passed to liblcm as the default instead, so it would otherwise get the reversal index + using var cache = Services.GetRequiredService().NewProject(fwDataProject, ["pt-BR"], ["fr"], uiWs: "en"); + + cache.LangProject.LexDbOA.ReversalIndexesOC.Select(ri => ri.WritingSystem).Should().Equal("pt-BR"); } } diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index 739a4001ff..09ff8e1d43 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,4 +1,3 @@ -using System.Text.RegularExpressions; using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; using LexBoxApi.Jobs; @@ -66,7 +65,7 @@ public async Task> InitFwDataProject( ?? InitFwDataProjectInput.ValidateUiWritingSystem(wsUi); if (wsError is not null) return Problem(wsError, statusCode: StatusCodes.Status400BadRequest); - if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) + if (code is null || code.Length < 4 || !ProjectCode.IsValidCode(code)) return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); if (string.IsNullOrWhiteSpace(name)) return Problem("A project name is required", statusCode: StatusCodes.Status400BadRequest); diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs index fc117f4ab9..ae0bbfa03e 100644 --- a/backend/LexCore/Entities/InitFwDataProjectInput.cs +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -32,11 +32,13 @@ public record InitFwDataProjectInput( return $"Empty {kind} writing system code"; if (tags.FirstOrDefault(t => t.Length > MaxWritingSystemTagLength) is { } tooLong) return $"The {kind} writing system code is longer than {MaxWritingSystemTagLength} characters: {tooLong[..32]}…"; - // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system - if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate) - return $"Duplicate {kind} writing system: {duplicate.Key}"; if (tags.FirstOrDefault(t => !IetfLanguageTag.IsValid(t)) is { } invalid) return $"Invalid {kind} writing system code: {invalid}"; + // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system. Tags that differ only + // by an implicit script ("fr", "fr-Latn") get through, because spotting those needs the SLDR, which + // LexBox doesn't load; ProjectLoader.NewProject merges them into one writing system. + if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate) + return $"Duplicate {kind} writing system: {duplicate.Key}"; return null; } diff --git a/backend/LexCore/Entities/ProjectCode.cs b/backend/LexCore/Entities/ProjectCode.cs index 51ac881f4f..edc50cb69f 100644 --- a/backend/LexCore/Entities/ProjectCode.cs +++ b/backend/LexCore/Entities/ProjectCode.cs @@ -34,10 +34,19 @@ private void AssertIsSafeRepoName(string name) { if (InvalidRepoNames.Contains(name)) throw new ArgumentException($"Invalid repo name: {name}."); - if (!ProjectCodeRegex().IsMatch(name)) + if (!IsValidCode(name)) throw new ArgumentException($"Invalid repo name: {name}."); } - [GeneratedRegex(Project.ProjectCodeRegex)] + /// + /// Whether the whole string matches . Use this rather than + /// Regex.IsMatch with that pattern: its $ also matches before a trailing newline, so "abcd" + /// plus a line feed would pass, and hg URLs drop the newline, so they'd point at project "abcd". + /// + public static bool IsValidCode(string? code) => code is not null && ProjectCodeRegex().IsMatch(code); + + // Project.ProjectCodeRegex with \z instead of $ (see IsValidCode). That one can stay as it is for + // [RegularExpression], which already requires the match to cover the whole string. + [GeneratedRegex(@"^[a-z\d][a-z-\d]*\z")] private static partial Regex ProjectCodeRegex(); } diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs index ba23287c37..97e9fe0cd9 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -83,10 +83,13 @@ public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything( problem.Detail.Should().Contain("UI writing system code is longer than"); } - [Fact] - public async Task Rejects_an_invalid_project_code() + [Theory] + [InlineData("Bad Code!")] + // A trailing newline passes a Regex.IsMatch with $, and the hg URL would then name project "abcd" + [InlineData("abcd\n")] + public async Task Rejects_an_invalid_project_code(string code) { - var result = await NewController().InitFwDataProject("Bad Code!", name: "My project", wsVernacular: ["fr"]); + var result = await NewController().InitFwDataProject(code, name: "My project", wsVernacular: ["fr"]); var problem = result.Result.Should().BeOfType() .Which.Value.Should().BeOfType().Which; problem.Status.Should().Be(StatusCodes.Status400BadRequest); diff --git a/backend/Testing/LexCore/ProjectCodeTests.cs b/backend/Testing/LexCore/ProjectCodeTests.cs index 8844176423..0b42711bc6 100644 --- a/backend/Testing/LexCore/ProjectCodeTests.cs +++ b/backend/Testing/LexCore/ProjectCodeTests.cs @@ -17,6 +17,8 @@ public class ProjectCodeTests [InlineData("!")] [InlineData("#")] [InlineData("-not-start-with-dash")] + [InlineData("test\n")] // Regex $ matches before a trailing newline; the hg URL would then drop it + [InlineData("test\r\n")] public void InvalidCodesThrows(string code) { Assert.Throws(() => new ProjectCode(code)); From de7235d374e4537a760e7606b11d7eddaefc2b0d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 17 Nov 2026 18:36:55 +0700 Subject: [PATCH 4/6] Link the liblcm issue from the pt-BR workaround Co-Authored-By: Claude Opus 5.5 --- backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index 1e2b22fc3a..6f248f279f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -101,7 +101,8 @@ public virtual LcmCache NewProject(FwDataProject project, // empty list), CreateNewLangProj names the empty list in the default analysis WS, which it looks up // with GetWsFromStr(IcuLocale). GetWsFromStr expects a language tag, so this only works when the // two match: "pt-BR" is "pt_BR" in ICU, the lookup returns 0, and creation throws with the project - // already on disk. Remove this workaround once liblcm passes the tag. Until then hand liblcm "en" as the default + // already on disk. Remove this workaround once https://github.com/sillsdev/liblcm/issues/416 is fixed and released. + // Until then hand liblcm "en" as the default // and pass every requested analysis WS as additional; SetRequestedWritingSystems fixes the default. var substituteDefaultAnalysis = !IcuLocaleMatchesTag(analysisDefinitions[0]); var defaultAnalysis = substituteDefaultAnalysis ? CreateWritingSystemDefinition("en") : analysisDefinitions[0]; From 32fa8893a142d7f8989469339482b69aa1902573 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 20 Nov 2026 12:32:25 +0100 Subject: [PATCH 5/6] Retry marking a created project ready when that fails Co-Authored-By: Claude Opus 5.5 --- .../LexBoxApi/Jobs/InitFwDataProjectJob.cs | 15 ++++- .../Jobs/InitFwDataProjectJobDbTests.cs | 59 +++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs diff --git a/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs index 38c90de1de..4b4d302b93 100644 --- a/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs +++ b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs @@ -85,9 +85,18 @@ protected override async Task ExecuteJob(IJobExecutionContext context) switch (outcome.Result) { case InitAttemptResult.Created: - await projectService.FinishFwDataInit(projectId); - logger.LogInformation("Initialised project {ProjectId} ({Code}) from the template", projectId, project.Code); - return; + try + { + await projectService.FinishFwDataInit(projectId); + logger.LogInformation("Initialised project {ProjectId} ({Code}) from the template", projectId, project.Code); + return; + } + catch (Exception ex) + { + // The repo is populated, so a later attempt sees Created again and retries marking it ready + logger.LogError(ex, "Failed to mark project {ProjectId} ({Code}) as initialised", projectId, project.Code); + } + break; case InitAttemptResult.Failed: logger.LogError("Creating project {ProjectId} ({Code}) from the template failed, rolling it back: {Error}", diff --git a/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs new file mode 100644 index 0000000000..7e7ea419eb --- /dev/null +++ b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs @@ -0,0 +1,59 @@ +using LexBoxApi.Jobs; +using LexBoxApi.Services; +using LexBoxApi.Services.Email; +using LexCore.Config; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexData; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using Quartz; +using Testing.Fixtures; + +namespace Testing.LexBoxApi.Jobs; + +[Collection(nameof(TestingServicesFixture))] +[Trait("Category", "RequiresDb")] +public class InitFwDataProjectJobDbTests(TestingServicesFixture testing) +{ + [Fact] + public async Task A_failure_marking_the_project_ready_is_retried() + { + var serviceProvider = testing.ConfigureServices(); + var dbContext = serviceProvider.GetRequiredService(); + var hgService = new Mock(); + // The push landed, so the attempt counts as created without asking FwHeadless + hgService.Setup(h => h.GetTipHash(It.IsAny(), It.IsAny())) + .ReturnsAsync("1234567890abcdef1234567890abcdef12345678"); + hgService.Setup(h => h.GetLastCommitTimeFromHg(It.IsAny())) + .ThrowsAsync(new HttpRequestException("hgweb blip")); + var projectService = new ProjectService(dbContext, hgService.Object, + serviceProvider.GetRequiredService>(), + Mock.Of(), Mock.Of(), null!); + var projectId = await projectService.CreateProject( + new(null, "Init finalize retry", "", "init-finalize-retry", ProjectType.FLEx, RetentionPolicy.Test, false, null, null), + fwDataInitStatus: FwDataInitStatus.InProgress); + var scheduler = new Mock(); + var schedulerFactory = Mock.Of(f => f.GetScheduler(It.IsAny()) == Task.FromResult(scheduler.Object)); + IJob job = new InitFwDataProjectJob(dbContext, projectService, + new FwHeadlessClient(new HttpClient(), NullLogger.Instance), + hgService.Object, schedulerFactory, Mock.Of(), NullLogger.Instance) + { + ProjectId = projectId.ToString(), + Input = System.Text.Json.JsonSerializer.Serialize(new InitFwDataProjectInput(["fr"], ["en"], "en")), + Attempt = "1", + }; + + await job.Execute(Mock.Of()); + + scheduler.Verify(s => s.ScheduleJob( + It.Is(t => t.JobDataMap.GetString(nameof(InitFwDataProjectJob.Attempt)) == "2"), + It.IsAny()), Times.Once); + var project = await dbContext.Projects.AsNoTracking().SingleAsync(p => p.Id == projectId); + project.FwDataInitStatus.Should().Be(FwDataInitStatus.InProgress); + } +} From 071e0da19ba122a1051aec1c911d7653a81cdd1b Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 20 Nov 2026 13:45:42 +0100 Subject: [PATCH 6/6] Check the finish step ran in the job retry test; group the job's tests Co-Authored-By: Claude Opus 5.5 --- backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs | 1 + .../Testing/LexBoxApi/{ => Jobs}/InitFwDataProjectJobTests.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) rename backend/Testing/LexBoxApi/{ => Jobs}/InitFwDataProjectJobTests.cs (99%) diff --git a/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs index 7e7ea419eb..e34e61b1f1 100644 --- a/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs +++ b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs @@ -50,6 +50,7 @@ public async Task A_failure_marking_the_project_ready_is_retried() await job.Execute(Mock.Of()); + hgService.Verify(h => h.GetLastCommitTimeFromHg(It.IsAny()), Times.Once); scheduler.Verify(s => s.ScheduleJob( It.Is(t => t.JobDataMap.GetString(nameof(InitFwDataProjectJob.Attempt)) == "2"), It.IsAny()), Times.Once); diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobTests.cs similarity index 99% rename from backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs rename to backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobTests.cs index 0029e420fb..ab5005868c 100644 --- a/backend/Testing/LexBoxApi/InitFwDataProjectJobTests.cs +++ b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobTests.cs @@ -8,7 +8,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Moq; -namespace Testing.LexBoxApi; +namespace Testing.LexBoxApi.Jobs; /// /// Unit tests for how one run of InitFwDataProjectJob decides what happened to a creation. Every run must be