diff --git a/backend/FwHeadless/FwHeadless.csproj b/backend/FwHeadless/FwHeadless.csproj index 4a6c2721e6..c54ddafba3 100644 --- a/backend/FwHeadless/FwHeadless.csproj +++ b/backend/FwHeadless/FwHeadless.csproj @@ -15,6 +15,9 @@ + + @@ -40,6 +43,11 @@ + + diff --git a/backend/FwHeadless/FwHeadlessConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs index a7edd76b72..84fe6a1863 100644 --- a/backend/FwHeadless/FwHeadlessConfig.cs +++ b/backend/FwHeadless/FwHeadlessConfig.cs @@ -23,6 +23,22 @@ public class FwHeadlessConfig public long MaxUploadFileSizeBytes => MaxUploadFileSizeKb * 1024; public string FdoDataModelVersion { get; init; } = "7000072"; + /// + /// The FLExBridge data version. FieldWorks/LfMergeBridge form the Mercurial branch a project's + /// data lives on by joining this and the FDO model version with a dot (see ). + /// Mirrors FlexBridgeConstants.FlexBridgeDataVersion in FLExBridge, which is internal so we can't + /// reference it directly; bump this if FLExBridge bumps its data version. + /// + public string FlexBridgeDataVersion { get; init; } = "7500002"; + + /// + /// The Mercurial branch FieldWorks/LfMergeBridge keep a project's data on: FlexBridgeDataVersion, + /// a dot, then the FDO model version (e.g. "7500002.7000072"). This is exactly the branch name + /// LfMergeBridge derives from , so a new project's genesis commit + /// must land on this branch for send/receive to find the data. + /// + public string SendReceiveBranchName => $"{FlexBridgeDataVersion}.{FdoDataModelVersion}"; + /// /// Project directory structure in FwHeadless: (Note that FwDataProject.ProjectsPath is the root of a SINGLE project) /// {ProjectStorageRoot}/ diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index e96157c7da..2604d58671 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -29,6 +29,7 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); // Singleton so its file lock is shared across requests and the sync worker @@ -37,6 +38,14 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) .AddLcmCrdtClientCore() .AddFwDataBridge(ServiceLifetime.Scoped) .AddFwLiteProjectSync(); + // The LCM project templates (NewLangProj.fwdata etc.) ship in our output (see FwHeadless.csproj). + // FwDataBridgeConfig's default TemplatesFolder points at the FieldWorks user-data dir, which + // doesn't exist in the container; point it at the shipped templates unless overridden in config. + services.Configure(config => + { + if (config.TemplatesFolder == new FwDataBridgeConfig().TemplatesFolder) + config.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + }); services.RemoveAll(typeof(IMediaAdapter)); services.AddScoped(); services.AddScoped(); diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index b5aa38ef19..643d731b80 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -93,6 +93,7 @@ app.MapMediaFileRoutes(); app.MapMergeRoutes(); app.MapLexiconRoutes(); +app.MapProjectRoutes(); // DELETE endpoint to delete the FieldWorks repo/project (and nothing else) app.MapDelete("/api/manage/repo/{projectId}", async (Guid projectId, ProjectDeletionService deletionService) => diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs index a54199a122..010a3ceec3 100644 --- a/backend/FwHeadless/Routes/MergeRoutes.cs +++ b/backend/FwHeadless/Routes/MergeRoutes.cs @@ -74,6 +74,10 @@ static async Task> ExecuteMergeRequest( switch (syncHostedService.QueueJob(projectId)) { + case QueueJobResult.ProjectBeingCreated: + activity?.SetStatus(ActivityStatusCode.Ok, "Project is still being created"); + return TypedResults.Problem("Project is still being created, try again once creation has finished", + statusCode: StatusCodes.Status409Conflict); case QueueJobResult.SyncAlreadyRunning: activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running"); return TypedResults.Problem("A sync is already running for this project, try again once it has finished", diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs new file mode 100644 index 0000000000..c54f1e67e9 --- /dev/null +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -0,0 +1,137 @@ +using System.Diagnostics; +using FwHeadless.Services; +using LexCore.Entities; +using LexCore.Sync; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.Extensions.Options; + +namespace FwHeadless.Routes; + +public static class ProjectRoutes +{ + public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app) + { + var group = app.MapGroup("/api/project"); + group.MapPost("/initFwDataProject", InitFwDataProject); + group.MapGet("/creation-status", GetCreationStatus); + group.MapGet("/await-creation-finished", AwaitCreationFinished); + return group; + } + + // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the + // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the + // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates. + private static async Task> InitFwDataProject( + Guid projectId, + InitFwDataProjectInput input, + IProjectLookupService projectLookupService, + ProjectCreationService projectCreationService, + ILogger logger) + { + // Null-safe against a malformed body (System.Text.Json leaves an absent list null). + var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", input.WsVernacular) + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", input.WsAnalysis) + ?? InitFwDataProjectInput.ValidateUiWritingSystem(input.WsUi); + if (wsError is not null) + return TypedResults.Problem(wsError, statusCode: StatusCodes.Status400BadRequest); + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) + { + logger.LogError("initFwDataProject request for non-existent project {ProjectId}", projectId); + return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound); + } + + await projectCreationService.InitFwDataProject( + projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi); + return TypedResults.Ok(); + } + + // Instantaneous creation status, for a caller that wants to poll. Mirrors /api/merge/status. + private static async Task, NotFound>> GetCreationStatus( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.project_id", projectId); + + var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config); + if (status is null) + { + activity?.SetStatus(ActivityStatusCode.Error, "Project not found"); + return TypedResults.NotFound(); + } + activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString()); + return TypedResults.Ok(status); + } + + // Long-poll until an in-flight creation finishes. Mirrors /api/merge/await-finished: giving up on the + // wait is reported as a status, not an error, because the creation itself keeps running either way. + private static async Task, NotFound>> AwaitCreationFinished( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config, + CancellationToken cancellationToken) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.project_id", projectId); + + try + { + var result = await syncHostedService.AwaitCreationFinished(projectId, cancellationToken); + if (result is not null) + { + activity?.SetStatus(ActivityStatusCode.Ok, result.Status.ToString()); + return TypedResults.Ok(result); + } + } + catch (OperationCanceledException) + { + // The caller's wait was cancelled; the creation was not (it takes no cancellation token). + // Report that distinctly so a caller can't read a timeout as a failed creation. + activity?.SetStatus(ActivityStatusCode.Unset, "Timed out awaiting creation"); + return TypedResults.Ok(ProjectCreationStatus.TimedOutAwaitingCreation); + } + + // Nothing in flight and nothing remembered, so fall back to what the project itself tells us. + var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config); + if (status is null) + { + activity?.SetStatus(ActivityStatusCode.Error, "Project not found"); + return TypedResults.NotFound(); + } + activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString()); + return TypedResults.Ok(status); + } + + /// + /// Creation status for a project, or null when LexBox has no such project. In-memory state wins + /// when we have it; otherwise the status is derived from durable state, so it survives a FwHeadless + /// restart and a result that has aged out of the cache. + /// + private static async Task DetermineCreationStatus( + Guid projectId, + SyncHostedService syncHostedService, + IProjectLookupService projectLookupService, + IOptions config) + { + if (syncHostedService.IsProjectBeingCreated(projectId)) return ProjectCreationStatus.Creating; + var recent = syncHostedService.TryGetRecentCreationResult(projectId); + if (recent is not null) return recent; + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) return null; + + // A successful creation leaves the .fwdata on disk; a failed one deletes the project folder + // (ProjectCreationService.CleanupLocalProject). Note this says "project data is here", not "this + // process created it" -- a project cloned by an ordinary sync looks the same, which is the right + // answer for a caller asking whether the project is ready to use. + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + return File.Exists(fwDataProject.FilePath) + ? ProjectCreationStatus.Created + : ProjectCreationStatus.NotCreated; + } +} diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs index 4755ff1ac8..b1835ec7f5 100644 --- a/backend/FwHeadless/Services/ISendReceiveService.cs +++ b/backend/FwHeadless/Services/ISendReceiveService.cs @@ -10,4 +10,6 @@ public interface ISendReceiveService Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode); Task PendingCommitCountBothWays(FwDataProject project, string? projectCode); Task CommitFile(string filePath, string commitMessage); + Task InitRepo(string folder); + Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = ""); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs new file mode 100644 index 0000000000..2f455f8b57 --- /dev/null +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -0,0 +1,216 @@ +using System.Xml; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.LcmUtils; +using LexCore.Exceptions; +using LexCore.Sync; +using Microsoft.Extensions.Options; + +namespace FwHeadless.Services; + +/// +/// Creates a brand-new FieldWorks project from the SIL.LCModel template and pushes it into the +/// (empty) Mercurial repo that LexBox has already initialised for the project. +/// +public class ProjectCreationService( + IOptions config, + ISendReceiveService srService, + IProjectLoader projectLoader, + SyncHostedService syncHostedService, + ILogger logger) +{ + private const string InitialCommitMessage = "Initial project creation"; + + // The one file FieldWorks/FLExBridge put in a brand-new FLEx repo's first commit: a minimal custom + // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge). + private const string CustomPropertiesFilename = "FLExProject.CustomProperties"; + + public async Task InitFwDataProject( + Guid projectId, + string projectCode, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems, + string uiWritingSystem) + { + if (vernacularWritingSystems.Count == 0) + throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); + if (analysisWritingSystems.Count == 0) + throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); + if (string.IsNullOrEmpty(uiWritingSystem)) + throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem)); + + // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. + if (!syncHostedService.TryStartProjectCreation(projectId)) + throw new ProjectSyncInProgressException(projectId); + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + // Published to status pollers in the finally below. Pre-set to a failure so an exit path that + // somehow reaches neither the success line nor the catch still releases pollers with an answer. + var outcome = ProjectCreationStatus.Failed("Project creation ended without reporting a result"); + try + { + // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/ + // FLExBridge create a repo, since Clone can't establish an empty remote and + // Language_Forge_Send_Receive refuses to make the *first* commit itself: + // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). + // 2. Build fw.fwdata into that folder from the SIL.LCModel NewLangProj template, + // configured with all of the requested writing systems, but do not commit it yet. + // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch to be + // revision 0 -- Chorus expects revision 0 to be on the default branch, because it uses + // that rev 0 commit to track a repo's history and catch when someone would try to push + // an unrelated repo (which would otherwise create two Mercurial heads). This is the + // same initial commit, with the same file, that FieldWorks creates. The fwdata itself + // is NOT committed; Send/Receive splits it into the nested files Mercurial actually + // tracks (fwdata is excluded from tracking). + // 4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, + // e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on + // that branch, so the split data must land there while the genesis stays on 'default'. + // 5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push. + await srService.InitRepo(fwDataProject.ProjectFolder); + + BuildFwDataFile(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); + FixLinkedFilesRootDirSeparator(fwDataProject); + AssertModelVersionMatches(fwDataProject); + + var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject); + await srService.CommitFile(customPropertiesFile, InitialCommitMessage); + + await srService.SwitchBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); + var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage); + if (!pushResult.Success) + throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); + + logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId); + outcome = ProjectCreationStatus.Created; + } + catch (Exception ex) + { + outcome = ProjectCreationStatus.Failed(ex.Message); + CleanupLocalProject(fwDataProject); + throw; + } + finally + { + syncHostedService.EndProjectCreation(projectId, outcome); + } + } + + private void BuildFwDataFile( + FwDataProject fwDataProject, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems, + string uiWs) + { + // NewProject copies the SIL.LCModel template, configured with all of the requested writing + // systems (the first of each list is the default of that type), and returns a loaded cache; + // dispose it right away so its file locks are released before the following hg add/commit. + using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs); + } + + /// + /// FieldWorks (Windows) compares LangProject.LinkedFilesRootDir with an exact string, expecting the + /// Windows-style default %proj%\LinkedFiles. liblcm builds that path with Path.Combine, which on + /// this Linux host yields %proj%/LinkedFiles (forward slash), so FieldWorks warns on its first + /// Send/Receive. Rewrite just that one value to the backslash form. + /// + /// We deliberately do NOT parse the (potentially multi-MB) fwdata as XML -- that would be a needless + /// cost. The value always sits on the line immediately after the <LinkedFilesRootDir> open tag, + /// so stream the file once, fix the <Uni> on the line following that tag, and copy every other + /// line through untouched. + /// + private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) + { + const string markerLinePrefix = "%proj%\LinkedFiles"; + + var sourcePath = fwDataProject.FilePath; + var tempPath = sourcePath + ".tmp"; + var madeChanges = false; + using (var reader = new StreamReader(sourcePath)) + using (var writer = new StreamWriter(tempPath)) + { + var nextLineIsValue = false; + var done = false; + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (nextLineIsValue) + { + var changedLine = line.Replace(forwardSlashValue, backslashValue); + if (string.Equals(changedLine, line, StringComparison.InvariantCulture)) + { + // Exit early and don't overwrite since we didn't need this change + break; + } + madeChanges = true; + line = changedLine; + nextLineIsValue = false; + done = true; + } + else if (!done && line.StartsWith(markerLinePrefix, StringComparison.Ordinal)) + { + nextLineIsValue = true; + } + writer.WriteLine(line); + } + } + if (madeChanges) + File.Move(tempPath, sourcePath, overwrite: true); + else + File.Delete(tempPath); + } + + private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) + { + var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename); + File.WriteAllText(path, """ + + + """); + return path; + } + + /// + /// The freshly-created project is stamped with the SIL.LCModel package's data-model version. + /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data + /// (see FwHeadless AGENTS.md), so fail before the push rather than after. + /// + private void AssertModelVersionMatches(FwDataProject fwDataProject) + { + var expected = config.Value.FdoDataModelVersion; + var actual = ReadModelVersion(fwDataProject.FilePath); + // Fail closed: an unreadable version ('actual' == null) is treated as a mismatch, since this is + // the last safety net before an irreversible push in a data-loss-risk area. + if (actual != expected) + { + throw new InvalidOperationException( + $"New project's data-model version '{actual ?? "(unreadable)"}' does not match FwHeadless " + + $"FdoDataModelVersion '{expected}'; refusing to push to avoid data corruption."); + } + } + + private static string? ReadModelVersion(string fwDataFilePath) + { + using var reader = XmlReader.Create(fwDataFilePath, new XmlReaderSettings { IgnoreComments = true, IgnoreWhitespace = true }); + // The .fwdata root element carries the FDO model version, e.g. . + return reader.MoveToContent() == XmlNodeType.Element ? reader.GetAttribute("version") : null; + } + + private void CleanupLocalProject(FwDataProject fwDataProject) + { + // Mirror SyncWorker.SetupFwData's failure cleanup so a retry starts clean. The remote repo + // (created by LexBox) is torn down by the caller on the LexBox side. + try + { + if (Directory.Exists(fwDataProject.ProjectFolder)) + Directory.Delete(fwDataProject.ProjectFolder, true); + if (Directory.Exists(fwDataProject.ProjectsPath) && + !Directory.EnumerateFileSystemEntries(fwDataProject.ProjectsPath).Any()) + Directory.Delete(fwDataProject.ProjectsPath); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to clean up local project folder after a failed creation: {ProjectFolder}", + fwDataProject.ProjectFolder); + } + } +} diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 509644a88a..a5066a58fd 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -166,6 +166,46 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)} {EscapeShellArg(filePath)}", fileDir, progress, allowedExitCodes: [0, 1]); } + public static async Task InitRepo(string folder, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + progress ??= new NullProgress(); + Directory.CreateDirectory(folder); + // Use Chorus rather than a bare `hg init`: CreateOrUseExisting also wires up the custom hg + // extensions FwHeadless relies on (e.g. fixutf8) so later send/receive works correctly. + await Task.Run(() => HgRepository.CreateOrUseExisting(folder, progress)); + } + + public static async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "", IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + activity?.SetTag("app.branch", branchName); + progress ??= new NullProgress(); + // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version + // (e.g. 7500002.7000072); the initial commit of a repo needs to be on the 'default' branch + // so that Chorus can use rev 0 to compare project origin, but then we need to switch branches + // to the numbered branch. The `hg branch` command records the branch that will be used for + // the next commit, but we need to add an empty commit in order to actually create the branch, + // otherwise LfMergeBridge will refuse to create a new branch (since in the Language Forge code + // that LfMergeBridge targets, creating a new branch would be an error). This results in a new + // project being created by this process having three initial commits, rather than the two + // initial commits that it would have if created by FieldWorks Classic, but that's an acceptable + // price to pay for such a simple workaround. If LfMergeBridge is updated to be okay with creating + // new branches on its own, then this method can be passed "createBranch: false" to skip running + // the `hg commit` step that creates that empty commit. + // + // Commit message defaults to "Switching to branch {branchName}", pass `commitMessage` param to change + await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress); + if (createBranch) + { + // Could run "hg log -r . --template {desc}" to reuse previous commit message if we wanted to + if (string.IsNullOrEmpty(commitMessage)) commitMessage = $"Switching to branch {branchName}"; + await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress); + } + } + private static string EscapeShellArg(string arg) { var quote = """ diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs index 735e90b8ae..0c410ba1f9 100644 --- a/backend/FwHeadless/Services/SendReceiveService.cs +++ b/backend/FwHeadless/Services/SendReceiveService.cs @@ -70,4 +70,14 @@ public async Task CommitFile(string filePath, string commitMessage) { await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress); } + + public async Task InitRepo(string folder) + { + await SendReceiveHelpers.InitRepo(folder, progress); + } + + public async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "") + { + await SendReceiveHelpers.SwitchBranch(folder, branchName, createBranch, commitMessage, progress); + } } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 33bc51334a..e7663c109c 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -20,6 +20,8 @@ public enum QueueJobResult { Queued, AlreadyQueued, + // A sync must not race a project that's still being created; the caller should retry later + ProjectBeingCreated, // The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes SyncAlreadyRunning, QueueFull, @@ -29,22 +31,41 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsToSync = Channel.CreateUnbounded(); - // Projects with a sync queued or running. A single atomic GetOrAdd decides whether a request queues a new sync, - // so no lock is needed (this requires that SyncHostedService be a singleton). - private readonly ConcurrentDictionary _syncJobs = new(); + // Projects with a sync queued or running, or currently being created and populated with an initial .fwdata file. + // Creations are tracked here (not just in ProjectCreationService) so a sync job can't be queued for a project + // mid-creation and race on the same fw/ folder and repo. Keeping both in one dictionary means a single atomic + // TryAdd/GetOrAdd decides who gets the project, so no lock is needed (this requires that SyncHostedService be a singleton). + private readonly ConcurrentDictionary _projectJobs = new(); /// - /// The TaskCompletionSource carries the outcome to anyone long-polling AwaitSyncFinished. - /// distinguishes a queued sync (which will pick up any changes made before it starts) + /// Exactly one of or is set. The TaskCompletionSource carries the + /// outcome to anyone long-polling AwaitSyncFinished or AwaitCreationFinished. + /// distinguishes a queued sync (which will pick up any changes made before it starts) /// from a running one (which might not). /// - private record SyncJob(TaskCompletionSource Result, bool Started = false) + private record ProjectJob( + TaskCompletionSource? Sync, + TaskCompletionSource? Creation, + bool SyncStarted = false) { // RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on - // the thread that finished the sync. - public static SyncJob New() => new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + // the thread that finished the sync or creation. + public static ProjectJob ForSync() => + new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), null); + + public static ProjectJob ForCreation() => + new(null, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); } + private TaskCompletionSource? GetSyncJob(Guid projectId) => + _projectJobs.TryGetValue(projectId, out var job) ? job.Sync : null; + + private TaskCompletionSource? GetCreationJob(Guid projectId) => + _projectJobs.TryGetValue(projectId, out var job) ? job.Creation : null; + + // Only removes the entry if it's still the given job, so ending one kind of job can never clear the other kind + private bool TryRemoveJob(Guid projectId, ProjectJob job) => _projectJobs.TryRemove(KeyValuePair.Create(projectId, job)); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) @@ -73,38 +94,93 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Give clients a bit more time to poll the status CacheRecentSyncResult(projectId, result); - if (_syncJobs.TryRemove(projectId, out var job)) - job.Result.TrySetResult(result); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is { } tcs && TryRemoveJob(projectId, job)) + tcs.TrySetResult(result); } } - // QueueJob never modifies an existing entry, and only ExecuteAsync removes one, so nothing can race this update + // QueueJob never modifies an existing sync entry, and only ExecuteAsync removes one, so nothing can race this update internal void MarkSyncStarted(Guid projectId) { - if (_syncJobs.TryGetValue(projectId, out var job)) - _syncJobs.TryUpdate(projectId, job with { Started = true }, job); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is not null) + _projectJobs.TryUpdate(projectId, job with { SyncStarted = true }, job); } public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _syncJobs.ContainsKey(projectId); + return _projectJobs.ContainsKey(projectId); + } + + /// + /// Reserves a project for creation-and-population, blocking a concurrent creation or sync of the + /// same project. Returns false if a sync is already queued/running or another creation is in flight. + /// Pair with in a finally block. + /// + public bool TryStartProjectCreation(Guid projectId) + { + return _projectJobs.TryAdd(projectId, ProjectJob.ForCreation()); + } + + /// + /// Releases the reservation and publishes the outcome to pollers. Must be called on every exit path + /// from a creation, or an AwaitCreationFinished caller waits until its own token trips. + /// + public void EndProjectCreation(Guid projectId, ProjectCreationStatus result) + { + // Cache before removing the reservation, so a poller that sees "not in flight" always finds the + // result waiting for it rather than a gap. + CacheRecentCreationResult(projectId, result); + if (_projectJobs.TryGetValue(projectId, out var job) && job.Creation is { } tcs && TryRemoveJob(projectId, job)) + tcs.TrySetResult(result); } + public bool IsProjectBeingCreated(Guid projectId) => GetCreationJob(projectId) is not null; + public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) { - if (_syncJobs.TryGetValue(projectId, out var job)) - return await job.Result.Task.WaitAsync(cancellationToken); + if (GetSyncJob(projectId) is { } tcs) + return await tcs.Task.WaitAsync(cancellationToken); return TryGetRecentSyncResult(projectId); } + /// + /// Waits for an in-flight creation to finish. Returns the recent result if one just finished, or + /// null when this FwHeadless has no record of a creation for the project (the caller should then + /// derive the status from durable state instead). + /// + public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken) + { + if (GetCreationJob(projectId) is { } tcs) + return await tcs.Task.WaitAsync(cancellationToken); + return TryGetRecentCreationResult(projectId); + } + + public ProjectCreationStatus? TryGetRecentCreationResult(Guid projectId) + { + return memoryCache.Get($"CreationResult|{projectId}"); + } + + private void CacheRecentCreationResult(Guid projectId, ProjectCreationStatus result) + { + // Longer than the sync equivalent's 30s: a caller whose own request timed out partway through a + // creation should still be able to come back and learn how it ended. + memoryCache.Set($"CreationResult|{projectId}", result, TimeSpan.FromMinutes(5)); + } + public QueueJobResult QueueJob(Guid projectId) { - var newJob = SyncJob.New(); - var job = _syncJobs.GetOrAdd(projectId, newJob); + var newJob = ProjectJob.ForSync(); + var job = _projectJobs.GetOrAdd(projectId, newJob); + //don't sync a project while it's still being created and populated with initial .fwdata (would race on the repo) + if (job.Creation is not null) + { + logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); + return QueueJobResult.ProjectBeingCreated; + } //will only queue job if it's not already queued if (!ReferenceEquals(job, newJob)) { - if (job.Started) + if (job.SyncStarted) { logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId); return QueueJobResult.SyncAlreadyRunning; @@ -116,7 +192,7 @@ public QueueJobResult QueueJob(Guid projectId) if (!_projectsToSync.Writer.TryWrite(projectId)) { logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId); - _syncJobs.TryRemove(KeyValuePair.Create(projectId, newJob)); + TryRemoveJob(projectId, newJob); return QueueJobResult.QueueFull; } diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index db3b5b83f6..6f248f279f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using Microsoft.Extensions.Options; using SIL.LCModel; +using SIL.LCModel.Infrastructure; using SIL.LCModel.Core.WritingSystems; using SIL.LCModel.Utils; using SIL.WritingSystems; @@ -96,28 +97,99 @@ public virtual LcmCache NewProject(FwDataProject project, var progress = new LcmThreadedProgress(); var analysisDefinitions = analysisWss.Select(CreateWritingSystemDefinition).ToList(); var vernacularDefinitions = vernacularWss.Select(CreateWritingSystemDefinition).ToList(); + // A liblcm bug: when no anthropology list file is passed (FLEx passes one unless the user picks an + // empty list), CreateNewLangProj names the empty list in the default analysis WS, which it looks up + // with GetWsFromStr(IcuLocale). GetWsFromStr expects a language tag, so this only works when the + // two match: "pt-BR" is "pt_BR" in ICU, the lookup returns 0, and creation throws with the project + // already on disk. Remove this workaround once https://github.com/sillsdev/liblcm/issues/416 is fixed and released. + // Until then hand liblcm "en" as the default + // and pass every requested analysis WS as additional; SetRequestedWritingSystems fixes the default. + var substituteDefaultAnalysis = !IcuLocaleMatchesTag(analysisDefinitions[0]); + var defaultAnalysis = substituteDefaultAnalysis ? CreateWritingSystemDefinition("en") : analysisDefinitions[0]; NewProject(progress, project.Name, lcmDirectories, progress.SynchronizeInvoke, - analysisDefinitions[0], + defaultAnalysis, vernacularDefinitions[0], uiWs, - AdditionalWritingSystems(analysisDefinitions), - AdditionalWritingSystems(vernacularDefinitions)); - return LoadCache(project); + AdditionalWritingSystems(analysisDefinitions, defaultAnalysis), + AdditionalWritingSystems(vernacularDefinitions, vernacularDefinitions[0])); + var cache = LoadCache(project); + try + { + SetRequestedWritingSystems(cache, analysisWss, vernacularWss, substituteDefaultAnalysis); + } + catch + { + cache.Dispose(); + throw; + } + return cache; } private static CoreWritingSystemDefinition CreateWritingSystemDefinition(string ws) => new(ws) { Id = ws }; - // The default writing system (the first of the list) is passed to CreateNewLangProj on its own, so - // the "additional" set is the whole list as a hash set with that default removed. The new project - // ends up with them in list order (NewProjectWritingSystemsTests checks this). - private static HashSet AdditionalWritingSystems(List definitions) + private static bool IcuLocaleMatchesTag(CoreWritingSystemDefinition definition) => + string.Equals(definition.IcuLocale, definition.Id, StringComparison.OrdinalIgnoreCase); + + // Everything except the default, which CreateNewLangProj takes on its own + private static HashSet AdditionalWritingSystems( + List definitions, + CoreWritingSystemDefinition defaultDefinition) => + definitions.Where(d => !string.Equals(d.Id, defaultDefinition.Id, StringComparison.OrdinalIgnoreCase)).ToHashSet(); + + /// + /// Makes the current analysis and vernacular lists exactly the requested writing systems, in the + /// requested order. CreateNewLangProj doesn't on its own: it adds "en" as a current analysis WS + /// whenever another one is the default, appends the "additional" sets without checking for one that's + /// already there (so "en" listed later, or "fr" plus "fr-Latn", end up twice), and takes them as + /// unordered sets. Writing systems it added that weren't asked for stay in the full lists, unselected. + /// + private static void SetRequestedWritingSystems(LcmCache cache, + IReadOnlyList analysisWss, + IReadOnlyList vernacularWss, + bool substitutedDefaultAnalysis) + { + var container = cache.ServiceLocator.WritingSystems; + var wsManager = cache.ServiceLocator.WritingSystemManager; + // GetOrSet resolves a tag to the WS liblcm already created for it, also when liblcm stored it under + // an equivalent tag (e.g. "fr" for "fr-Latn") + List Resolve(IReadOnlyList tags) => tags.Select(tag => + { + wsManager.GetOrSet(tag, out var ws); + return ws; + }).Distinct().ToList(); + + NonUndoableUnitOfWorkHelper.Do(cache.ActionHandlerAccessor, () => + { + var analysis = Resolve(analysisWss); + var vernacular = Resolve(vernacularWss); + // liblcm doesn't keep the full lists in the same order as the current lists, so set both + SetList(container.AnalysisWritingSystems, [.. analysis, .. container.AnalysisWritingSystems.Except(analysis)]); + SetList(container.VernacularWritingSystems, [.. vernacular, .. container.VernacularWritingSystems.Except(vernacular)]); + SetList(container.CurrentAnalysisWritingSystems, analysis); + SetList(container.CurrentVernacularWritingSystems, vernacular); + if (substitutedDefaultAnalysis) MoveReversalIndexToDefaultAnalysis(cache, analysis[0]); + }); + cache.ActionHandlerAccessor.Commit(); + } + + private static void SetList(ICollection list, List wanted) + { + if (list.SequenceEqual(wanted)) return; + list.Clear(); + foreach (var ws in wanted) list.Add(ws); + } + + // CreateNewLangProj creates a reversal index for the default analysis WS, which was "en" only because + // the requested default couldn't be passed in. Give the requested default the index instead. + private static void MoveReversalIndexToDefaultAnalysis(LcmCache cache, CoreWritingSystemDefinition defaultAnalysis) { - var additional = definitions.ToHashSet(); - additional.Remove(definitions[0]); - return additional; + cache.ServiceLocator.GetInstance().FindOrCreateIndexForWs(defaultAnalysis.Handle); + if (cache.ServiceLocator.WritingSystems.CurrentAnalysisWritingSystems.Any(ws => ws.Id == "en")) return; + var englishIndex = cache.LangProject.LexDbOA.ReversalIndexesOC.FirstOrDefault(ri => ri.WritingSystem == "en"); + if (englishIndex is not null) cache.LangProject.LexDbOA.ReversalIndexesOC.Remove(englishIndex); } private static void NewProject(IThreadedProgress progress, diff --git a/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs index aaa4e80d39..4ee17a0b19 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs @@ -37,24 +37,47 @@ public async Task DisposeAsync() await _rootServiceProvider.DisposeAsync(); } - [Fact] - public async Task NewProjectAddsEveryRequestedWritingSystemToTheCorrectListInOrder() + [Theory] + // Distinct codes per type so a swapped analysis/vernacular argument would fail, and several additional + // ones per type because CreateNewLangProj takes those as an unordered set + [InlineData("en,de,pt,ru", "fr,es,it,nl", "en,de,pt,ru", "fr,es,it,nl")] + // liblcm adds "en" as a current analysis WS whenever another one is the default + [InlineData("de,pt", "fr", "de,pt", "fr")] + // ...and appends a later "en" a second time + [InlineData("de,en,pt", "fr", "de,en,pt", "fr")] + // liblcm can't take a default analysis WS whose ICU locale differs from its tag ("pt_BR") + [InlineData("pt-BR,en", "fr", "pt-BR,en", "fr")] + [InlineData("qaa-x-foo", "fr", "qaa-x-foo", "fr")] + // Tags liblcm treats as the same WS end up once + [InlineData("en", "fr,fr-Latn", "en", "fr")] + public async Task NewProjectHasExactlyTheRequestedWritingSystemsInOrder( + string analysis, string vernacular, string expectedAnalysis, string expectedVernacular) { var fwDataBridgeConfig = Services.GetRequiredService>().Value; // The SIL.LCModel template ships next to the test assembly; point liblcm at it (this project, // unlike FwDataMiniLcmBridge.Tests, doesn't otherwise configure TemplatesFolder). fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); var fwDataProject = new FwDataProject("multi-ws-source", fwDataBridgeConfig.ProjectsFolder); - // First of each list is the default; the rest go through CreateNewLangProj's additional-WS sets. - // Distinct codes per type so a swapped analysis/vernacular argument would fail this test, and - // several additional ones per type because CreateNewLangProj takes those as an unordered set. - string[] analysis = ["en", "de", "pt", "ru"]; - string[] vernacular = ["fr", "es", "it", "nl"]; - Services.GetRequiredService().NewProject(fwDataProject, analysis, vernacular, uiWs: "en").Dispose(); + Services.GetRequiredService() + .NewProject(fwDataProject, analysis.Split(','), vernacular.Split(','), uiWs: "en").Dispose(); + // Reopened from disk, so this also checks the fixed-up lists were saved using var api = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); var writingSystems = await api.GetWritingSystems(); - writingSystems.Analysis.Select(ws => ws.WsId.Code).Should().Equal(analysis); - writingSystems.Vernacular.Select(ws => ws.WsId.Code).Should().Equal(vernacular); + writingSystems.Analysis.Select(ws => ws.WsId.Code).Should().Equal(expectedAnalysis.Split(',')); + writingSystems.Vernacular.Select(ws => ws.WsId.Code).Should().Equal(expectedVernacular.Split(',')); + } + + [Fact] + public void ADefaultAnalysisWritingSystemLiblcmCantTakeStillGetsTheReversalIndex() + { + var fwDataBridgeConfig = Services.GetRequiredService>().Value; + fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + var fwDataProject = new FwDataProject("reversal-source", fwDataBridgeConfig.ProjectsFolder); + + // "en" is passed to liblcm as the default instead, so it would otherwise get the reversal index + using var cache = Services.GetRequiredService().NewProject(fwDataProject, ["pt-BR"], ["fr"], uiWs: "en"); + + cache.LangProject.LexDbOA.ReversalIndexesOC.Select(ri => ri.WritingSystem).Should().Equal("pt-BR"); } } diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index b05e8b058c..09ff8e1d43 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,13 +1,16 @@ using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; using LexBoxApi.Jobs; +using LexBoxApi.Models.Project; using LexBoxApi.Services; using LexCore.Entities; using LexCore.Exceptions; using LexCore.ServiceInterfaces; +using LexCore.Sync; using LexData; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; using Quartz; using Path = System.IO.Path; @@ -20,9 +23,157 @@ public class ProjectController( IHgService hgService, LexBoxDbContext lexBoxDbContext, IPermissionService permissionService, - ISchedulerFactory scheduler) + ISchedulerFactory scheduler, + ILogger logger) : ControllerBase { + /// + /// Admin-only: create a new FLEx project whose repo is populated with a template .fwdata + /// (from the SIL.LCModel package) configured for the requested writing systems. + /// + /// At some future date this API may be opened up to trusted users, but the projectOrigin + /// parameter will remain admin-only. + /// + /// Building and pushing the project takes too long to hold a request open for, so this creates the + /// project row and empty repo, queues the rest as a background job, and returns 202 Accepted with the + /// new project's ID. Poll to learn when it's ready. + /// + /// Project code for the new project. + /// Display name for the new project. + /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. + /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. + /// Optional user interface writing system id; defaults to "en" if not given. + /// Optional to record as the project's origin; admin-only. + [HttpPost("initFwDataProject")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status202Accepted)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status409Conflict)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public async Task> InitFwDataProject( + string code, + string name, + [FromQuery] string[] wsVernacular, + [FromQuery] string[]? wsAnalysis = null, + string wsUi = "en", + string? projectOrigin = null) + { + // Validate before anything is created, so a bad request never leaves a project to tear down. + var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); + var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", wsVernacular) + ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", wsAnalysisOrDefault) + ?? InitFwDataProjectInput.ValidateUiWritingSystem(wsUi); + if (wsError is not null) + return Problem(wsError, statusCode: StatusCodes.Status400BadRequest); + if (code is null || code.Length < 4 || !ProjectCode.IsValidCode(code)) + return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); + if (string.IsNullOrWhiteSpace(name)) + return Problem("A project name is required", statusCode: StatusCodes.Status400BadRequest); + + ProjectMigrationStatus? origin = null; + if (!string.IsNullOrEmpty(projectOrigin)) + { + // Only admins may say where a project came from. Currently redundant with [AdminRequired], + // but this endpoint is expected to open up to non-admins later; this check must stay. + permissionService.AssertIsAdmin(); + if (!TryParseProjectOrigin(projectOrigin, out var parsedOrigin)) + return Problem($"Invalid project origin '{projectOrigin}'", statusCode: StatusCodes.Status400BadRequest); + origin = parsedOrigin; + } + if (await projectService.ProjectExists(code)) + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + + Guid projectId; + try + { + projectId = await projectService.CreateProject(new CreateProjectInput( + Id: null, + Name: name, + Description: string.Empty, + Code: code, + Type: ProjectType.FLEx, + RetentionPolicy: RetentionPolicy.Verified, + IsConfidential: false, + ProjectManagerId: null, + OrgId: null), + projectOrigin: origin, + // In the same insert as the row itself, so the project is never visible without it + fwDataInitStatus: FwDataInitStatus.InProgress); + } + catch (DbUpdateException e) when (DbError.CreateErrorFrom(e).Code == DbErrorCode.DuplicateProjectCode) + { + // ProjectExists above is only a fast path: a concurrent request can take the code after that check, + // and the unique index on Project.Code is what actually catches it. Nothing was created by this request. + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + } + + // The project row (marked as being initialised) and its empty repo now exist. Populating the repo + // happens in the job, which also marks the project ready or rolls it back. + try + { + await InitFwDataProjectJob.Queue(scheduler, projectId, + new InitFwDataProjectInput(wsVernacular, wsAnalysisOrDefault, wsUi)); + } + catch (Exception ex) + { + // Nothing has been sent to FwHeadless yet, so tearing the project down is unambiguous + logger.LogError(ex, "Failed to queue initialising project {ProjectId} ({Code}), rolling it back", projectId, code); + await CleanupFailedCreation(projectId, code); + throw; + } + return AcceptedAtAction(nameof(FwDataProjectCreationStatus), new { projectId }, projectId); + } + + /// + /// Admin-only: how the creation started by is going. + /// Creating while the job is still running; Created once the project exists and isn't being + /// initialised (so also for any project that wasn't created that way). A failed creation is rolled + /// back, deleting the project, after which this reports CreationFailed with the reason for an hour, + /// as long as LexBox hasn't restarted, and 404 after that. + /// + [HttpGet("fwDataProjectCreationStatus/{projectId}")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task> FwDataProjectCreationStatus( + Guid projectId, + [FromServices] IMemoryCache memoryCache) + { + var initStatus = await lexBoxDbContext.Projects + .Where(p => p.Id == projectId) + .Select(p => (FwDataInitStatus?)p.FwDataInitStatus) + .SingleOrDefaultAsync(); + if (initStatus is not null) + return initStatus == FwDataInitStatus.InProgress ? ProjectCreationStatus.Creating : ProjectCreationStatus.Created; + var failure = InitFwDataProjectJob.TryGetRecentFailure(memoryCache, projectId); + return failure is null ? NotFound() : ProjectCreationStatus.Failed(failure); + } + + /// + /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists + /// ("Migrated,Migrating"), so the parsed value must round-trip to the name that was passed in. + /// + public static bool TryParseProjectOrigin(string projectOrigin, out ProjectMigrationStatus origin) => + Enum.TryParse(projectOrigin, ignoreCase: true, out origin) + && Enum.IsDefined(origin) + && string.Equals(origin.ToString(), projectOrigin, StringComparison.OrdinalIgnoreCase); + + /// Analysis writing systems to use, defaulting to English when none are supplied. + public static string[] AnalysisWritingSystemsOrDefault(string[]? wsAnalysis) => + wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; + + private async Task CleanupFailedCreation(Guid projectId, string code) + { + try + { + await projectService.CleanupFailedProjectCreation(projectId, code); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}); it may need manual cleanup", projectId, code); + } + } + [HttpPost("refreshProjectLastChanged")] public async Task RefreshProjectLastChanged(string projectCode) { diff --git a/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs new file mode 100644 index 0000000000..4b4d302b93 --- /dev/null +++ b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs @@ -0,0 +1,220 @@ +using System.Net; +using System.Text.Json; +using LexBoxApi.Services; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexCore.Sync; +using LexData; +using Microsoft.Extensions.Caching.Memory; +using Quartz; + +namespace LexBoxApi.Jobs; + +/// +/// Has FwHeadless populate the empty repo of a project created by ProjectController.InitFwDataProject, +/// then marks the project ready, or rolls it back if FwHeadless reports that the creation failed. +/// +/// +/// Quartz keeps this job in the database and reruns it if LexBox stops partway through (RequestRecovery), +/// so every run must be safe to repeat. Each run therefore starts by finding out how far an earlier run +/// got, using LexBox's own repo as the source of truth. When the outcome is unknown (FwHeadless is +/// unreachable, busy, or still running a creation), the job schedules another attempt rather than guess. +/// +public class InitFwDataProjectJob( + LexBoxDbContext dbContext, + ProjectService projectService, + FwHeadlessClient fwHeadlessClient, + IHgService hgService, + ISchedulerFactory schedulerFactory, + IMemoryCache memoryCache, + ILogger logger) : LexJob +{ + public const int MaxAttempts = 10; + public static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + public static JobKey Key { get; } = new("InitFwDataProjectJob", "ProjectCreation"); + + public static async Task Queue(ISchedulerFactory schedulerFactory, + Guid projectId, + InitFwDataProjectInput input, + int attempt = 1, + TimeSpan? delay = null, + CancellationToken cancellationToken = default) + { + await QueueJob(schedulerFactory, + Key, + new JobDataMap + { + { nameof(ProjectId), projectId.ToString() }, + // The job store only takes strings, so the writing systems travel as JSON + { nameof(Input), JsonSerializer.Serialize(input) }, + { nameof(Attempt), attempt.ToString() }, + }, + delay, + cancellationToken); + } + + public string? ProjectId { get; set; } + public string? Input { get; set; } + public string? Attempt { get; set; } + + protected override async Task ExecuteJob(IJobExecutionContext context) + { + ArgumentException.ThrowIfNullOrEmpty(ProjectId); + ArgumentException.ThrowIfNullOrEmpty(Input); + ArgumentException.ThrowIfNullOrEmpty(Attempt); + if (!Guid.TryParse(ProjectId, out var projectId)) throw new ArgumentException($"Invalid project ID {ProjectId}", nameof(ProjectId)); + if (!int.TryParse(Attempt, out var attempt)) throw new ArgumentException($"Invalid number {Attempt}", nameof(Attempt)); + var input = JsonSerializer.Deserialize(Input) + ?? throw new ArgumentException("Missing creation input", nameof(Input)); + + var project = await dbContext.Projects.FindAsync(projectId); + if (project is null) + { + logger.LogInformation("Project {ProjectId} no longer exists, nothing to initialise", projectId); + return; + } + if (project.FwDataInitStatus != FwDataInitStatus.InProgress) + { + // An earlier run finished the job and this is a duplicate (e.g. a recovered run) + logger.LogInformation("Project {ProjectId} ({Code}) has already been initialised", projectId, project.Code); + return; + } + + var outcome = await AttemptInit(fwHeadlessClient, hgService, logger, projectId, project.Code, input); + switch (outcome.Result) + { + case InitAttemptResult.Created: + try + { + await projectService.FinishFwDataInit(projectId); + logger.LogInformation("Initialised project {ProjectId} ({Code}) from the template", projectId, project.Code); + return; + } + catch (Exception ex) + { + // The repo is populated, so a later attempt sees Created again and retries marking it ready + logger.LogError(ex, "Failed to mark project {ProjectId} ({Code}) as initialised", projectId, project.Code); + } + break; + + case InitAttemptResult.Failed: + logger.LogError("Creating project {ProjectId} ({Code}) from the template failed, rolling it back: {Error}", + projectId, project.Code, outcome.Detail); + // Before the rollback, so a poller that finds the project gone always finds the reason too + RecordFailure(memoryCache, projectId, outcome.Detail); + try + { + await projectService.CleanupFailedProjectCreation(projectId, project.Code); + return; + } + catch (Exception ex) + { + // The project is still marked as being initialised, so a later attempt starts over: it + // either creates the project after all, or fails again and retries the rollback + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code})", projectId, project.Code); + } + break; + + case InitAttemptResult.RetryLater: + logger.LogWarning("Couldn't tell whether project {ProjectId} ({Code}) was created (attempt {Attempt} of {MaxAttempts}): {Reason}", + projectId, project.Code, attempt, MaxAttempts, outcome.Detail); + break; + } + + if (attempt >= MaxAttempts) + { + logger.LogError( + "Giving up on initialising project {ProjectId} ({Code}) after {Attempts} attempts; it is still marked as being initialised and needs manual cleanup", + projectId, project.Code, attempt); + return; + } + await Queue(schedulerFactory, projectId, input, attempt + 1, RetryDelay); + } + + /// + /// One attempt at getting the project populated, picking up wherever an earlier attempt got to. + /// Never throws: anything unexpected means the outcome is unknown, so it's reported as RetryLater. + /// + /// + /// None of these calls take a cancellation token, for the same reason ProjectController used not to + /// pass one: FwHeadless finishes a creation whatever happens to our side of the connection, so + /// abandoning the wait would only stop us learning the outcome. If LexBox shuts down mid-attempt, + /// Quartz recovers the job and the next run picks the creation up again. + /// + public static async Task AttemptInit(FwHeadlessClient fwHeadlessClient, + IHgService hgService, + ILogger logger, + Guid projectId, + string code, + InitFwDataProjectInput input) + { + try + { + // FwHeadless's push is the last step of a creation, so a non-empty repo means an earlier attempt + // succeeded, even if we never heard back. Trust this over FwHeadless's own status, which says + // Created as soon as the .fwdata exists on its disk, even if FwHeadless died before pushing it. + if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created; + + var status = await fwHeadlessClient.CreationStatus(projectId); + if (status?.Status == ProjectCreationStatusEnum.Creating) + { + // An earlier attempt is still running in FwHeadless (e.g. this job was interrupted and + // recovered); starting another would be refused, so wait for that one instead + status = await fwHeadlessClient.AwaitCreationFinished(projectId); + if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created; + return status?.Status == ProjectCreationStatusEnum.CreationFailed + ? InitAttemptOutcome.Failed(status.Error ?? "FwHeadless reported that the creation failed") + : InitAttemptOutcome.RetryLater($"Waiting for FwHeadless's creation ended with {status?.Status.ToString() ?? "no status"}"); + } + + // Clear anything an interrupted attempt left in FwHeadless's local project folder, which would + // otherwise break the new attempt. Throws ProjectSyncInProgressException if FwHeadless is busy + // with this project. + await fwHeadlessClient.DeleteProject(projectId); + var (statusCode, error) = await fwHeadlessClient.InitFwDataProject(projectId, input.WsVernacular, input.WsAnalysis, input.WsUi); + if (error is null) return InitAttemptOutcome.Created; + // FwHeadless refused to start because something else holds the project, so nothing was built + if (statusCode == HttpStatusCode.Conflict) return InitAttemptOutcome.RetryLater(error); + // FwHeadless answered with a failure, so the creation isn't running and FwHeadless has already + // cleaned up its own local state + return InitAttemptOutcome.Failed(error); + } + catch (Exception ex) + { + // Most likely FwHeadless is unreachable (e.g. restarting) or our request timed out. FwHeadless + // may still be creating the project, so only a later attempt can tell how this ended. + logger.LogWarning(ex, "Attempt to initialise project {ProjectId} ({Code}) ended without an answer", projectId, code); + return InitAttemptOutcome.RetryLater(ex.Message); + } + } + + private static async Task RepoHasCommits(IHgService hgService, string code) => + await hgService.GetTipHash(code) != HgService.AllZeroHash; + + // In memory only: once the rollback has deleted the project row, this is all that's left to tell a + // poller why. It doesn't survive a LexBox restart, after which the poller sees "not found". + private static string FailureCacheKey(Guid projectId) => $"InitFwDataProjectFailure|{projectId}"; + + private static void RecordFailure(IMemoryCache memoryCache, Guid projectId, string? error) => + memoryCache.Set(FailureCacheKey(projectId), error ?? "Creating the project from the template failed", TimeSpan.FromHours(1)); + + public static string? TryGetRecentFailure(IMemoryCache memoryCache, Guid projectId) => + memoryCache.Get(FailureCacheKey(projectId)); +} + +public enum InitAttemptResult +{ + Created, + // FwHeadless definitely isn't creating the project and didn't create it + Failed, + // The outcome is unknown, e.g. FwHeadless was unreachable or is still busy + RetryLater, +} + +public record InitAttemptOutcome(InitAttemptResult Result, string? Detail = null) +{ + public static InitAttemptOutcome Created { get; } = new(InitAttemptResult.Created); + public static InitAttemptOutcome Failed(string error) => new(InitAttemptResult.Failed, error); + public static InitAttemptOutcome RetryLater(string reason) => new(InitAttemptResult.RetryLater, reason); +} diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 209e2fe849..2db9c3b5c6 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -69,7 +69,13 @@ public static void AddLexBoxApi(this IServiceCollection services, config.Enabled = false; }); services.AddServiceDiscovery(); - services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless")) + services.AddHttpClient(client => + { + client.BaseAddress = new ("http://fwHeadless"); + // initFwDataProject runs inline (hg init + build the project from the template + push), + // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom. + client.Timeout = TimeSpan.FromMinutes(5); + }) .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config services.AddHttpContextAccessor(); services.AddMemoryCache(); diff --git a/backend/LexBoxApi/ScheduledTasksKernel.cs b/backend/LexBoxApi/ScheduledTasksKernel.cs index 5293d20502..5345071d68 100644 --- a/backend/LexBoxApi/ScheduledTasksKernel.cs +++ b/backend/LexBoxApi/ScheduledTasksKernel.cs @@ -37,6 +37,9 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur q.AddJob(DeleteTempDirectoryJob.Key, j => j.StoreDurably()); q.AddJob(UpdateProjectMetadataJob.Key, j => j.StoreDurably()); q.AddJob(RetryEmailJob.Key, j => j.StoreDurably()); + // RequestRecovery: rerun a creation that was interrupted by LexBox stopping, rather than leave the + // project marked as being initialised forever. The job is written to be safe to rerun. + q.AddJob(InitFwDataProjectJob.Key, j => j.StoreDurably().RequestRecovery()); q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key) .WithIdentity("WeeklyCleanupTrigger") //every sunday at 2am diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 4b2d8d7cde..857757fe44 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -1,5 +1,7 @@ using System.Net; +using System.Net.Http.Json; using System.Text.Json; +using LexCore.Entities; using LexCore.Exceptions; using LexCore.Sync; @@ -209,4 +211,94 @@ private static async Task FailedResponse(HttpResponseMessag } return body; } + + /// + /// Initializes a newly-created (empty) project's repo with a new .fwdata configured for the + /// given writing systems. Runs inline (a new empty project is small). Returns a null Error on + /// success; on failure returns the FwHeadless status code and error body so the caller can + /// propagate the right status (e.g. surface a 400 rather than flattening it to 500). + /// + public async Task<(HttpStatusCode StatusCode, string? Error)> InitFwDataProject(Guid projectId, + IReadOnlyList wsVernacular, + IReadOnlyList wsAnalysis, + string wsUi, + CancellationToken cancellationToken = default) + { + var input = new InitFwDataProjectInput(wsVernacular, wsAnalysis, wsUi); + var response = await httpClient.PostAsJsonAsync($"/api/project/initFwDataProject?projectId={projectId}", + input, + cancellationToken); + if (response.IsSuccessStatusCode) return (response.StatusCode, null); + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + logger.LogError("Failed to init fwdata project: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + responseBody); + // Never null on failure: AttemptInit reads a null Error as success, and a failure body can be empty + return (response.StatusCode, string.IsNullOrWhiteSpace(responseBody) + ? $"FwHeadless returned HTTP {(int)response.StatusCode}" + : responseBody); + } + + /// + /// Current creation status for a project, or null if FwHeadless doesn't know the project or the + /// call failed (both are logged). Cheap and safe to poll. + /// + /// + /// Unlike , this one does take a cancellation token, and callers may + /// safely pass HttpContext.RequestAborted. Abandoning a status read costs nothing: it starts no + /// durable work and tells the caller nothing it has to act on. Abandoning the creation call is what + /// is dangerous, because the creation keeps running on the FwHeadless side either way. + /// + public async Task CreationStatus(Guid projectId, CancellationToken cancellationToken = default) + { + var response = await httpClient.GetAsync($"/api/project/creation-status?projectId={projectId}", cancellationToken); + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(cancellationToken); + // 404 is an ordinary answer ("no such project"), not a fault worth an error log. + if (response.StatusCode != HttpStatusCode.NotFound) + { + logger.LogError("Failed to get project creation status: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync(cancellationToken)); + } + return null; + } + + /// + /// Waits for an in-flight creation to finish and returns how it ended, or null if FwHeadless doesn't + /// know the project or the call failed. Returns TimedOutAwaitingCreation if the wait is given up on + /// -- by either side -- which means "still running, ask again", never "creation failed". + /// + public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken = default) + { + HttpResponseMessage response; + try + { + response = await httpClient.GetAsync($"/api/project/await-creation-finished?projectId={projectId}", cancellationToken); + } + catch (OperationCanceledException) + { + // We gave up first, so FwHeadless's own TimedOutAwaitingCreation response is on a connection + // we already dropped. Report the same thing ourselves, so a caller sees one contract no + // matter which side stopped waiting. The creation itself is untouched and still running. + logger.LogInformation("Gave up awaiting creation of project {ProjectId}; it is still running", projectId); + return ProjectCreationStatus.TimedOutAwaitingCreation; + } + + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(cancellationToken); + if (response.StatusCode != HttpStatusCode.NotFound) + { + logger.LogError("Failed to await project creation: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + await response.Content.ReadAsStringAsync(cancellationToken)); + } + return null; + } } diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 8aac13b26b..55d7fd3e2b 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -20,7 +20,7 @@ public partial class HgService : IHgService { public const string ManageClientName = "HgResumeManage"; private const string ManageSecretHeader = "X-Manage-Secret"; - private const string AllZeroHash = "0000000000000000000000000000000000000000"; + public const string AllZeroHash = "0000000000000000000000000000000000000000"; private readonly IOptions _options; private readonly Lazy _hgClient; diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 459f14b138..a050169c49 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -25,7 +25,9 @@ public class ProjectService( IEmailService emailService, FwHeadlessClient fwHeadless) { - public async Task CreateProject(CreateProjectInput input) + public async Task CreateProject(CreateProjectInput input, + ProjectMigrationStatus? projectOrigin = null, + FwDataInitStatus fwDataInitStatus = FwDataInitStatus.None) { await using var transaction = await dbContext.Database.BeginTransactionAsync(); var projectId = input.Id ?? Guid.NewGuid(); @@ -41,7 +43,8 @@ public async Task CreateProject(CreateProjectInput input) Id = projectId, Code = input.Code, Name = input.Name, - ProjectOrigin = ProjectMigrationStatus.Migrated, + ProjectOrigin = projectOrigin ?? ProjectMigrationStatus.Migrated, + FwDataInitStatus = fwDataInitStatus, Description = input.Description, Type = input.Type, LastCommit = null, @@ -232,6 +235,52 @@ public async Task FinishReset(string code, Stream? zipFile = null) return project; } + /// + /// Marks a project whose repo FwHeadless has populated from the template as ready, recording the + /// commit it pushed. + /// + public async Task FinishFwDataInit(Guid projectId) + { + var project = await dbContext.Projects.FindAsync(projectId); + if (project is null) return; + project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code); + project.FwDataInitStatus = FwDataInitStatus.None; + await dbContext.SaveChangesAsync(); + } + + /// + /// Compensating cleanup for a project whose template population failed after CreateProject: + /// deletes FwHeadless's local project state, then removes the repo and the project row and + /// invalidates the same caches as a permanent delete. + /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a + /// project that InitFwDataProject just created. + /// + public async Task CleanupFailedProjectCreation(Guid projectId, string code) + { + // do this first, because it throws if a creation or sync is in progress, which stops us from + // tearing down an active or already-completed creation (mirrors DeleteProjectPermanently) + await fwHeadless.DeleteProject(projectId); + try + { + // Repo before row: if deleting the repo fails, the project row survives for an admin to see and + // delete, instead of an orphaned repo with no row pointing at it + await hgService.DeleteRepoIfExists(code); + var project = await dbContext.Projects.FindAsync(projectId); + if (project is not null) + { + dbContext.Projects.Remove(project); + await dbContext.SaveChangesAsync(); + } + } + finally + { + // Don't forget to add more Invalidate calls here if we add new caches + InvalidateProjectCodeCache(code); + InvalidateProjectConfidentialityCache(projectId); + InvalidateProjectOrgIdsCache(projectId); + } + } + public async ValueTask LookupProjectOrgIds(Guid projectId) { var cacheKey = $"ProjectOrgsForId:{projectId}"; diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs new file mode 100644 index 0000000000..ae0bbfa03e --- /dev/null +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -0,0 +1,59 @@ +using SIL.WritingSystems; + +namespace LexCore.Entities; + +/// +/// Body of the internal FwHeadless "init fwdata project" call. The vernacular/analysis +/// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one +/// vernacular; analysis and UI default to "en"); FwHeadless validates them again before using them. +/// +public record InitFwDataProjectInput( + IReadOnlyList WsVernacular, + IReadOnlyList WsAnalysis, + string WsUi) +{ + // Deliberately generous: real projects use far fewer writing systems than this, and real tags are far + // shorter, but private-use subtags after "-x-" may carry arbitrary metadata and we don't want to forbid + // any reasonable use of it. + public const int MaxWritingSystemsPerList = 100; + public const int MaxWritingSystemTagLength = 128; + + /// + /// Checks shared by LexBoxApi (before it creates anything) and FwHeadless. Returns an error + /// message, or null if the list is acceptable. + /// + public static string? ValidateWritingSystemList(string kind, IReadOnlyList? tags) + { + if (tags is not { Count: > 0 }) + return $"At least one {kind} writing system is required"; + if (tags.Count > MaxWritingSystemsPerList) + return $"At most {MaxWritingSystemsPerList} {kind} writing systems are allowed"; + if (tags.Any(string.IsNullOrWhiteSpace)) + return $"Empty {kind} writing system code"; + if (tags.FirstOrDefault(t => t.Length > MaxWritingSystemTagLength) is { } tooLong) + return $"The {kind} writing system code is longer than {MaxWritingSystemTagLength} characters: {tooLong[..32]}…"; + if (tags.FirstOrDefault(t => !IetfLanguageTag.IsValid(t)) is { } invalid) + return $"Invalid {kind} writing system code: {invalid}"; + // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system. Tags that differ only + // by an implicit script ("fr", "fr-Latn") get through, because spotting those needs the SLDR, which + // LexBox doesn't load; ProjectLoader.NewProject merges them into one writing system. + if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate) + return $"Duplicate {kind} writing system: {duplicate.Key}"; + return null; + } + + /// + /// Checks for the UI writing system, shared like . Returns an + /// error message, or null if the tag is acceptable. + /// + public static string? ValidateUiWritingSystem(string? tag) + { + if (string.IsNullOrWhiteSpace(tag)) + return "A UI writing system is required"; + if (tag.Length > MaxWritingSystemTagLength) + return $"The UI writing system code is longer than {MaxWritingSystemTagLength} characters: {tag[..32]}…"; + if (!IetfLanguageTag.IsValid(tag)) + return $"Invalid UI writing system code: {tag}"; + return null; + } +} diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index 85d795c720..d3a5406a28 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -25,6 +25,7 @@ public class Project : EntityBase public int? RepoSizeInKb { get; set; } public DateTimeOffset? DeletedDate { get; set; } public ResetStatus ResetStatus { get; set; } = ResetStatus.None; + public FwDataInitStatus FwDataInitStatus { get; set; } = FwDataInitStatus.None; //historical reference for if this project originated here (migrated), or came from redmine, public or private public ProjectMigrationStatus ProjectOrigin { get; set; } = ProjectMigrationStatus.Migrated; @@ -97,6 +98,7 @@ public enum ProjectMigrationStatus Migrating, PrivateRedmine, PublicRedmine, + LanguageForgeNonSR, } public enum ResetStatus @@ -105,6 +107,13 @@ public enum ResetStatus InProgress = 1 } +public enum FwDataInitStatus +{ + None = 0, + // Created by InitFwDataProject, and FwHeadless hasn't yet finished populating its repo from the template + InProgress = 1 +} + public enum ProjectType { Unknown = 0, diff --git a/backend/LexCore/Entities/ProjectCode.cs b/backend/LexCore/Entities/ProjectCode.cs index 51ac881f4f..edc50cb69f 100644 --- a/backend/LexCore/Entities/ProjectCode.cs +++ b/backend/LexCore/Entities/ProjectCode.cs @@ -34,10 +34,19 @@ private void AssertIsSafeRepoName(string name) { if (InvalidRepoNames.Contains(name)) throw new ArgumentException($"Invalid repo name: {name}."); - if (!ProjectCodeRegex().IsMatch(name)) + if (!IsValidCode(name)) throw new ArgumentException($"Invalid repo name: {name}."); } - [GeneratedRegex(Project.ProjectCodeRegex)] + /// + /// Whether the whole string matches . Use this rather than + /// Regex.IsMatch with that pattern: its $ also matches before a trailing newline, so "abcd" + /// plus a line feed would pass, and hg URLs drop the newline, so they'd point at project "abcd". + /// + public static bool IsValidCode(string? code) => code is not null && ProjectCodeRegex().IsMatch(code); + + // Project.ProjectCodeRegex with \z instead of $ (see IsValidCode). That one can stay as it is for + // [RegularExpression], which already requires the match to cover the whole string. + [GeneratedRegex(@"^[a-z\d][a-z-\d]*\z")] private static partial Regex ProjectCodeRegex(); } diff --git a/backend/LexCore/LexCore.csproj b/backend/LexCore/LexCore.csproj index 8a1b8a72ff..cded3b2bad 100644 --- a/backend/LexCore/LexCore.csproj +++ b/backend/LexCore/LexCore.csproj @@ -10,6 +10,8 @@ LexCore is a plain library (no ASP.NET framework reference), so these come as packages. --> + + diff --git a/backend/LexCore/Sync/ProjectCreationStatus.cs b/backend/LexCore/Sync/ProjectCreationStatus.cs new file mode 100644 index 0000000000..67260ec19b --- /dev/null +++ b/backend/LexCore/Sync/ProjectCreationStatus.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; + +namespace LexCore.Sync; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ProjectCreationStatusEnum +{ + /// No creation is in flight and no project data exists in FwHeadless for this project. + NotCreated, + /// A creation is running right now. + Creating, + /// Project data exists in FwHeadless, i.e. a creation finished successfully. + Created, + /// A creation was attempted and failed. says why. + CreationFailed, + /// + /// The caller's await gave up before creation finished. The creation itself is unaffected and is + /// still running -- poll again rather than treating this as a failure. + /// + TimedOutAwaitingCreation, +} + +/// +/// Status of a FwHeadless project creation, for callers that want to poll (or long-poll) instead of +/// holding a request open for the whole operation. Mirrors , which does +/// the same job for merges. +/// +public record ProjectCreationStatus( + [property: JsonConverter(typeof(JsonStringEnumConverter))] + ProjectCreationStatusEnum Status, + string? Error = null) +{ + public static ProjectCreationStatus NotCreated => new(ProjectCreationStatusEnum.NotCreated); + public static ProjectCreationStatus Creating => new(ProjectCreationStatusEnum.Creating); + public static ProjectCreationStatus Created => new(ProjectCreationStatusEnum.Created); + public static ProjectCreationStatus TimedOutAwaitingCreation => new(ProjectCreationStatusEnum.TimedOutAwaitingCreation); + + public static ProjectCreationStatus Failed(string error) => new(ProjectCreationStatusEnum.CreationFailed, error); + + /// True once creation has stopped running, whether it succeeded or failed. + [JsonIgnore] + public bool IsFinished => Status is ProjectCreationStatusEnum.Created or ProjectCreationStatusEnum.CreationFailed; +} diff --git a/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs new file mode 100644 index 0000000000..9795c46fde --- /dev/null +++ b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs @@ -0,0 +1,1425 @@ +// +using System; +using System.Collections.Generic; +using LexData; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace LexData.Migrations +{ + [DbContext(typeof(LexBoxDbContext))] + [Migration("20261109090102_AddProjectFwDataInitStatus")] + partial class AddProjectFwDataInitStatus + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BlobData") + .HasColumnType("bytea") + .HasColumnName("blob_data"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_blob_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCalendar", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Calendar") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("calendar"); + + b.HasKey("SchedulerName", "CalendarName"); + + b.ToTable("qrtz_calendars", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CronExpression") + .IsRequired() + .HasColumnType("text") + .HasColumnName("cron_expression"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_cron_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzFiredTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("EntryId") + .HasColumnType("text") + .HasColumnName("entry_id"); + + b.Property("FiredTime") + .HasColumnType("bigint") + .HasColumnName("fired_time"); + + b.Property("InstanceName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.Property("ScheduledTime") + .HasColumnType("bigint") + .HasColumnName("sched_time"); + + b.Property("State") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state"); + + b.Property("TriggerGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("TriggerName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.HasKey("SchedulerName", "EntryId"); + + b.HasIndex("InstanceName") + .HasDatabaseName("idx_qrtz_ft_trig_inst_name"); + + b.HasIndex("JobGroup") + .HasDatabaseName("idx_qrtz_ft_job_group"); + + b.HasIndex("JobName") + .HasDatabaseName("idx_qrtz_ft_job_name"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_ft_job_req_recovery"); + + b.HasIndex("TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_group"); + + b.HasIndex("TriggerName") + .HasDatabaseName("idx_qrtz_ft_trig_name"); + + b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup") + .HasDatabaseName("idx_qrtz_ft_trig_nm_gp"); + + b.ToTable("qrtz_fired_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("JobName") + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("JobGroup") + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("IsDurable") + .HasColumnType("bool") + .HasColumnName("is_durable"); + + b.Property("IsNonConcurrent") + .HasColumnType("bool") + .HasColumnName("is_nonconcurrent"); + + b.Property("IsUpdateData") + .HasColumnType("bool") + .HasColumnName("is_update_data"); + + b.Property("JobClassName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_class_name"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("RequestsRecovery") + .HasColumnType("bool") + .HasColumnName("requests_recovery"); + + b.HasKey("SchedulerName", "JobName", "JobGroup"); + + b.HasIndex("RequestsRecovery") + .HasDatabaseName("idx_qrtz_j_req_recovery"); + + b.ToTable("qrtz_job_details", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzLock", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("LockName") + .HasColumnType("text") + .HasColumnName("lock_name"); + + b.HasKey("SchedulerName", "LockName"); + + b.ToTable("qrtz_locks", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzPausedTriggerGroup", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.HasKey("SchedulerName", "TriggerGroup"); + + b.ToTable("qrtz_paused_trigger_grps", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSchedulerState", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("InstanceName") + .HasColumnType("text") + .HasColumnName("instance_name"); + + b.Property("CheckInInterval") + .HasColumnType("bigint") + .HasColumnName("checkin_interval"); + + b.Property("LastCheckInTime") + .HasColumnType("bigint") + .HasColumnName("last_checkin_time"); + + b.HasKey("SchedulerName", "InstanceName"); + + b.ToTable("qrtz_scheduler_state", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("BooleanProperty1") + .HasColumnType("bool") + .HasColumnName("bool_prop_1"); + + b.Property("BooleanProperty2") + .HasColumnType("bool") + .HasColumnName("bool_prop_2"); + + b.Property("DecimalProperty1") + .HasColumnType("numeric") + .HasColumnName("dec_prop_1"); + + b.Property("DecimalProperty2") + .HasColumnType("numeric") + .HasColumnName("dec_prop_2"); + + b.Property("IntegerProperty1") + .HasColumnType("integer") + .HasColumnName("int_prop_1"); + + b.Property("IntegerProperty2") + .HasColumnType("integer") + .HasColumnName("int_prop_2"); + + b.Property("LongProperty1") + .HasColumnType("bigint") + .HasColumnName("long_prop_1"); + + b.Property("LongProperty2") + .HasColumnType("bigint") + .HasColumnName("long_prop_2"); + + b.Property("StringProperty1") + .HasColumnType("text") + .HasColumnName("str_prop_1"); + + b.Property("StringProperty2") + .HasColumnType("text") + .HasColumnName("str_prop_2"); + + b.Property("StringProperty3") + .HasColumnType("text") + .HasColumnName("str_prop_3"); + + b.Property("TimeZoneId") + .HasColumnType("text") + .HasColumnName("time_zone_id"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_simprop_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("RepeatCount") + .HasColumnType("bigint") + .HasColumnName("repeat_count"); + + b.Property("RepeatInterval") + .HasColumnType("bigint") + .HasColumnName("repeat_interval"); + + b.Property("TimesTriggered") + .HasColumnType("bigint") + .HasColumnName("times_triggered"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.ToTable("qrtz_simple_triggers", "quartz"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.Property("SchedulerName") + .HasColumnType("text") + .HasColumnName("sched_name"); + + b.Property("TriggerName") + .HasColumnType("text") + .HasColumnName("trigger_name"); + + b.Property("TriggerGroup") + .HasColumnType("text") + .HasColumnName("trigger_group"); + + b.Property("CalendarName") + .HasColumnType("text") + .HasColumnName("calendar_name"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EndTime") + .HasColumnType("bigint") + .HasColumnName("end_time"); + + b.Property("JobData") + .HasColumnType("bytea") + .HasColumnName("job_data"); + + b.Property("JobGroup") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_group"); + + b.Property("JobName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("job_name"); + + b.Property("MisfireInstruction") + .HasColumnType("smallint") + .HasColumnName("misfire_instr"); + + b.Property("MisfireOriginalFireTime") + .HasColumnType("bigint") + .HasColumnName("misfire_orig_fire_time"); + + b.Property("NextFireTime") + .HasColumnType("bigint") + .HasColumnName("next_fire_time"); + + b.Property("PreviousFireTime") + .HasColumnType("bigint") + .HasColumnName("prev_fire_time"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("StartTime") + .HasColumnType("bigint") + .HasColumnName("start_time"); + + b.Property("TriggerState") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_state"); + + b.Property("TriggerType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("trigger_type"); + + b.HasKey("SchedulerName", "TriggerName", "TriggerGroup"); + + b.HasIndex("NextFireTime") + .HasDatabaseName("idx_qrtz_t_next_fire_time"); + + b.HasIndex("TriggerState") + .HasDatabaseName("idx_qrtz_t_state"); + + b.HasIndex("NextFireTime", "TriggerState") + .HasDatabaseName("idx_qrtz_t_nft_st"); + + b.HasIndex("SchedulerName", "JobName", "JobGroup"); + + b.ToTable("qrtz_triggers", "quartz"); + }); + + modelBuilder.Entity("LexCore.Entities.DraftProject", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("IsConfidential") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("ProjectManagerId") + .HasColumnType("uuid"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ProjectManagerId"); + + b.ToTable("DraftProjects"); + }); + + modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b => + { + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("FlexModelVersion") + .HasColumnType("integer"); + + b.Property("LangProjectId") + .HasColumnType("uuid"); + + b.Property("LexEntryCount") + .HasColumnType("integer"); + + b.HasKey("ProjectId"); + + b.ToTable("FlexProjectMetadata"); + }); + + modelBuilder.Entity("LexCore.Entities.MediaFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("text"); + + b.Property("Metadata") + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValueSql("'{}'"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Revision") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("LexCore.Entities.OrgMember", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("OrgId"); + + b.HasIndex("UserId", "OrgId") + .IsUnique(); + + b.ToTable("OrgMembers", (string)null); + }); + + modelBuilder.Entity("LexCore.Entities.OrgProjects", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("OrgId") + .HasColumnType("uuid"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("OrgId", "ProjectId") + .IsUnique(); + + b.ToTable("OrgProjects"); + }); + + modelBuilder.Entity("LexCore.Entities.Organization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Orgs", (string)null); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("DeletedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("FwDataInitStatus") + .HasColumnType("integer"); + + b.Property("IsConfidential") + .HasColumnType("boolean"); + + b.Property("LastCommit") + .HasColumnType("timestamp with time zone"); + + b.Property("MigratedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("ProjectOrigin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("RepoSizeInKb") + .HasColumnType("integer"); + + b.Property("ResetStatus") + .HasColumnType("integer"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ParentId"); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("UserId", "ProjectId") + .IsUnique(); + + b.ToTable("ProjectUsers"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CanCreateProjects") + .HasColumnType("boolean"); + + b.Property("CreatedById") + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Email") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("EmailVerified") + .HasColumnType("boolean"); + + b.Property>("FeatureFlags") + .HasColumnType("text[]"); + + b.Property("GoogleId") + .HasColumnType("text"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("LastActive") + .HasColumnType("timestamp with time zone"); + + b.Property("LocalizationCode") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("en"); + + b.Property("Locked") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OptedOutOfAnalytics") + .HasColumnType("boolean"); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("PasswordStrength") + .HasColumnType("integer"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Username") + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("Email") + .IsUnique(); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ClientId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ClientSecret") + .HasColumnType("text"); + + b.Property("ClientType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ConsentType") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("DisplayNames") + .HasColumnType("text"); + + b.Property("JsonWebKeySet") + .HasColumnType("text"); + + b.Property("Permissions") + .HasColumnType("text"); + + b.Property("PostLogoutRedirectUris") + .HasColumnType("text"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RedirectUris") + .HasColumnType("text"); + + b.Property("Requirements") + .HasColumnType("text"); + + b.Property("Settings") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ClientId") + .IsUnique(); + + b.ToTable("OpenIddictApplications", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("Scopes") + .HasColumnType("text"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)"); + + b.Property("Type") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("Id"); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type"); + + b.ToTable("OpenIddictAuthorizations", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreScope", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Descriptions") + .HasColumnType("text"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("DisplayNames") + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("Resources") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("OpenIddictScopes", (string)null); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("text"); + + b.Property("ApplicationId") + .HasColumnType("text"); + + b.Property("AuthorizationId") + .HasColumnType("text"); + + b.Property("ConcurrencyToken") + .IsConcurrencyToken() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CreationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Payload") + .HasColumnType("text"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RedemptionDate") + .HasColumnType("timestamp with time zone"); + + b.Property("ReferenceId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Status") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Subject") + .HasMaxLength(400) + .HasColumnType("character varying(400)"); + + b.Property("Type") + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.HasKey("Id"); + + b.HasIndex("AuthorizationId"); + + b.HasIndex("ReferenceId") + .IsUnique(); + + b.HasIndex("ApplicationId", "Status", "Subject", "Type"); + + b.ToTable("OpenIddictTokens", (string)null); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChangeEntities") + .HasColumnType("jsonb"); + + b.Property("ClientId") + .HasColumnType("uuid"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 => + { + b1.IsRequired(); + + b1.Property("Counter") + .HasColumnType("bigint"); + + b1.Property("DateTime") + .HasColumnType("timestamp with time zone"); + }); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.ToTable("CrdtCommits", (string)null); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("BlobTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("CronTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("SimplePropertyTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger") + .WithMany("SimpleTriggers") + .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Trigger"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", "JobDetail") + .WithMany("Triggers") + .HasForeignKey("SchedulerName", "JobName", "JobGroup") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobDetail"); + }); + + modelBuilder.Entity("LexCore.Entities.DraftProject", b => + { + b.HasOne("LexCore.Entities.User", "ProjectManager") + .WithMany() + .HasForeignKey("ProjectManagerId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ProjectManager"); + }); + + modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithOne("FlexProjectMetadata") + .HasForeignKey("LexCore.Entities.FlexProjectMetadata", "ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("LexCore.Entities.ProjectWritingSystems", "WritingSystems", b1 => + { + b1.Property("FlexProjectMetadataProjectId"); + + b1.HasKey("FlexProjectMetadataProjectId"); + + b1.ToTable("FlexProjectMetadata"); + + b1 + .ToJson("WritingSystems") + .HasColumnType("jsonb"); + + b1.WithOwner() + .HasForeignKey("FlexProjectMetadataProjectId"); + + b1.OwnsMany("LexCore.Entities.FLExWsId", "AnalysisWss", b2 => + { + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); + + b2.Property("__synthesizedOrdinal") + .ValueGeneratedOnAdd(); + + b2.Property("IsActive"); + + b2.Property("IsDefault"); + + b2.Property("Tag") + .IsRequired(); + + b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); + + b2.ToTable("FlexProjectMetadata"); + + b2.WithOwner() + .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId"); + }); + + b1.OwnsMany("LexCore.Entities.FLExWsId", "VernacularWss", b2 => + { + b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId"); + + b2.Property("__synthesizedOrdinal") + .ValueGeneratedOnAdd(); + + b2.Property("IsActive"); + + b2.Property("IsDefault"); + + b2.Property("Tag") + .IsRequired(); + + b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal"); + + b2.ToTable("FlexProjectMetadata"); + + b2.WithOwner() + .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId"); + }); + + b1.Navigation("AnalysisWss"); + + b1.Navigation("VernacularWss"); + }); + + b.Navigation("WritingSystems"); + }); + + modelBuilder.Entity("LexCore.Entities.MediaFile", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("LexCore.Entities.OrgMember", b => + { + b.HasOne("LexCore.Entities.Organization", "Organization") + .WithMany("Members") + .HasForeignKey("OrgId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Organizations") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Organization"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.OrgProjects", b => + { + b.HasOne("LexCore.Entities.Organization", "Org") + .WithMany() + .HasForeignKey("OrgId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany() + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Org"); + + b.Navigation("Project"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ParentId"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany("Users") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Projects") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Project"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.HasOne("LexCore.Entities.User", "CreatedBy") + .WithMany("UsersICreated") + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Authorizations") + .HasForeignKey("ApplicationId"); + + b.Navigation("Application"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b => + { + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application") + .WithMany("Tokens") + .HasForeignKey("ApplicationId"); + + b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", "Authorization") + .WithMany("Tokens") + .HasForeignKey("AuthorizationId"); + + b.Navigation("Application"); + + b.Navigation("Authorization"); + }); + + modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany("HarmonyCommits") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b => + { + b.Navigation("Triggers"); + }); + + modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b => + { + b.Navigation("BlobTriggers"); + + b.Navigation("CronTriggers"); + + b.Navigation("SimplePropertyTriggers"); + + b.Navigation("SimpleTriggers"); + }); + + modelBuilder.Entity("LexCore.Entities.Organization", b => + { + b.Navigation("Members"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Navigation("FlexProjectMetadata"); + + b.Navigation("HarmonyCommits"); + + b.Navigation("Users"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Navigation("Organizations"); + + b.Navigation("Projects"); + + b.Navigation("UsersICreated"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b => + { + b.Navigation("Authorizations"); + + b.Navigation("Tokens"); + }); + + modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b => + { + b.Navigation("Tokens"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs new file mode 100644 index 0000000000..7fd0ffdad8 --- /dev/null +++ b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace LexData.Migrations +{ + /// + public partial class AddProjectFwDataInitStatus : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "FwDataInitStatus", + table: "Projects", + type: "integer", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "FwDataInitStatus", + table: "Projects"); + } + } +} diff --git a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs index a8a113dd7c..6678f08f20 100644 --- a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs +++ b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs @@ -694,6 +694,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Description") .HasColumnType("text"); + b.Property("FwDataInitStatus") + .HasColumnType("integer"); + b.Property("IsConfidential") .HasColumnType("boolean"); diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs new file mode 100644 index 0000000000..bb8b796ec8 --- /dev/null +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -0,0 +1,131 @@ +using FwHeadless.Services; +using LexCore.Sync; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.FwHeadless; + +/// +/// Unit tests for the per-project creation reservation that stops a sync from racing a project that +/// is still being created from a template (and stops two concurrent creations of the same project). +/// +public class SyncHostedServiceCreationReservationTests +{ + /// Bounded so a regression that never completes the wait fails the test instead of hanging it. + private static CancellationToken TestTimeout => new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token; + + private static SyncHostedService NewService() => + new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + + [Fact] + public void TryStartProjectCreation_blocks_a_second_concurrent_creation_and_is_reusable_after_release() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a creation is already in flight for this project"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); + svc.TryStartProjectCreation(projectId).Should().BeTrue("the reservation is released and reusable"); + } + + [Fact] + public void QueueJob_is_refused_while_a_project_is_being_created() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.QueueJob(projectId).Should().Be(QueueJobResult.ProjectBeingCreated, "a sync must not race a project that's still being created"); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued, "syncing is allowed once creation has finished"); + } + + [Fact] + public void A_queued_sync_blocks_a_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); + } + + [Fact] + public void QueueJob_is_refused_while_a_sync_is_running() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued); + svc.MarkSyncStarted(projectId); + + svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning, + "the running sync may have missed the caller's changes, so it must not be reported as covering them"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is running for this project"); + } + + [Fact] + public async Task AwaitCreationFinished_returns_the_outcome_the_creation_reported() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + + var awaiting = svc.AwaitCreationFinished(projectId, TestTimeout); + awaiting.IsCompleted.Should().BeFalse("the creation is still in flight"); + + svc.EndProjectCreation(projectId, ProjectCreationStatus.Failed("the push failed")); + + var result = await awaiting; + result.Should().NotBeNull(); + result!.Status.Should().Be(ProjectCreationStatusEnum.CreationFailed); + result.Error.Should().Be("the push failed"); + } + + [Fact] + public async Task AwaitCreationFinished_returns_a_result_that_finished_before_the_caller_asked() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.EndProjectCreation(projectId, ProjectCreationStatus.Created); + + // The caller shows up late (e.g. its own request timed out and it came back to ask). It must + // still learn the outcome rather than being told nothing ever happened. + var result = await svc.AwaitCreationFinished(projectId, TestTimeout); + result.Should().NotBeNull(); + result!.Status.Should().Be(ProjectCreationStatusEnum.Created); + } + + [Fact] + public async Task AwaitCreationFinished_returns_null_when_no_creation_is_known() + { + var svc = NewService(); + + var result = await svc.AwaitCreationFinished(Guid.NewGuid(), TestTimeout); + result.Should().BeNull("the caller should fall back to deriving the status from the project itself"); + } + + [Fact] + public async Task AwaitCreationFinished_honours_the_caller_token_without_ending_the_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + + using var cts = new CancellationTokenSource(); + var awaiting = svc.AwaitCreationFinished(projectId, cts.Token); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => awaiting); + svc.IsProjectBeingCreated(projectId).Should().BeTrue("giving up on the wait must not cancel the creation"); + } +} diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs new file mode 100644 index 0000000000..97e9fe0cd9 --- /dev/null +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -0,0 +1,189 @@ +using LexBoxApi.Controllers; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Testing.LexBoxApi; + +/// +/// Unit tests for ProjectController.InitFwDataProject input validation. The 400 branches run before +/// any injected service is touched, so the controller can be built with null dependencies; only a +/// ProblemDetailsFactory (from AddControllers) is needed for Problem() to render. +/// +public class InitFwDataProjectValidationTests +{ + private static ProjectController NewController(IPermissionService? permissionService = null) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves + return new ProjectController(null!, null!, null!, permissionService!, null!, NullLogger.Instance) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() } + } + }; + } + + [Fact] + public async Task Rejects_when_no_vernacular_writing_system_is_supplied() + { + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: []); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("vernacular"); + } + + // The controller's services are all null, so reaching the 400 proves nothing was created first + [Fact] + public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_anything() + { + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr", "fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Duplicate vernacular writing system"); + } + + [Fact] + public async Task Rejects_an_invalid_vernacular_writing_system_before_creating_anything() + { + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["not_a_tag"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Be("Invalid vernacular writing system code: not_a_tag"); + } + + [Fact] + public async Task Rejects_too_many_analysis_writing_systems_before_creating_anything() + { + var analysis = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsAnalysis: analysis); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("analysis writing systems are allowed"); + } + + [Fact] + public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything() + { + var wsUi = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsUi: wsUi); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("UI writing system code is longer than"); + } + + [Theory] + [InlineData("Bad Code!")] + // A trailing newline passes a Regex.IsMatch with $, and the hg URL would then name project "abcd" + [InlineData("abcd\n")] + public async Task Rejects_an_invalid_project_code(string code) + { + var result = await NewController().InitFwDataProject(code, name: "My project", wsVernacular: ["fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project code"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task Rejects_a_missing_project_name(string? name) + { + var result = await NewController().InitFwDataProject("myproj", name: name!, wsVernacular: ["fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("A project name is required"); + } + + /// A permission service that either passes or fails the admin assertion. + private static IPermissionService PermissionService(bool isAdmin) + { + var mock = new Mock(); + if (!isAdmin) mock.Setup(p => p.AssertIsAdmin()).Throws(); + return mock.Object; + } + + [Fact] + public async Task Rejects_a_project_origin_that_is_not_a_migration_status() + { + var result = await NewController(PermissionService(isAdmin: true)) + .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin"); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project origin"); + } + + [Fact] + public async Task Rejects_a_project_origin_from_a_non_admin() + { + // The admin check must happen before the value is even parsed, and must outlive [AdminRequired], + // which is expected to be relaxed for this endpoint later. + var act = () => NewController(PermissionService(isAdmin: false)) + .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "Migrated"); + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task Ignores_an_empty_project_origin_without_requiring_admin() + { + // An empty value isn't "specified", so it needs no admin check; a null permission service + // proves AssertIsAdmin() was never called. Fails later (null project service) than validation. + var act = () => NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: ""); + await act.Should().ThrowAsync(); + } + + [Theory] + [InlineData("Migrated", ProjectMigrationStatus.Migrated)] + [InlineData("migrating", ProjectMigrationStatus.Migrating)] + [InlineData("PUBLICREDMINE", ProjectMigrationStatus.PublicRedmine)] + [InlineData("languageforgenonsr", ProjectMigrationStatus.LanguageForgeNonSR)] + public void Project_origin_parses_case_insensitively(string input, ProjectMigrationStatus expected) + { + ProjectController.TryParseProjectOrigin(input, out var origin).Should().BeTrue(); + origin.Should().Be(expected); + } + + [Theory] + [InlineData("NotAnOrigin")] + [InlineData("1")] // Enum.TryParse accepts raw numbers; the API takes names only + [InlineData("42")] + [InlineData("Migrated,Migrating")] // ...and comma-separated lists, which OR into a defined value here + [InlineData("")] + public void Project_origin_rejects_values_that_are_not_named_members(string input) + { + ProjectController.TryParseProjectOrigin(input, out _).Should().BeFalse(); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_null() + { + ProjectController.AnalysisWritingSystemsOrDefault(null).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_empty() + { + ProjectController.AnalysisWritingSystemsOrDefault([]).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_are_kept_when_supplied() + { + ProjectController.AnalysisWritingSystemsOrDefault(["fr", "es"]).Should().Equal("fr", "es"); + } +} diff --git a/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs new file mode 100644 index 0000000000..e34e61b1f1 --- /dev/null +++ b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs @@ -0,0 +1,60 @@ +using LexBoxApi.Jobs; +using LexBoxApi.Services; +using LexBoxApi.Services.Email; +using LexCore.Config; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexData; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using Quartz; +using Testing.Fixtures; + +namespace Testing.LexBoxApi.Jobs; + +[Collection(nameof(TestingServicesFixture))] +[Trait("Category", "RequiresDb")] +public class InitFwDataProjectJobDbTests(TestingServicesFixture testing) +{ + [Fact] + public async Task A_failure_marking_the_project_ready_is_retried() + { + var serviceProvider = testing.ConfigureServices(); + var dbContext = serviceProvider.GetRequiredService(); + var hgService = new Mock(); + // The push landed, so the attempt counts as created without asking FwHeadless + hgService.Setup(h => h.GetTipHash(It.IsAny(), It.IsAny())) + .ReturnsAsync("1234567890abcdef1234567890abcdef12345678"); + hgService.Setup(h => h.GetLastCommitTimeFromHg(It.IsAny())) + .ThrowsAsync(new HttpRequestException("hgweb blip")); + var projectService = new ProjectService(dbContext, hgService.Object, + serviceProvider.GetRequiredService>(), + Mock.Of(), Mock.Of(), null!); + var projectId = await projectService.CreateProject( + new(null, "Init finalize retry", "", "init-finalize-retry", ProjectType.FLEx, RetentionPolicy.Test, false, null, null), + fwDataInitStatus: FwDataInitStatus.InProgress); + var scheduler = new Mock(); + var schedulerFactory = Mock.Of(f => f.GetScheduler(It.IsAny()) == Task.FromResult(scheduler.Object)); + IJob job = new InitFwDataProjectJob(dbContext, projectService, + new FwHeadlessClient(new HttpClient(), NullLogger.Instance), + hgService.Object, schedulerFactory, Mock.Of(), NullLogger.Instance) + { + ProjectId = projectId.ToString(), + Input = System.Text.Json.JsonSerializer.Serialize(new InitFwDataProjectInput(["fr"], ["en"], "en")), + Attempt = "1", + }; + + await job.Execute(Mock.Of()); + + hgService.Verify(h => h.GetLastCommitTimeFromHg(It.IsAny()), Times.Once); + scheduler.Verify(s => s.ScheduleJob( + It.Is(t => t.JobDataMap.GetString(nameof(InitFwDataProjectJob.Attempt)) == "2"), + It.IsAny()), Times.Once); + var project = await dbContext.Projects.AsNoTracking().SingleAsync(p => p.Id == projectId); + project.FwDataInitStatus.Should().Be(FwDataInitStatus.InProgress); + } +} diff --git a/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobTests.cs b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobTests.cs new file mode 100644 index 0000000000..ab5005868c --- /dev/null +++ b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobTests.cs @@ -0,0 +1,196 @@ +using System.Net; +using System.Net.Http.Json; +using LexBoxApi.Jobs; +using LexBoxApi.Services; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using LexCore.Sync; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Testing.LexBoxApi.Jobs; + +/// +/// Unit tests for how one run of InitFwDataProjectJob decides what happened to a creation. Every run must be +/// safe to repeat (Quartz reruns the job after an interruption), so these cover picking up after each way an +/// earlier run could have stopped. FwHeadless is a fake HTTP handler and the repo is a mocked IHgService. +/// +public class InitFwDataProjectJobTests +{ + private const string Code = "test-proj"; + private const string PushedTip = "1234567890abcdef1234567890abcdef12345678"; + private static readonly Guid ProjectId = Guid.NewGuid(); + private static readonly InitFwDataProjectInput Input = new(["fr"], ["en"], "en"); + + private string _tip = HgService.AllZeroHash; + private readonly List _requests = []; + private readonly Mock _hgService = new(); + + // Responses by request, as "METHOD path". An unlisted request throws, which AttemptInit reports as + // RetryLater like any other exception, so the tests also check exactly which requests were made. + private readonly Dictionary> _fwHeadless = []; + + private const string CreationStatus = "GET /api/project/creation-status"; + private const string AwaitCreation = "GET /api/project/await-creation-finished"; + private static readonly string DeleteLocalProject = $"DELETE /api/manage/project/{ProjectId}"; + private const string InitProject = "POST /api/project/initFwDataProject"; + + public InitFwDataProjectJobTests() + { + _hgService.Setup(h => h.GetTipHash(Code, It.IsAny())).ReturnsAsync(() => _tip); + } + + private Task Attempt() + { + var client = new FwHeadlessClient(new HttpClient(new FakeHandler(this)) { BaseAddress = new Uri("http://fwheadless/") }, + NullLogger.Instance); + return InitFwDataProjectJob.AttemptInit(client, _hgService.Object, NullLogger.Instance, ProjectId, Code, Input); + } + + private static HttpResponseMessage Status(ProjectCreationStatus status) => + new(HttpStatusCode.OK) { Content = JsonContent.Create(status) }; + + private static HttpResponseMessage Response(HttpStatusCode code, string body = "") => + new(code) { Content = new StringContent(body) }; + + [Fact] + public async Task A_project_whose_push_already_landed_is_created_without_asking_FwHeadless() + { + _tip = PushedTip; + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().BeEmpty("the repo already shows that an earlier run's creation succeeded"); + } + + [Fact] + public async Task Waits_for_a_creation_still_running_in_FwHeadless_instead_of_starting_another() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => + { + _tip = PushedTip; + return Status(ProjectCreationStatus.Created); + }; + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().Equal(CreationStatus, AwaitCreation); + } + + [Fact] + public async Task A_creation_FwHeadless_reports_as_failed_while_waiting_is_a_failure() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => Status(ProjectCreationStatus.Failed("the push failed")); + + var outcome = await Attempt(); + + outcome.Should().Be(InitAttemptOutcome.Failed("the push failed")); + } + + [Fact] + public async Task A_creation_still_running_after_the_wait_is_retried_later() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.Creating); + _fwHeadless[AwaitCreation] = () => Status(ProjectCreationStatus.TimedOutAwaitingCreation); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, AwaitCreation); + } + + [Fact] + public async Task Clears_FwHeadless_local_state_before_starting_a_creation() + { + _fwHeadless[CreationStatus] = () => Response(HttpStatusCode.NotFound); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => Response(HttpStatusCode.OK); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.Created); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task FwHeadless_being_busy_with_the_project_is_retried_later_without_starting_a_creation() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.Conflict); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, DeleteLocalProject); + } + + [Theory] + [InlineData(HttpStatusCode.Conflict, InitAttemptResult.RetryLater)] // something else holds the project, so nothing was built + [InlineData(HttpStatusCode.BadRequest, InitAttemptResult.Failed)] + [InlineData(HttpStatusCode.InternalServerError, InitAttemptResult.Failed)] + public async Task An_answer_from_FwHeadless_decides_the_outcome(HttpStatusCode statusCode, InitAttemptResult expected) + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => Response(statusCode, "what went wrong"); + + var outcome = await Attempt(); + + outcome.Should().Be(new InitAttemptOutcome(expected, "what went wrong")); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task No_answer_from_FwHeadless_is_retried_later_because_it_may_still_be_creating() + { + _fwHeadless[CreationStatus] = () => Status(ProjectCreationStatus.NotCreated); + _fwHeadless[DeleteLocalProject] = () => Response(HttpStatusCode.OK); + _fwHeadless[InitProject] = () => throw new HttpRequestException("Connection refused"); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().Equal(CreationStatus, DeleteLocalProject, InitProject); + } + + [Fact] + public async Task Not_being_able_to_read_the_repo_is_retried_later() + { + _hgService.Setup(h => h.GetTipHash(Code, It.IsAny())).ThrowsAsync(new HttpRequestException("hg is down")); + + var outcome = await Attempt(); + + outcome.Result.Should().Be(InitAttemptResult.RetryLater); + _requests.Should().BeEmpty(); + } + + [Fact] + public void The_creation_input_survives_the_job_store() + { + // The job store only takes strings, so the input travels as JSON; it's a positional record with + // IReadOnlyList properties, which System.Text.Json has to bind through the constructor + var roundTripped = System.Text.Json.JsonSerializer.Deserialize( + System.Text.Json.JsonSerializer.Serialize(new InitFwDataProjectInput(["fr", "es"], ["en", "de"], "pt"))); + + roundTripped.Should().NotBeNull(); + roundTripped!.WsVernacular.Should().Equal("fr", "es"); + roundTripped.WsAnalysis.Should().Equal("en", "de"); + roundTripped.WsUi.Should().Be("pt"); + } + + private class FakeHandler(InitFwDataProjectJobTests test) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var key = $"{request.Method} {request.RequestUri!.AbsolutePath}"; + test._requests.Add(key); + if (!test._fwHeadless.TryGetValue(key, out var respond)) + throw new InvalidOperationException($"Unexpected request to FwHeadless: {key}"); + return Task.FromResult(respond()); + } + } +} diff --git a/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs b/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs index 36fb1f75c5..6e7041825b 100644 --- a/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs +++ b/backend/Testing/LexBoxApi/Services/FwHeadlessClientTests.cs @@ -81,4 +81,16 @@ public async Task DeleteRepo_ThrowsProjectSyncInProgress_OnConflict() await act.Should().ThrowAsync(); } + + [Fact] + public async Task InitFwDataProject_ReturnsAnError_ForAFailureWithAnEmptyBody() + { + // e.g. a FwHeadless that doesn't have the route yet. A null error would read as success. + var client = ClientReturning(HttpStatusCode.NotFound, "", "text/plain"); + + var (statusCode, error) = await client.InitFwDataProject(Guid.NewGuid(), ["fr"], ["en"], "en"); + + statusCode.Should().Be(HttpStatusCode.NotFound); + error.Should().Be("FwHeadless returned HTTP 404"); + } } diff --git a/backend/Testing/LexCore/InitFwDataProjectInputTests.cs b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs new file mode 100644 index 0000000000..fa40c1aaa9 --- /dev/null +++ b/backend/Testing/LexCore/InitFwDataProjectInputTests.cs @@ -0,0 +1,132 @@ +using LexCore.Entities; +using FluentAssertions; + +namespace Testing.LexCore; + +public class InitFwDataProjectInputTests +{ + private static string? Validate(params string[] tags) => + InitFwDataProjectInput.ValidateWritingSystemList("vernacular", tags); + + // A valid BCP 47 tag of the given length. Private-use subtags are at most 8 characters each, and + // libpalaso also rejects a repeated one, so each is different. + private static string ValidTagOfLength(int length) + { + var tag = "qaa-x"; + var i = 0; + while (tag.Length + 9 <= length) tag += $"-p{i++:D7}"; + if (tag.Length < length) tag += "-" + $"q{i:D7}"[..(length - tag.Length - 1)]; + return tag; + } + + [Fact] + public void Accepts_a_typical_list() + { + Validate("fr", "en-US", "qaa-Latn-x-mycustom").Should().BeNull(); + } + + [Fact] + public void Rejects_a_null_list() + { + InitFwDataProjectInput.ValidateWritingSystemList("vernacular", null) + .Should().Contain("At least one vernacular writing system is required"); + } + + [Fact] + public void Rejects_an_empty_list() + { + Validate().Should().Contain("At least one vernacular writing system is required"); + } + + [Fact] + public void Accepts_the_maximum_number_of_writing_systems() + { + var tags = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList).Select(i => $"qaa-x-ws{i}").ToArray(); + Validate(tags).Should().BeNull(); + } + + [Fact] + public void Rejects_more_than_the_maximum_number_of_writing_systems() + { + var tags = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray(); + Validate(tags).Should().Contain($"At most {InitFwDataProjectInput.MaxWritingSystemsPerList}"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Rejects_an_empty_tag(string tag) + { + Validate("fr", tag).Should().Contain("Empty vernacular writing system code"); + } + + [Fact] + public void Accepts_a_tag_of_the_maximum_length() + { + var tag = ValidTagOfLength(InitFwDataProjectInput.MaxWritingSystemTagLength); + tag.Length.Should().Be(InitFwDataProjectInput.MaxWritingSystemTagLength); + Validate(tag).Should().BeNull(); + } + + [Fact] + public void Rejects_a_tag_longer_than_the_maximum_and_shows_its_start() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var error = Validate("fr", tag); + error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); + error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); + } + + [Theory] + [InlineData("not_a_tag")] + [InlineData("qaa-x-thissubtagistoolong")] + public void Rejects_a_tag_that_is_not_valid_BCP_47(string tag) + { + Validate("fr", tag).Should().Be($"Invalid vernacular writing system code: {tag}"); + } + + [Theory] + [InlineData("fr", "fr")] + [InlineData("fr", "FR")] // BCP 47 tags are case-insensitive + public void Rejects_a_duplicate_tag(string first, string second) + { + Validate(first, "en", second).Should().Contain($"Duplicate vernacular writing system: {first}"); + } + + [Fact] + public void Accepts_a_typical_UI_writing_system() + { + InitFwDataProjectInput.ValidateUiWritingSystem("en").Should().BeNull(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Rejects_a_missing_UI_writing_system(string? tag) + { + InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().Contain("A UI writing system is required"); + } + + [Fact] + public void Accepts_a_UI_writing_system_of_the_maximum_length() + { + var tag = ValidTagOfLength(InitFwDataProjectInput.MaxWritingSystemTagLength); + InitFwDataProjectInput.ValidateUiWritingSystem(tag).Should().BeNull(); + } + + [Fact] + public void Rejects_a_UI_writing_system_longer_than_the_maximum_and_shows_its_start() + { + var tag = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength); + var error = InitFwDataProjectInput.ValidateUiWritingSystem(tag); + error.Should().Contain($"longer than {InitFwDataProjectInput.MaxWritingSystemTagLength} characters"); + error.Should().Contain(tag[..32]).And.NotContain(tag[..33]); + } + + [Fact] + public void Rejects_a_UI_writing_system_that_is_not_valid_BCP_47() + { + InitFwDataProjectInput.ValidateUiWritingSystem("not_a_tag").Should().Be("Invalid UI writing system code: not_a_tag"); + } +} diff --git a/backend/Testing/LexCore/ProjectCodeTests.cs b/backend/Testing/LexCore/ProjectCodeTests.cs index 8844176423..0b42711bc6 100644 --- a/backend/Testing/LexCore/ProjectCodeTests.cs +++ b/backend/Testing/LexCore/ProjectCodeTests.cs @@ -17,6 +17,8 @@ public class ProjectCodeTests [InlineData("!")] [InlineData("#")] [InlineData("-not-start-with-dash")] + [InlineData("test\n")] // Regex $ matches before a trailing newline; the hg URL would then drop it + [InlineData("test\r\n")] public void InvalidCodesThrows(string code) { Assert.Throws(() => new ProjectCode(code)); diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs new file mode 100644 index 0000000000..aac5a48b53 --- /dev/null +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -0,0 +1,234 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Xml.Linq; +using FluentAssertions; +using LexCore.Entities; +using LexCore.Sync; +using Testing.ApiTests; +using Testing.Fixtures; +using Testing.Services; +using Xunit.Abstractions; +using static Testing.Services.Constants; + +namespace Testing.SyncReverseProxy; + +/// +/// End-to-end coverage for the admin "init fwdata project" API. This is the one path that +/// can't be unit-tested: LexBox creates an empty hg repo and FwHeadless does the first push of a +/// template-built .fwdata into it (via LfMergeBridge/Chorus against the real hgweb). Requires the +/// lexbox stack to be up and running, so only run it in CI or with `task test:integration`. +/// +[Trait("Category", "Integration")] +public class InitFwDataProjectTests : IClassFixture +{ + private readonly ITestOutputHelper _output; + private readonly ApiTestBase _adminApiTester; + + public InitFwDataProjectTests(ITestOutputHelper output, IntegrationFixture fixture) + { + _output = output; + _adminApiTester = fixture.AdminApiTester; + } + + [Fact] + public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWritingSystems() + { + // Valid code (lowercase/digits/hyphen, doesn't start with a hyphen), unique per run. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var vernacular = new[] { "fr", "es", "ko", "ru", "my" }; + var analysis = new[] { "de", "pt", "th", "id", "he", "el" }; + var query = $"?code={code}&name={code}" + + string.Concat(vernacular.Select(ws => $"&wsVernacular={ws}")) + + string.Concat(analysis.Select(ws => $"&wsAnalysis={ws}")); + + Guid projectId = default; + try + { + // 1. Create the project via the admin endpoint. It creates the DB row + empty repo and returns + // 202 at once; a background job then has FwHeadless build the template .fwdata and push it + // into the empty repo, and we poll until that has finished. + // The POST isn't idempotent. The shared tester's HttpClient retries transient failures, and a + // retried attempt (after the first already created the project row) would return 409 "already + // exists", so use a client that doesn't retry. Log it in itself (cookie auth) rather than + // reusing the shared tester's captured JWT, which may be stale. + using var createClient = ApiTestBase.NewHttpClient(_adminApiTester.BaseUrl, retry: false).Client; + await JwtHelper.ExecuteLogin(AdminAuth, includeDefaultScope: true, createClient); + var response = await createClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject{query}", null); + response.StatusCode.Should().Be(HttpStatusCode.Accepted, + "creation should be accepted; body: {0}", await response.Content.ReadAsStringAsync()); + projectId = await response.Content.ReadFromJsonAsync(); + projectId.Should().NotBe(Guid.Empty); + response.Headers.Location?.ToString().Should().EndWith($"/api/project/fwDataProjectCreationStatus/{projectId}"); + + var status = await AwaitCreationFinished(createClient, projectId); + status.Status.Should().Be(ProjectCreationStatusEnum.Created, "creation should succeed; error: {0}", status.Error); + + // 2. The first push landed a commit server-side (the empty repo is no longer empty). + var lastCommit = await _adminApiTester.GetProjectLastCommit(code); + lastCommit.Should().NotBeNull("the template project should have been pushed to the repo"); + + var tagsResponse = await _adminApiTester.HttpClient.GetAsync($"{_adminApiTester.BaseUrl}/hg/{code}/tags?style=json"); + tagsResponse.EnsureSuccessStatusCode(); + var tip = (await tagsResponse.Content.ReadFromJsonAsync())?["node"]?.ToString(); + tip.Should().NotBeNullOrEmpty(); + tip!.Replace("0", "").Should().NotBeEmpty("the repo tip should not be the all-zero empty-repo hash"); + + // 3. The pushed project carries the requested writing systems. Send/Receive split the + // template .fwdata into the nested files hg actually tracks; the LangProject's current + // analysis/vernacular writing systems live in General/LanguageProject.langproj (XML). + var langprojResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/General/LanguageProject.langproj"); + langprojResponse.EnsureSuccessStatusCode(); + var langprojXml = await langprojResponse.Content.ReadAsStringAsync(); + langprojXml.Should().NotBeEmpty(); + + // de en pt + // fr es ... + var langProject = XDocument.Parse(langprojXml).Root?.Element("LangProject"); + langProject.Should().NotBeNull("LanguageProject.langproj should contain a LangProject element"); + var allAnalysisWss = SpaceSeparatedUni(langProject!, "AnalysisWss"); + var allVernWss = SpaceSeparatedUni(langProject!, "VernWss"); + var curAnalysisWss = SpaceSeparatedUni(langProject!, "CurAnalysisWss"); + var curVernWss = SpaceSeparatedUni(langProject!, "CurVernWss"); + + // 4. The largest file survived the push. Chorus's LargeFileFilter caps a file at 1 MB unless + // some loaded file-type handler raises the cap for that extension, and the handler that knows + // about .list ships in LibFLExBridge-ChorusPlugin.dll. If plugin discovery ever regresses + // (sillsdev/chorus#393), .list silently falls back to the 1 MB default and Chorus *forgets* + // SemanticDomainList.list instead of pushing it -- no error, no failed send/receive, just a + // project that looks fine here and is missing its semantic domains when FLEx opens it. + // Every other assertion in this test reads a small file, so they all stay green through + // that failure; this is the only one that catches it. + var semDomResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/Linguistics/Lexicon/SemanticDomainList.list"); + semDomResponse.StatusCode.Should().Be(HttpStatusCode.OK, + "SemanticDomainList.list should have been pushed; anything else (404 in particular) means " + + "Chorus filtered it out as oversized, i.e. the FieldWorks file-type handlers never loaded"); + + // The requested writing systems should be current. FieldWorks may add its own defaults (e.g. + // "en" as an analysis WS), so assert each requested code is present rather than exact equality. + foreach (var ws in analysis) + curAnalysisWss.Should().Contain(ws, "analysis writing system {0} should be current in the project", ws); + foreach (var ws in vernacular) + curVernWss.Should().Contain(ws, "vernacular writing system {0} should be current in the project", ws); + + // The requested writing systems should *also* be in the correct order. Any writing systems added + // by FieldWorks should come last. + allAnalysisWss.Take(analysis.Length).Should().Equal(analysis, "analysis writing systems were in the wrong order"); + allVernWss.Take(vernacular.Length).Should().Equal(vernacular, "vernacular writing systems were in the wrong order"); + curAnalysisWss.Should().Equal(analysis, "current analysis writing systems should not contain any extras"); + curVernWss.Should().Equal(vernacular, "current vernacular writing systems should not contain any extras"); + } + finally + { + if (projectId != default) await SoftDeleteProject(projectId); + } + } + + /// + /// Only admins may say where a project came from, so a non-admin passing projectOrigin + /// must be rejected. Today it's [AdminRequired] on the endpoint that rejects it (403); once this + /// endpoint opens up to non-admins and that attribute goes away, the controller's own + /// AssertIsAdmin() will reject it (401). Either way the request must fail and no project may be + /// created, which is what this test pins down across that change. + /// To check it isn't passing for the wrong reason, temporarily delete [AdminRequired] from + /// ProjectController.InitFwDataProject, rebuild, and re-run: it must still pass. + /// + [Fact] + public async Task InitFwDataProject_RejectsProjectOriginFromANonAdmin() + { + var managerApiTester = new ApiTestBase(); + await managerApiTester.LoginAs(ManagerAuth.Username, ManagerAuth.Password); + + // Everything except projectOrigin is valid, so a 400 from input validation can't be what + // rejects this, and the origin itself is a real ProjectMigrationStatus name. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await managerApiTester.HttpClient.PostAsync( + $"{managerApiTester.BaseUrl}/api/project/initFwDataProject" + + $"?code={code}&name={code}&wsVernacular=fr&projectOrigin={nameof(ProjectMigrationStatus.LanguageForgeNonSR)}", null); + + if (response.StatusCode == HttpStatusCode.InternalServerError) + { + // In developer mode, a thrown System.UnauthorizedAccessException exception results in an HTTP 500 + // instead of a 4xx code, with a JSON structure that includes `"title":"System.UnauthorizedAccessException"`. + // We check for that special case here; any non-JSON or JSON that has a different title would indicate a + // different problem and be a test failure. + JsonNode? problemDetails = null; + try + { + var body = await response.Content.ReadAsStringAsync(); + problemDetails = JsonNode.Parse(body); + } + catch (JsonException) + { + // Do nothing here, the Should().NotBeNull() check below handles this + } + var failureReasonMsg = "HTTP 500 should have been caused by developer page printing " + + "a traceback due to " + nameof(UnauthorizedAccessException); + problemDetails.Should().NotBeNull(failureReasonMsg); + problemDetails?["title"]?.ToString().Should().EndWithEquivalentOf( + nameof(UnauthorizedAccessException), failureReasonMsg); + } + else + { + response.StatusCode.Should().BeOneOf([HttpStatusCode.Forbidden, HttpStatusCode.Unauthorized], + "a non-admin may not set projectOrigin; body: {0}", await response.Content.ReadAsStringAsync()); + } + } + + [Fact] + public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() + { + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await _adminApiTester.HttpClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject?code={code}&name={code}&wsAnalysis=en", null); + response.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } + + /// Polls the creation status until the background job has finished, for up to five minutes. + private async Task AwaitCreationFinished(HttpClient client, Guid projectId) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + while (true) + { + var response = await client.GetAsync( + $"{_adminApiTester.BaseUrl}/api/project/fwDataProjectCreationStatus/{projectId}", timeout.Token); + response.StatusCode.Should().Be(HttpStatusCode.OK, + "the project's creation status should be known; body: {0}", await response.Content.ReadAsStringAsync()); + var status = await response.Content.ReadFromJsonAsync(timeout.Token); + status.Should().NotBeNull(); + _output.WriteLine($"Creation status: {status!.Status}"); + if (status.IsFinished) return status; + await Task.Delay(TimeSpan.FromSeconds(2), timeout.Token); + } + } + + // Reads a LangProject child element's text (a space-separated writing-system list) and splits it. + private static string[] SpaceSeparatedUni(XElement langProject, string elementName) + { + var uni = langProject.Element(elementName)?.Element("Uni")?.Value ?? ""; + return uni.Split(' ', StringSplitOptions.RemoveEmptyEntries); + } + + private async Task SoftDeleteProject(Guid projectId) + { + try + { + await _adminApiTester.ExecuteGql($$""" + mutation { + softDeleteProject(input: { projectId: "{{projectId}}" }) { + project { id } + errors { __typename } + } + } + """); + } + catch (Exception ex) + { + _output.WriteLine($"[InitFwDataProjectTests] Ignored cleanup exception: {ex}"); + } + } +} diff --git a/frontend/schema.graphql b/frontend/schema.graphql index bf39d4915b..1936019704 100644 --- a/frontend/schema.graphql +++ b/frontend/schema.graphql @@ -451,6 +451,7 @@ type Project { repoSizeInKb: Int deletedDate: DateTime resetStatus: ResetStatus! + fwDataInitStatus: FwDataInitStatus! projectOrigin: ProjectMigrationStatus! migratedDate: DateTime userCount: Int! @@ -980,6 +981,13 @@ input FlexProjectMetadataSortInput { flexModelVersion: SortEnumType @cost(weight: "10") } +input FwDataInitStatusOperationFilterInput { + eq: FwDataInitStatus @cost(weight: "10") + neq: FwDataInitStatus @cost(weight: "10") + in: [FwDataInitStatus!] @cost(weight: "10") + nin: [FwDataInitStatus!] @cost(weight: "10") +} + input HybridDateTimeFilterInput { and: [HybridDateTimeFilterInput!] or: [HybridDateTimeFilterInput!] @@ -1228,6 +1236,7 @@ input ProjectFilterInput { repoSizeInKb: IntOperationFilterInput deletedDate: DateTimeOperationFilterInput resetStatus: ResetStatusOperationFilterInput + fwDataInitStatus: FwDataInitStatusOperationFilterInput projectOrigin: ProjectMigrationStatusOperationFilterInput migratedDate: DateTimeOperationFilterInput harmonyCommits: ListFilterInputTypeOfServerCommitFilterInput @@ -1264,6 +1273,7 @@ input ProjectSortInput { repoSizeInKb: SortEnumType @cost(weight: "10") deletedDate: SortEnumType @cost(weight: "10") resetStatus: SortEnumType @cost(weight: "10") + fwDataInitStatus: SortEnumType @cost(weight: "10") projectOrigin: SortEnumType @cost(weight: "10") migratedDate: SortEnumType @cost(weight: "10") userCount: SortEnumType @cost(weight: "10") @@ -1501,6 +1511,11 @@ enum FeatureFlag { FW_LITE_BETA } +enum FwDataInitStatus { + NONE + IN_PROGRESS +} + enum JsonValueKind { UNDEFINED OBJECT @@ -1552,6 +1567,7 @@ enum ProjectMigrationStatus { MIGRATING PRIVATE_REDMINE PUBLIC_REDMINE + LANGUAGE_FORGE_NON_SR } enum ProjectRole {