diff --git a/backend/FwHeadless/FwHeadless.csproj b/backend/FwHeadless/FwHeadless.csproj
index 4a6c2721e6..c54ddafba3 100644
--- a/backend/FwHeadless/FwHeadless.csproj
+++ b/backend/FwHeadless/FwHeadless.csproj
@@ -15,6 +15,9 @@
+
+
@@ -40,6 +43,11 @@
+
+
diff --git a/backend/FwHeadless/FwHeadlessConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs
index a7edd76b72..84fe6a1863 100644
--- a/backend/FwHeadless/FwHeadlessConfig.cs
+++ b/backend/FwHeadless/FwHeadlessConfig.cs
@@ -23,6 +23,22 @@ public class FwHeadlessConfig
public long MaxUploadFileSizeBytes => MaxUploadFileSizeKb * 1024;
public string FdoDataModelVersion { get; init; } = "7000072";
+ ///
+ /// The FLExBridge data version. FieldWorks/LfMergeBridge form the Mercurial branch a project's
+ /// data lives on by joining this and the FDO model version with a dot (see ).
+ /// Mirrors FlexBridgeConstants.FlexBridgeDataVersion in FLExBridge, which is internal so we can't
+ /// reference it directly; bump this if FLExBridge bumps its data version.
+ ///
+ public string FlexBridgeDataVersion { get; init; } = "7500002";
+
+ ///
+ /// The Mercurial branch FieldWorks/LfMergeBridge keep a project's data on: FlexBridgeDataVersion,
+ /// a dot, then the FDO model version (e.g. "7500002.7000072"). This is exactly the branch name
+ /// LfMergeBridge derives from , so a new project's genesis commit
+ /// must land on this branch for send/receive to find the data.
+ ///
+ public string SendReceiveBranchName => $"{FlexBridgeDataVersion}.{FdoDataModelVersion}";
+
///
/// Project directory structure in FwHeadless: (Note that FwDataProject.ProjectsPath is the root of a SINGLE project)
/// {ProjectStorageRoot}/
diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs
index e96157c7da..2604d58671 100644
--- a/backend/FwHeadless/FwHeadlessKernel.cs
+++ b/backend/FwHeadless/FwHeadlessKernel.cs
@@ -29,6 +29,7 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services)
services.AddScoped();
services.AddScoped();
services.AddScoped();
+ services.AddScoped();
services.AddScoped();
services.AddScoped();
// Singleton so its file lock is shared across requests and the sync worker
@@ -37,6 +38,14 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services)
.AddLcmCrdtClientCore()
.AddFwDataBridge(ServiceLifetime.Scoped)
.AddFwLiteProjectSync();
+ // The LCM project templates (NewLangProj.fwdata etc.) ship in our output (see FwHeadless.csproj).
+ // FwDataBridgeConfig's default TemplatesFolder points at the FieldWorks user-data dir, which
+ // doesn't exist in the container; point it at the shipped templates unless overridden in config.
+ services.Configure(config =>
+ {
+ if (config.TemplatesFolder == new FwDataBridgeConfig().TemplatesFolder)
+ config.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates");
+ });
services.RemoveAll(typeof(IMediaAdapter));
services.AddScoped();
services.AddScoped();
diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs
index b5aa38ef19..643d731b80 100644
--- a/backend/FwHeadless/Program.cs
+++ b/backend/FwHeadless/Program.cs
@@ -93,6 +93,7 @@
app.MapMediaFileRoutes();
app.MapMergeRoutes();
app.MapLexiconRoutes();
+app.MapProjectRoutes();
// DELETE endpoint to delete the FieldWorks repo/project (and nothing else)
app.MapDelete("/api/manage/repo/{projectId}", async (Guid projectId, ProjectDeletionService deletionService) =>
diff --git a/backend/FwHeadless/Routes/MergeRoutes.cs b/backend/FwHeadless/Routes/MergeRoutes.cs
index a54199a122..010a3ceec3 100644
--- a/backend/FwHeadless/Routes/MergeRoutes.cs
+++ b/backend/FwHeadless/Routes/MergeRoutes.cs
@@ -74,6 +74,10 @@ static async Task> ExecuteMergeRequest(
switch (syncHostedService.QueueJob(projectId))
{
+ case QueueJobResult.ProjectBeingCreated:
+ activity?.SetStatus(ActivityStatusCode.Ok, "Project is still being created");
+ return TypedResults.Problem("Project is still being created, try again once creation has finished",
+ statusCode: StatusCodes.Status409Conflict);
case QueueJobResult.SyncAlreadyRunning:
activity?.SetStatus(ActivityStatusCode.Ok, "Sync already running");
return TypedResults.Problem("A sync is already running for this project, try again once it has finished",
diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs
new file mode 100644
index 0000000000..c54f1e67e9
--- /dev/null
+++ b/backend/FwHeadless/Routes/ProjectRoutes.cs
@@ -0,0 +1,137 @@
+using System.Diagnostics;
+using FwHeadless.Services;
+using LexCore.Entities;
+using LexCore.Sync;
+using Microsoft.AspNetCore.Http.HttpResults;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless.Routes;
+
+public static class ProjectRoutes
+{
+ public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app)
+ {
+ var group = app.MapGroup("/api/project");
+ group.MapPost("/initFwDataProject", InitFwDataProject);
+ group.MapGet("/creation-status", GetCreationStatus);
+ group.MapGet("/await-creation-finished", AwaitCreationFinished);
+ return group;
+ }
+
+ // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the
+ // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the
+ // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates.
+ private static async Task> InitFwDataProject(
+ Guid projectId,
+ InitFwDataProjectInput input,
+ IProjectLookupService projectLookupService,
+ ProjectCreationService projectCreationService,
+ ILogger logger)
+ {
+ // Null-safe against a malformed body (System.Text.Json leaves an absent list null).
+ var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", input.WsVernacular)
+ ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", input.WsAnalysis)
+ ?? InitFwDataProjectInput.ValidateUiWritingSystem(input.WsUi);
+ if (wsError is not null)
+ return TypedResults.Problem(wsError, statusCode: StatusCodes.Status400BadRequest);
+
+ var projectCode = await projectLookupService.GetProjectCode(projectId);
+ if (projectCode is null)
+ {
+ logger.LogError("initFwDataProject request for non-existent project {ProjectId}", projectId);
+ return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound);
+ }
+
+ await projectCreationService.InitFwDataProject(
+ projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi);
+ return TypedResults.Ok();
+ }
+
+ // Instantaneous creation status, for a caller that wants to poll. Mirrors /api/merge/status.
+ private static async Task, NotFound>> GetCreationStatus(
+ Guid projectId,
+ SyncHostedService syncHostedService,
+ IProjectLookupService projectLookupService,
+ IOptions config)
+ {
+ using var activity = FwHeadlessActivitySource.Value.StartActivity();
+ activity?.SetTag("app.project_id", projectId);
+
+ var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config);
+ if (status is null)
+ {
+ activity?.SetStatus(ActivityStatusCode.Error, "Project not found");
+ return TypedResults.NotFound();
+ }
+ activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString());
+ return TypedResults.Ok(status);
+ }
+
+ // Long-poll until an in-flight creation finishes. Mirrors /api/merge/await-finished: giving up on the
+ // wait is reported as a status, not an error, because the creation itself keeps running either way.
+ private static async Task, NotFound>> AwaitCreationFinished(
+ Guid projectId,
+ SyncHostedService syncHostedService,
+ IProjectLookupService projectLookupService,
+ IOptions config,
+ CancellationToken cancellationToken)
+ {
+ using var activity = FwHeadlessActivitySource.Value.StartActivity();
+ activity?.SetTag("app.project_id", projectId);
+
+ try
+ {
+ var result = await syncHostedService.AwaitCreationFinished(projectId, cancellationToken);
+ if (result is not null)
+ {
+ activity?.SetStatus(ActivityStatusCode.Ok, result.Status.ToString());
+ return TypedResults.Ok(result);
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ // The caller's wait was cancelled; the creation was not (it takes no cancellation token).
+ // Report that distinctly so a caller can't read a timeout as a failed creation.
+ activity?.SetStatus(ActivityStatusCode.Unset, "Timed out awaiting creation");
+ return TypedResults.Ok(ProjectCreationStatus.TimedOutAwaitingCreation);
+ }
+
+ // Nothing in flight and nothing remembered, so fall back to what the project itself tells us.
+ var status = await DetermineCreationStatus(projectId, syncHostedService, projectLookupService, config);
+ if (status is null)
+ {
+ activity?.SetStatus(ActivityStatusCode.Error, "Project not found");
+ return TypedResults.NotFound();
+ }
+ activity?.SetStatus(ActivityStatusCode.Ok, status.Status.ToString());
+ return TypedResults.Ok(status);
+ }
+
+ ///
+ /// Creation status for a project, or null when LexBox has no such project. In-memory state wins
+ /// when we have it; otherwise the status is derived from durable state, so it survives a FwHeadless
+ /// restart and a result that has aged out of the cache.
+ ///
+ private static async Task DetermineCreationStatus(
+ Guid projectId,
+ SyncHostedService syncHostedService,
+ IProjectLookupService projectLookupService,
+ IOptions config)
+ {
+ if (syncHostedService.IsProjectBeingCreated(projectId)) return ProjectCreationStatus.Creating;
+ var recent = syncHostedService.TryGetRecentCreationResult(projectId);
+ if (recent is not null) return recent;
+
+ var projectCode = await projectLookupService.GetProjectCode(projectId);
+ if (projectCode is null) return null;
+
+ // A successful creation leaves the .fwdata on disk; a failed one deletes the project folder
+ // (ProjectCreationService.CleanupLocalProject). Note this says "project data is here", not "this
+ // process created it" -- a project cloned by an ordinary sync looks the same, which is the right
+ // answer for a caller asking whether the project is ready to use.
+ var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId);
+ return File.Exists(fwDataProject.FilePath)
+ ? ProjectCreationStatus.Created
+ : ProjectCreationStatus.NotCreated;
+ }
+}
diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs
index 4755ff1ac8..b1835ec7f5 100644
--- a/backend/FwHeadless/Services/ISendReceiveService.cs
+++ b/backend/FwHeadless/Services/ISendReceiveService.cs
@@ -10,4 +10,6 @@ public interface ISendReceiveService
Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode);
Task PendingCommitCountBothWays(FwDataProject project, string? projectCode);
Task CommitFile(string filePath, string commitMessage);
+ Task InitRepo(string folder);
+ Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "");
}
diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs
new file mode 100644
index 0000000000..2f455f8b57
--- /dev/null
+++ b/backend/FwHeadless/Services/ProjectCreationService.cs
@@ -0,0 +1,216 @@
+using System.Xml;
+using FwDataMiniLcmBridge;
+using FwDataMiniLcmBridge.LcmUtils;
+using LexCore.Exceptions;
+using LexCore.Sync;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless.Services;
+
+///
+/// Creates a brand-new FieldWorks project from the SIL.LCModel template and pushes it into the
+/// (empty) Mercurial repo that LexBox has already initialised for the project.
+///
+public class ProjectCreationService(
+ IOptions config,
+ ISendReceiveService srService,
+ IProjectLoader projectLoader,
+ SyncHostedService syncHostedService,
+ ILogger logger)
+{
+ private const string InitialCommitMessage = "Initial project creation";
+
+ // The one file FieldWorks/FLExBridge put in a brand-new FLEx repo's first commit: a minimal custom
+ // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge).
+ private const string CustomPropertiesFilename = "FLExProject.CustomProperties";
+
+ public async Task InitFwDataProject(
+ Guid projectId,
+ string projectCode,
+ IReadOnlyList vernacularWritingSystems,
+ IReadOnlyList analysisWritingSystems,
+ string uiWritingSystem)
+ {
+ if (vernacularWritingSystems.Count == 0)
+ throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems));
+ if (analysisWritingSystems.Count == 0)
+ throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems));
+ if (string.IsNullOrEmpty(uiWritingSystem))
+ throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem));
+
+ // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo.
+ if (!syncHostedService.TryStartProjectCreation(projectId))
+ throw new ProjectSyncInProgressException(projectId);
+ var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId);
+ // Published to status pollers in the finally below. Pre-set to a failure so an exit path that
+ // somehow reaches neither the success line nor the catch still releases pollers with an answer.
+ var outcome = ProjectCreationStatus.Failed("Project creation ended without reporting a result");
+ try
+ {
+ // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/
+ // FLExBridge create a repo, since Clone can't establish an empty remote and
+ // Language_Forge_Send_Receive refuses to make the *first* commit itself:
+ // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone).
+ // 2. Build fw.fwdata into that folder from the SIL.LCModel NewLangProj template,
+ // configured with all of the requested writing systems, but do not commit it yet.
+ // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch to be
+ // revision 0 -- Chorus expects revision 0 to be on the default branch, because it uses
+ // that rev 0 commit to track a repo's history and catch when someone would try to push
+ // an unrelated repo (which would otherwise create two Mercurial heads). This is the
+ // same initial commit, with the same file, that FieldWorks creates. The fwdata itself
+ // is NOT committed; Send/Receive splits it into the nested files Mercurial actually
+ // tracks (fwdata is excluded from tracking).
+ // 4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion,
+ // e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on
+ // that branch, so the split data must land there while the genesis stays on 'default'.
+ // 5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push.
+ await srService.InitRepo(fwDataProject.ProjectFolder);
+
+ BuildFwDataFile(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem);
+ FixLinkedFilesRootDirSeparator(fwDataProject);
+ AssertModelVersionMatches(fwDataProject);
+
+ var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject);
+ await srService.CommitFile(customPropertiesFile, InitialCommitMessage);
+
+ await srService.SwitchBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName);
+ var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage);
+ if (!pushResult.Success)
+ throw new SendReceiveException("Pushing the new project to the repo failed", pushResult);
+
+ logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId);
+ outcome = ProjectCreationStatus.Created;
+ }
+ catch (Exception ex)
+ {
+ outcome = ProjectCreationStatus.Failed(ex.Message);
+ CleanupLocalProject(fwDataProject);
+ throw;
+ }
+ finally
+ {
+ syncHostedService.EndProjectCreation(projectId, outcome);
+ }
+ }
+
+ private void BuildFwDataFile(
+ FwDataProject fwDataProject,
+ IReadOnlyList vernacularWritingSystems,
+ IReadOnlyList analysisWritingSystems,
+ string uiWs)
+ {
+ // NewProject copies the SIL.LCModel template, configured with all of the requested writing
+ // systems (the first of each list is the default of that type), and returns a loaded cache;
+ // dispose it right away so its file locks are released before the following hg add/commit.
+ using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs);
+ }
+
+ ///
+ /// FieldWorks (Windows) compares LangProject.LinkedFilesRootDir with an exact string, expecting the
+ /// Windows-style default %proj%\LinkedFiles. liblcm builds that path with Path.Combine, which on
+ /// this Linux host yields %proj%/LinkedFiles (forward slash), so FieldWorks warns on its first
+ /// Send/Receive. Rewrite just that one value to the backslash form.
+ ///
+ /// We deliberately do NOT parse the (potentially multi-MB) fwdata as XML -- that would be a needless
+ /// cost. The value always sits on the line immediately after the <LinkedFilesRootDir> open tag,
+ /// so stream the file once, fix the <Uni> on the line following that tag, and copy every other
+ /// line through untouched.
+ ///
+ private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject)
+ {
+ const string markerLinePrefix = "%proj%\LinkedFiles";
+
+ var sourcePath = fwDataProject.FilePath;
+ var tempPath = sourcePath + ".tmp";
+ var madeChanges = false;
+ using (var reader = new StreamReader(sourcePath))
+ using (var writer = new StreamWriter(tempPath))
+ {
+ var nextLineIsValue = false;
+ var done = false;
+ string? line;
+ while ((line = reader.ReadLine()) is not null)
+ {
+ if (nextLineIsValue)
+ {
+ var changedLine = line.Replace(forwardSlashValue, backslashValue);
+ if (string.Equals(changedLine, line, StringComparison.InvariantCulture))
+ {
+ // Exit early and don't overwrite since we didn't need this change
+ break;
+ }
+ madeChanges = true;
+ line = changedLine;
+ nextLineIsValue = false;
+ done = true;
+ }
+ else if (!done && line.StartsWith(markerLinePrefix, StringComparison.Ordinal))
+ {
+ nextLineIsValue = true;
+ }
+ writer.WriteLine(line);
+ }
+ }
+ if (madeChanges)
+ File.Move(tempPath, sourcePath, overwrite: true);
+ else
+ File.Delete(tempPath);
+ }
+
+ private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject)
+ {
+ var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename);
+ File.WriteAllText(path, """
+
+
+ """);
+ return path;
+ }
+
+ ///
+ /// The freshly-created project is stamped with the SIL.LCModel package's data-model version.
+ /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data
+ /// (see FwHeadless AGENTS.md), so fail before the push rather than after.
+ ///
+ private void AssertModelVersionMatches(FwDataProject fwDataProject)
+ {
+ var expected = config.Value.FdoDataModelVersion;
+ var actual = ReadModelVersion(fwDataProject.FilePath);
+ // Fail closed: an unreadable version ('actual' == null) is treated as a mismatch, since this is
+ // the last safety net before an irreversible push in a data-loss-risk area.
+ if (actual != expected)
+ {
+ throw new InvalidOperationException(
+ $"New project's data-model version '{actual ?? "(unreadable)"}' does not match FwHeadless " +
+ $"FdoDataModelVersion '{expected}'; refusing to push to avoid data corruption.");
+ }
+ }
+
+ private static string? ReadModelVersion(string fwDataFilePath)
+ {
+ using var reader = XmlReader.Create(fwDataFilePath, new XmlReaderSettings { IgnoreComments = true, IgnoreWhitespace = true });
+ // The .fwdata root element carries the FDO model version, e.g. .
+ return reader.MoveToContent() == XmlNodeType.Element ? reader.GetAttribute("version") : null;
+ }
+
+ private void CleanupLocalProject(FwDataProject fwDataProject)
+ {
+ // Mirror SyncWorker.SetupFwData's failure cleanup so a retry starts clean. The remote repo
+ // (created by LexBox) is torn down by the caller on the LexBox side.
+ try
+ {
+ if (Directory.Exists(fwDataProject.ProjectFolder))
+ Directory.Delete(fwDataProject.ProjectFolder, true);
+ if (Directory.Exists(fwDataProject.ProjectsPath) &&
+ !Directory.EnumerateFileSystemEntries(fwDataProject.ProjectsPath).Any())
+ Directory.Delete(fwDataProject.ProjectsPath);
+ }
+ catch (Exception ex)
+ {
+ logger.LogError(ex, "Failed to clean up local project folder after a failed creation: {ProjectFolder}",
+ fwDataProject.ProjectFolder);
+ }
+ }
+}
diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs
index 509644a88a..a5066a58fd 100644
--- a/backend/FwHeadless/Services/SendReceiveHelpers.cs
+++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs
@@ -166,6 +166,46 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg
await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)} {EscapeShellArg(filePath)}", fileDir, progress, allowedExitCodes: [0, 1]);
}
+ public static async Task InitRepo(string folder, IProgress? progress = null)
+ {
+ using var activity = FwHeadlessActivitySource.Value.StartActivity();
+ activity?.SetTag("app.folder", folder);
+ progress ??= new NullProgress();
+ Directory.CreateDirectory(folder);
+ // Use Chorus rather than a bare `hg init`: CreateOrUseExisting also wires up the custom hg
+ // extensions FwHeadless relies on (e.g. fixutf8) so later send/receive works correctly.
+ await Task.Run(() => HgRepository.CreateOrUseExisting(folder, progress));
+ }
+
+ public static async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "", IProgress? progress = null)
+ {
+ using var activity = FwHeadlessActivitySource.Value.StartActivity();
+ activity?.SetTag("app.folder", folder);
+ activity?.SetTag("app.branch", branchName);
+ progress ??= new NullProgress();
+ // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version
+ // (e.g. 7500002.7000072); the initial commit of a repo needs to be on the 'default' branch
+ // so that Chorus can use rev 0 to compare project origin, but then we need to switch branches
+ // to the numbered branch. The `hg branch` command records the branch that will be used for
+ // the next commit, but we need to add an empty commit in order to actually create the branch,
+ // otherwise LfMergeBridge will refuse to create a new branch (since in the Language Forge code
+ // that LfMergeBridge targets, creating a new branch would be an error). This results in a new
+ // project being created by this process having three initial commits, rather than the two
+ // initial commits that it would have if created by FieldWorks Classic, but that's an acceptable
+ // price to pay for such a simple workaround. If LfMergeBridge is updated to be okay with creating
+ // new branches on its own, then this method can be passed "createBranch: false" to skip running
+ // the `hg commit` step that creates that empty commit.
+ //
+ // Commit message defaults to "Switching to branch {branchName}", pass `commitMessage` param to change
+ await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress);
+ if (createBranch)
+ {
+ // Could run "hg log -r . --template {desc}" to reuse previous commit message if we wanted to
+ if (string.IsNullOrEmpty(commitMessage)) commitMessage = $"Switching to branch {branchName}";
+ await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress);
+ }
+ }
+
private static string EscapeShellArg(string arg)
{
var quote = """
diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs
index 735e90b8ae..0c410ba1f9 100644
--- a/backend/FwHeadless/Services/SendReceiveService.cs
+++ b/backend/FwHeadless/Services/SendReceiveService.cs
@@ -70,4 +70,14 @@ public async Task CommitFile(string filePath, string commitMessage)
{
await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress);
}
+
+ public async Task InitRepo(string folder)
+ {
+ await SendReceiveHelpers.InitRepo(folder, progress);
+ }
+
+ public async Task SwitchBranch(string folder, string branchName, bool createBranch = true, string commitMessage = "")
+ {
+ await SendReceiveHelpers.SwitchBranch(folder, branchName, createBranch, commitMessage, progress);
+ }
}
diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs
index 33bc51334a..e7663c109c 100644
--- a/backend/FwHeadless/Services/SyncHostedService.cs
+++ b/backend/FwHeadless/Services/SyncHostedService.cs
@@ -20,6 +20,8 @@ public enum QueueJobResult
{
Queued,
AlreadyQueued,
+ // A sync must not race a project that's still being created; the caller should retry later
+ ProjectBeingCreated,
// The running sync may already have read the repo before the caller's changes arrived; the caller should retry once it finishes
SyncAlreadyRunning,
QueueFull,
@@ -29,22 +31,41 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsToSync = Channel.CreateUnbounded();
- // Projects with a sync queued or running. A single atomic GetOrAdd decides whether a request queues a new sync,
- // so no lock is needed (this requires that SyncHostedService be a singleton).
- private readonly ConcurrentDictionary _syncJobs = new();
+ // Projects with a sync queued or running, or currently being created and populated with an initial .fwdata file.
+ // Creations are tracked here (not just in ProjectCreationService) so a sync job can't be queued for a project
+ // mid-creation and race on the same fw/ folder and repo. Keeping both in one dictionary means a single atomic
+ // TryAdd/GetOrAdd decides who gets the project, so no lock is needed (this requires that SyncHostedService be a singleton).
+ private readonly ConcurrentDictionary _projectJobs = new();
///
- /// The TaskCompletionSource carries the outcome to anyone long-polling AwaitSyncFinished.
- /// distinguishes a queued sync (which will pick up any changes made before it starts)
+ /// Exactly one of or is set. The TaskCompletionSource carries the
+ /// outcome to anyone long-polling AwaitSyncFinished or AwaitCreationFinished.
+ /// distinguishes a queued sync (which will pick up any changes made before it starts)
/// from a running one (which might not).
///
- private record SyncJob(TaskCompletionSource Result, bool Started = false)
+ private record ProjectJob(
+ TaskCompletionSource? Sync,
+ TaskCompletionSource? Creation,
+ bool SyncStarted = false)
{
// RunContinuationsAsynchronously so a long-polling awaiter's continuation can't run inline on
- // the thread that finished the sync.
- public static SyncJob New() => new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously));
+ // the thread that finished the sync or creation.
+ public static ProjectJob ForSync() =>
+ new(new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), null);
+
+ public static ProjectJob ForCreation() =>
+ new(null, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously));
}
+ private TaskCompletionSource? GetSyncJob(Guid projectId) =>
+ _projectJobs.TryGetValue(projectId, out var job) ? job.Sync : null;
+
+ private TaskCompletionSource? GetCreationJob(Guid projectId) =>
+ _projectJobs.TryGetValue(projectId, out var job) ? job.Creation : null;
+
+ // Only removes the entry if it's still the given job, so ending one kind of job can never clear the other kind
+ private bool TryRemoveJob(Guid projectId, ProjectJob job) => _projectJobs.TryRemove(KeyValuePair.Create(projectId, job));
+
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken))
@@ -73,38 +94,93 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
// Give clients a bit more time to poll the status
CacheRecentSyncResult(projectId, result);
- if (_syncJobs.TryRemove(projectId, out var job))
- job.Result.TrySetResult(result);
+ if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is { } tcs && TryRemoveJob(projectId, job))
+ tcs.TrySetResult(result);
}
}
- // QueueJob never modifies an existing entry, and only ExecuteAsync removes one, so nothing can race this update
+ // QueueJob never modifies an existing sync entry, and only ExecuteAsync removes one, so nothing can race this update
internal void MarkSyncStarted(Guid projectId)
{
- if (_syncJobs.TryGetValue(projectId, out var job))
- _syncJobs.TryUpdate(projectId, job with { Started = true }, job);
+ if (_projectJobs.TryGetValue(projectId, out var job) && job.Sync is not null)
+ _projectJobs.TryUpdate(projectId, job with { SyncStarted = true }, job);
}
public virtual bool IsJobQueuedOrRunning(Guid projectId)
{
- return _syncJobs.ContainsKey(projectId);
+ return _projectJobs.ContainsKey(projectId);
+ }
+
+ ///
+ /// Reserves a project for creation-and-population, blocking a concurrent creation or sync of the
+ /// same project. Returns false if a sync is already queued/running or another creation is in flight.
+ /// Pair with in a finally block.
+ ///
+ public bool TryStartProjectCreation(Guid projectId)
+ {
+ return _projectJobs.TryAdd(projectId, ProjectJob.ForCreation());
+ }
+
+ ///
+ /// Releases the reservation and publishes the outcome to pollers. Must be called on every exit path
+ /// from a creation, or an AwaitCreationFinished caller waits until its own token trips.
+ ///
+ public void EndProjectCreation(Guid projectId, ProjectCreationStatus result)
+ {
+ // Cache before removing the reservation, so a poller that sees "not in flight" always finds the
+ // result waiting for it rather than a gap.
+ CacheRecentCreationResult(projectId, result);
+ if (_projectJobs.TryGetValue(projectId, out var job) && job.Creation is { } tcs && TryRemoveJob(projectId, job))
+ tcs.TrySetResult(result);
}
+ public bool IsProjectBeingCreated(Guid projectId) => GetCreationJob(projectId) is not null;
+
public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken)
{
- if (_syncJobs.TryGetValue(projectId, out var job))
- return await job.Result.Task.WaitAsync(cancellationToken);
+ if (GetSyncJob(projectId) is { } tcs)
+ return await tcs.Task.WaitAsync(cancellationToken);
return TryGetRecentSyncResult(projectId);
}
+ ///
+ /// Waits for an in-flight creation to finish. Returns the recent result if one just finished, or
+ /// null when this FwHeadless has no record of a creation for the project (the caller should then
+ /// derive the status from durable state instead).
+ ///
+ public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken)
+ {
+ if (GetCreationJob(projectId) is { } tcs)
+ return await tcs.Task.WaitAsync(cancellationToken);
+ return TryGetRecentCreationResult(projectId);
+ }
+
+ public ProjectCreationStatus? TryGetRecentCreationResult(Guid projectId)
+ {
+ return memoryCache.Get($"CreationResult|{projectId}");
+ }
+
+ private void CacheRecentCreationResult(Guid projectId, ProjectCreationStatus result)
+ {
+ // Longer than the sync equivalent's 30s: a caller whose own request timed out partway through a
+ // creation should still be able to come back and learn how it ended.
+ memoryCache.Set($"CreationResult|{projectId}", result, TimeSpan.FromMinutes(5));
+ }
+
public QueueJobResult QueueJob(Guid projectId)
{
- var newJob = SyncJob.New();
- var job = _syncJobs.GetOrAdd(projectId, newJob);
+ var newJob = ProjectJob.ForSync();
+ var job = _projectJobs.GetOrAdd(projectId, newJob);
+ //don't sync a project while it's still being created and populated with initial .fwdata (would race on the repo)
+ if (job.Creation is not null)
+ {
+ logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId);
+ return QueueJobResult.ProjectBeingCreated;
+ }
//will only queue job if it's not already queued
if (!ReferenceEquals(job, newJob))
{
- if (job.Started)
+ if (job.SyncStarted)
{
logger.LogInformation("Project {ProjectId} is already syncing, not queueing another sync job", projectId);
return QueueJobResult.SyncAlreadyRunning;
@@ -116,7 +192,7 @@ public QueueJobResult QueueJob(Guid projectId)
if (!_projectsToSync.Writer.TryWrite(projectId))
{
logger.LogError("Failed to queue sync job for project {ProjectId}, the channel is full", projectId);
- _syncJobs.TryRemove(KeyValuePair.Create(projectId, newJob));
+ TryRemoveJob(projectId, newJob);
return QueueJobResult.QueueFull;
}
diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs
index db3b5b83f6..6f248f279f 100644
--- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs
+++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs
@@ -3,6 +3,7 @@
using System.Runtime.InteropServices;
using Microsoft.Extensions.Options;
using SIL.LCModel;
+using SIL.LCModel.Infrastructure;
using SIL.LCModel.Core.WritingSystems;
using SIL.LCModel.Utils;
using SIL.WritingSystems;
@@ -96,28 +97,99 @@ public virtual LcmCache NewProject(FwDataProject project,
var progress = new LcmThreadedProgress();
var analysisDefinitions = analysisWss.Select(CreateWritingSystemDefinition).ToList();
var vernacularDefinitions = vernacularWss.Select(CreateWritingSystemDefinition).ToList();
+ // A liblcm bug: when no anthropology list file is passed (FLEx passes one unless the user picks an
+ // empty list), CreateNewLangProj names the empty list in the default analysis WS, which it looks up
+ // with GetWsFromStr(IcuLocale). GetWsFromStr expects a language tag, so this only works when the
+ // two match: "pt-BR" is "pt_BR" in ICU, the lookup returns 0, and creation throws with the project
+ // already on disk. Remove this workaround once https://github.com/sillsdev/liblcm/issues/416 is fixed and released.
+ // Until then hand liblcm "en" as the default
+ // and pass every requested analysis WS as additional; SetRequestedWritingSystems fixes the default.
+ var substituteDefaultAnalysis = !IcuLocaleMatchesTag(analysisDefinitions[0]);
+ var defaultAnalysis = substituteDefaultAnalysis ? CreateWritingSystemDefinition("en") : analysisDefinitions[0];
NewProject(progress,
project.Name,
lcmDirectories,
progress.SynchronizeInvoke,
- analysisDefinitions[0],
+ defaultAnalysis,
vernacularDefinitions[0],
uiWs,
- AdditionalWritingSystems(analysisDefinitions),
- AdditionalWritingSystems(vernacularDefinitions));
- return LoadCache(project);
+ AdditionalWritingSystems(analysisDefinitions, defaultAnalysis),
+ AdditionalWritingSystems(vernacularDefinitions, vernacularDefinitions[0]));
+ var cache = LoadCache(project);
+ try
+ {
+ SetRequestedWritingSystems(cache, analysisWss, vernacularWss, substituteDefaultAnalysis);
+ }
+ catch
+ {
+ cache.Dispose();
+ throw;
+ }
+ return cache;
}
private static CoreWritingSystemDefinition CreateWritingSystemDefinition(string ws) => new(ws) { Id = ws };
- // The default writing system (the first of the list) is passed to CreateNewLangProj on its own, so
- // the "additional" set is the whole list as a hash set with that default removed. The new project
- // ends up with them in list order (NewProjectWritingSystemsTests checks this).
- private static HashSet AdditionalWritingSystems(List definitions)
+ private static bool IcuLocaleMatchesTag(CoreWritingSystemDefinition definition) =>
+ string.Equals(definition.IcuLocale, definition.Id, StringComparison.OrdinalIgnoreCase);
+
+ // Everything except the default, which CreateNewLangProj takes on its own
+ private static HashSet AdditionalWritingSystems(
+ List definitions,
+ CoreWritingSystemDefinition defaultDefinition) =>
+ definitions.Where(d => !string.Equals(d.Id, defaultDefinition.Id, StringComparison.OrdinalIgnoreCase)).ToHashSet();
+
+ ///
+ /// Makes the current analysis and vernacular lists exactly the requested writing systems, in the
+ /// requested order. CreateNewLangProj doesn't on its own: it adds "en" as a current analysis WS
+ /// whenever another one is the default, appends the "additional" sets without checking for one that's
+ /// already there (so "en" listed later, or "fr" plus "fr-Latn", end up twice), and takes them as
+ /// unordered sets. Writing systems it added that weren't asked for stay in the full lists, unselected.
+ ///
+ private static void SetRequestedWritingSystems(LcmCache cache,
+ IReadOnlyList analysisWss,
+ IReadOnlyList vernacularWss,
+ bool substitutedDefaultAnalysis)
+ {
+ var container = cache.ServiceLocator.WritingSystems;
+ var wsManager = cache.ServiceLocator.WritingSystemManager;
+ // GetOrSet resolves a tag to the WS liblcm already created for it, also when liblcm stored it under
+ // an equivalent tag (e.g. "fr" for "fr-Latn")
+ List Resolve(IReadOnlyList tags) => tags.Select(tag =>
+ {
+ wsManager.GetOrSet(tag, out var ws);
+ return ws;
+ }).Distinct().ToList();
+
+ NonUndoableUnitOfWorkHelper.Do(cache.ActionHandlerAccessor, () =>
+ {
+ var analysis = Resolve(analysisWss);
+ var vernacular = Resolve(vernacularWss);
+ // liblcm doesn't keep the full lists in the same order as the current lists, so set both
+ SetList(container.AnalysisWritingSystems, [.. analysis, .. container.AnalysisWritingSystems.Except(analysis)]);
+ SetList(container.VernacularWritingSystems, [.. vernacular, .. container.VernacularWritingSystems.Except(vernacular)]);
+ SetList(container.CurrentAnalysisWritingSystems, analysis);
+ SetList(container.CurrentVernacularWritingSystems, vernacular);
+ if (substitutedDefaultAnalysis) MoveReversalIndexToDefaultAnalysis(cache, analysis[0]);
+ });
+ cache.ActionHandlerAccessor.Commit();
+ }
+
+ private static void SetList(ICollection list, List wanted)
+ {
+ if (list.SequenceEqual(wanted)) return;
+ list.Clear();
+ foreach (var ws in wanted) list.Add(ws);
+ }
+
+ // CreateNewLangProj creates a reversal index for the default analysis WS, which was "en" only because
+ // the requested default couldn't be passed in. Give the requested default the index instead.
+ private static void MoveReversalIndexToDefaultAnalysis(LcmCache cache, CoreWritingSystemDefinition defaultAnalysis)
{
- var additional = definitions.ToHashSet();
- additional.Remove(definitions[0]);
- return additional;
+ cache.ServiceLocator.GetInstance().FindOrCreateIndexForWs(defaultAnalysis.Handle);
+ if (cache.ServiceLocator.WritingSystems.CurrentAnalysisWritingSystems.Any(ws => ws.Id == "en")) return;
+ var englishIndex = cache.LangProject.LexDbOA.ReversalIndexesOC.FirstOrDefault(ri => ri.WritingSystem == "en");
+ if (englishIndex is not null) cache.LangProject.LexDbOA.ReversalIndexesOC.Remove(englishIndex);
}
private static void NewProject(IThreadedProgress progress,
diff --git a/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs
index aaa4e80d39..4ee17a0b19 100644
--- a/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs
+++ b/backend/FwLite/FwLiteProjectSync.Tests/NewProjectWritingSystemsTests.cs
@@ -37,24 +37,47 @@ public async Task DisposeAsync()
await _rootServiceProvider.DisposeAsync();
}
- [Fact]
- public async Task NewProjectAddsEveryRequestedWritingSystemToTheCorrectListInOrder()
+ [Theory]
+ // Distinct codes per type so a swapped analysis/vernacular argument would fail, and several additional
+ // ones per type because CreateNewLangProj takes those as an unordered set
+ [InlineData("en,de,pt,ru", "fr,es,it,nl", "en,de,pt,ru", "fr,es,it,nl")]
+ // liblcm adds "en" as a current analysis WS whenever another one is the default
+ [InlineData("de,pt", "fr", "de,pt", "fr")]
+ // ...and appends a later "en" a second time
+ [InlineData("de,en,pt", "fr", "de,en,pt", "fr")]
+ // liblcm can't take a default analysis WS whose ICU locale differs from its tag ("pt_BR")
+ [InlineData("pt-BR,en", "fr", "pt-BR,en", "fr")]
+ [InlineData("qaa-x-foo", "fr", "qaa-x-foo", "fr")]
+ // Tags liblcm treats as the same WS end up once
+ [InlineData("en", "fr,fr-Latn", "en", "fr")]
+ public async Task NewProjectHasExactlyTheRequestedWritingSystemsInOrder(
+ string analysis, string vernacular, string expectedAnalysis, string expectedVernacular)
{
var fwDataBridgeConfig = Services.GetRequiredService>().Value;
// The SIL.LCModel template ships next to the test assembly; point liblcm at it (this project,
// unlike FwDataMiniLcmBridge.Tests, doesn't otherwise configure TemplatesFolder).
fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates");
var fwDataProject = new FwDataProject("multi-ws-source", fwDataBridgeConfig.ProjectsFolder);
- // First of each list is the default; the rest go through CreateNewLangProj's additional-WS sets.
- // Distinct codes per type so a swapped analysis/vernacular argument would fail this test, and
- // several additional ones per type because CreateNewLangProj takes those as an unordered set.
- string[] analysis = ["en", "de", "pt", "ru"];
- string[] vernacular = ["fr", "es", "it", "nl"];
- Services.GetRequiredService().NewProject(fwDataProject, analysis, vernacular, uiWs: "en").Dispose();
+ Services.GetRequiredService()
+ .NewProject(fwDataProject, analysis.Split(','), vernacular.Split(','), uiWs: "en").Dispose();
+ // Reopened from disk, so this also checks the fixed-up lists were saved
using var api = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false);
var writingSystems = await api.GetWritingSystems();
- writingSystems.Analysis.Select(ws => ws.WsId.Code).Should().Equal(analysis);
- writingSystems.Vernacular.Select(ws => ws.WsId.Code).Should().Equal(vernacular);
+ writingSystems.Analysis.Select(ws => ws.WsId.Code).Should().Equal(expectedAnalysis.Split(','));
+ writingSystems.Vernacular.Select(ws => ws.WsId.Code).Should().Equal(expectedVernacular.Split(','));
+ }
+
+ [Fact]
+ public void ADefaultAnalysisWritingSystemLiblcmCantTakeStillGetsTheReversalIndex()
+ {
+ var fwDataBridgeConfig = Services.GetRequiredService>().Value;
+ fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates");
+ var fwDataProject = new FwDataProject("reversal-source", fwDataBridgeConfig.ProjectsFolder);
+
+ // "en" is passed to liblcm as the default instead, so it would otherwise get the reversal index
+ using var cache = Services.GetRequiredService().NewProject(fwDataProject, ["pt-BR"], ["fr"], uiWs: "en");
+
+ cache.LangProject.LexDbOA.ReversalIndexesOC.Select(ri => ri.WritingSystem).Should().Equal("pt-BR");
}
}
diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs
index b05e8b058c..09ff8e1d43 100644
--- a/backend/LexBoxApi/Controllers/ProjectController.cs
+++ b/backend/LexBoxApi/Controllers/ProjectController.cs
@@ -1,13 +1,16 @@
using LexBoxApi.Auth.Attributes;
using LexBoxApi.Controllers.ActionResults;
using LexBoxApi.Jobs;
+using LexBoxApi.Models.Project;
using LexBoxApi.Services;
using LexCore.Entities;
using LexCore.Exceptions;
using LexCore.ServiceInterfaces;
+using LexCore.Sync;
using LexData;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Caching.Memory;
using Quartz;
using Path = System.IO.Path;
@@ -20,9 +23,157 @@ public class ProjectController(
IHgService hgService,
LexBoxDbContext lexBoxDbContext,
IPermissionService permissionService,
- ISchedulerFactory scheduler)
+ ISchedulerFactory scheduler,
+ ILogger logger)
: ControllerBase
{
+ ///
+ /// Admin-only: create a new FLEx project whose repo is populated with a template .fwdata
+ /// (from the SIL.LCModel package) configured for the requested writing systems.
+ ///
+ /// At some future date this API may be opened up to trusted users, but the projectOrigin
+ /// parameter will remain admin-only.
+ ///
+ /// Building and pushing the project takes too long to hold a request open for, so this creates the
+ /// project row and empty repo, queues the rest as a background job, and returns 202 Accepted with the
+ /// new project's ID. Poll to learn when it's ready.
+ ///
+ /// Project code for the new project.
+ /// Display name for the new project.
+ /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple.
+ /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple.
+ /// Optional user interface writing system id; defaults to "en" if not given.
+ /// Optional to record as the project's origin; admin-only.
+ [HttpPost("initFwDataProject")]
+ [AdminRequired]
+ [ProducesResponseType(StatusCodes.Status202Accepted)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ [ProducesResponseType(StatusCodes.Status409Conflict)]
+ [ProducesResponseType(StatusCodes.Status500InternalServerError)]
+ public async Task> InitFwDataProject(
+ string code,
+ string name,
+ [FromQuery] string[] wsVernacular,
+ [FromQuery] string[]? wsAnalysis = null,
+ string wsUi = "en",
+ string? projectOrigin = null)
+ {
+ // Validate before anything is created, so a bad request never leaves a project to tear down.
+ var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis);
+ var wsError = InitFwDataProjectInput.ValidateWritingSystemList("vernacular", wsVernacular)
+ ?? InitFwDataProjectInput.ValidateWritingSystemList("analysis", wsAnalysisOrDefault)
+ ?? InitFwDataProjectInput.ValidateUiWritingSystem(wsUi);
+ if (wsError is not null)
+ return Problem(wsError, statusCode: StatusCodes.Status400BadRequest);
+ if (code is null || code.Length < 4 || !ProjectCode.IsValidCode(code))
+ return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest);
+ if (string.IsNullOrWhiteSpace(name))
+ return Problem("A project name is required", statusCode: StatusCodes.Status400BadRequest);
+
+ ProjectMigrationStatus? origin = null;
+ if (!string.IsNullOrEmpty(projectOrigin))
+ {
+ // Only admins may say where a project came from. Currently redundant with [AdminRequired],
+ // but this endpoint is expected to open up to non-admins later; this check must stay.
+ permissionService.AssertIsAdmin();
+ if (!TryParseProjectOrigin(projectOrigin, out var parsedOrigin))
+ return Problem($"Invalid project origin '{projectOrigin}'", statusCode: StatusCodes.Status400BadRequest);
+ origin = parsedOrigin;
+ }
+ if (await projectService.ProjectExists(code))
+ return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict);
+
+ Guid projectId;
+ try
+ {
+ projectId = await projectService.CreateProject(new CreateProjectInput(
+ Id: null,
+ Name: name,
+ Description: string.Empty,
+ Code: code,
+ Type: ProjectType.FLEx,
+ RetentionPolicy: RetentionPolicy.Verified,
+ IsConfidential: false,
+ ProjectManagerId: null,
+ OrgId: null),
+ projectOrigin: origin,
+ // In the same insert as the row itself, so the project is never visible without it
+ fwDataInitStatus: FwDataInitStatus.InProgress);
+ }
+ catch (DbUpdateException e) when (DbError.CreateErrorFrom(e).Code == DbErrorCode.DuplicateProjectCode)
+ {
+ // ProjectExists above is only a fast path: a concurrent request can take the code after that check,
+ // and the unique index on Project.Code is what actually catches it. Nothing was created by this request.
+ return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict);
+ }
+
+ // The project row (marked as being initialised) and its empty repo now exist. Populating the repo
+ // happens in the job, which also marks the project ready or rolls it back.
+ try
+ {
+ await InitFwDataProjectJob.Queue(scheduler, projectId,
+ new InitFwDataProjectInput(wsVernacular, wsAnalysisOrDefault, wsUi));
+ }
+ catch (Exception ex)
+ {
+ // Nothing has been sent to FwHeadless yet, so tearing the project down is unambiguous
+ logger.LogError(ex, "Failed to queue initialising project {ProjectId} ({Code}), rolling it back", projectId, code);
+ await CleanupFailedCreation(projectId, code);
+ throw;
+ }
+ return AcceptedAtAction(nameof(FwDataProjectCreationStatus), new { projectId }, projectId);
+ }
+
+ ///
+ /// Admin-only: how the creation started by is going.
+ /// Creating while the job is still running; Created once the project exists and isn't being
+ /// initialised (so also for any project that wasn't created that way). A failed creation is rolled
+ /// back, deleting the project, after which this reports CreationFailed with the reason for an hour,
+ /// as long as LexBox hasn't restarted, and 404 after that.
+ ///
+ [HttpGet("fwDataProjectCreationStatus/{projectId}")]
+ [AdminRequired]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status404NotFound)]
+ public async Task> FwDataProjectCreationStatus(
+ Guid projectId,
+ [FromServices] IMemoryCache memoryCache)
+ {
+ var initStatus = await lexBoxDbContext.Projects
+ .Where(p => p.Id == projectId)
+ .Select(p => (FwDataInitStatus?)p.FwDataInitStatus)
+ .SingleOrDefaultAsync();
+ if (initStatus is not null)
+ return initStatus == FwDataInitStatus.InProgress ? ProjectCreationStatus.Creating : ProjectCreationStatus.Created;
+ var failure = InitFwDataProjectJob.TryGetRecentFailure(memoryCache, projectId);
+ return failure is null ? NotFound() : ProjectCreationStatus.Failed(failure);
+ }
+
+ ///
+ /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists
+ /// ("Migrated,Migrating"), so the parsed value must round-trip to the name that was passed in.
+ ///
+ public static bool TryParseProjectOrigin(string projectOrigin, out ProjectMigrationStatus origin) =>
+ Enum.TryParse(projectOrigin, ignoreCase: true, out origin)
+ && Enum.IsDefined(origin)
+ && string.Equals(origin.ToString(), projectOrigin, StringComparison.OrdinalIgnoreCase);
+
+ /// Analysis writing systems to use, defaulting to English when none are supplied.
+ public static string[] AnalysisWritingSystemsOrDefault(string[]? wsAnalysis) =>
+ wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"];
+
+ private async Task CleanupFailedCreation(Guid projectId, string code)
+ {
+ try
+ {
+ await projectService.CleanupFailedProjectCreation(projectId, code);
+ }
+ catch (Exception ex)
+ {
+ logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}); it may need manual cleanup", projectId, code);
+ }
+ }
+
[HttpPost("refreshProjectLastChanged")]
public async Task RefreshProjectLastChanged(string projectCode)
{
diff --git a/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs
new file mode 100644
index 0000000000..4b4d302b93
--- /dev/null
+++ b/backend/LexBoxApi/Jobs/InitFwDataProjectJob.cs
@@ -0,0 +1,220 @@
+using System.Net;
+using System.Text.Json;
+using LexBoxApi.Services;
+using LexCore.Entities;
+using LexCore.ServiceInterfaces;
+using LexCore.Sync;
+using LexData;
+using Microsoft.Extensions.Caching.Memory;
+using Quartz;
+
+namespace LexBoxApi.Jobs;
+
+///
+/// Has FwHeadless populate the empty repo of a project created by ProjectController.InitFwDataProject,
+/// then marks the project ready, or rolls it back if FwHeadless reports that the creation failed.
+///
+///
+/// Quartz keeps this job in the database and reruns it if LexBox stops partway through (RequestRecovery),
+/// so every run must be safe to repeat. Each run therefore starts by finding out how far an earlier run
+/// got, using LexBox's own repo as the source of truth. When the outcome is unknown (FwHeadless is
+/// unreachable, busy, or still running a creation), the job schedules another attempt rather than guess.
+///
+public class InitFwDataProjectJob(
+ LexBoxDbContext dbContext,
+ ProjectService projectService,
+ FwHeadlessClient fwHeadlessClient,
+ IHgService hgService,
+ ISchedulerFactory schedulerFactory,
+ IMemoryCache memoryCache,
+ ILogger logger) : LexJob
+{
+ public const int MaxAttempts = 10;
+ public static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1);
+
+ public static JobKey Key { get; } = new("InitFwDataProjectJob", "ProjectCreation");
+
+ public static async Task Queue(ISchedulerFactory schedulerFactory,
+ Guid projectId,
+ InitFwDataProjectInput input,
+ int attempt = 1,
+ TimeSpan? delay = null,
+ CancellationToken cancellationToken = default)
+ {
+ await QueueJob(schedulerFactory,
+ Key,
+ new JobDataMap
+ {
+ { nameof(ProjectId), projectId.ToString() },
+ // The job store only takes strings, so the writing systems travel as JSON
+ { nameof(Input), JsonSerializer.Serialize(input) },
+ { nameof(Attempt), attempt.ToString() },
+ },
+ delay,
+ cancellationToken);
+ }
+
+ public string? ProjectId { get; set; }
+ public string? Input { get; set; }
+ public string? Attempt { get; set; }
+
+ protected override async Task ExecuteJob(IJobExecutionContext context)
+ {
+ ArgumentException.ThrowIfNullOrEmpty(ProjectId);
+ ArgumentException.ThrowIfNullOrEmpty(Input);
+ ArgumentException.ThrowIfNullOrEmpty(Attempt);
+ if (!Guid.TryParse(ProjectId, out var projectId)) throw new ArgumentException($"Invalid project ID {ProjectId}", nameof(ProjectId));
+ if (!int.TryParse(Attempt, out var attempt)) throw new ArgumentException($"Invalid number {Attempt}", nameof(Attempt));
+ var input = JsonSerializer.Deserialize(Input)
+ ?? throw new ArgumentException("Missing creation input", nameof(Input));
+
+ var project = await dbContext.Projects.FindAsync(projectId);
+ if (project is null)
+ {
+ logger.LogInformation("Project {ProjectId} no longer exists, nothing to initialise", projectId);
+ return;
+ }
+ if (project.FwDataInitStatus != FwDataInitStatus.InProgress)
+ {
+ // An earlier run finished the job and this is a duplicate (e.g. a recovered run)
+ logger.LogInformation("Project {ProjectId} ({Code}) has already been initialised", projectId, project.Code);
+ return;
+ }
+
+ var outcome = await AttemptInit(fwHeadlessClient, hgService, logger, projectId, project.Code, input);
+ switch (outcome.Result)
+ {
+ case InitAttemptResult.Created:
+ try
+ {
+ await projectService.FinishFwDataInit(projectId);
+ logger.LogInformation("Initialised project {ProjectId} ({Code}) from the template", projectId, project.Code);
+ return;
+ }
+ catch (Exception ex)
+ {
+ // The repo is populated, so a later attempt sees Created again and retries marking it ready
+ logger.LogError(ex, "Failed to mark project {ProjectId} ({Code}) as initialised", projectId, project.Code);
+ }
+ break;
+
+ case InitAttemptResult.Failed:
+ logger.LogError("Creating project {ProjectId} ({Code}) from the template failed, rolling it back: {Error}",
+ projectId, project.Code, outcome.Detail);
+ // Before the rollback, so a poller that finds the project gone always finds the reason too
+ RecordFailure(memoryCache, projectId, outcome.Detail);
+ try
+ {
+ await projectService.CleanupFailedProjectCreation(projectId, project.Code);
+ return;
+ }
+ catch (Exception ex)
+ {
+ // The project is still marked as being initialised, so a later attempt starts over: it
+ // either creates the project after all, or fails again and retries the rollback
+ logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code})", projectId, project.Code);
+ }
+ break;
+
+ case InitAttemptResult.RetryLater:
+ logger.LogWarning("Couldn't tell whether project {ProjectId} ({Code}) was created (attempt {Attempt} of {MaxAttempts}): {Reason}",
+ projectId, project.Code, attempt, MaxAttempts, outcome.Detail);
+ break;
+ }
+
+ if (attempt >= MaxAttempts)
+ {
+ logger.LogError(
+ "Giving up on initialising project {ProjectId} ({Code}) after {Attempts} attempts; it is still marked as being initialised and needs manual cleanup",
+ projectId, project.Code, attempt);
+ return;
+ }
+ await Queue(schedulerFactory, projectId, input, attempt + 1, RetryDelay);
+ }
+
+ ///
+ /// One attempt at getting the project populated, picking up wherever an earlier attempt got to.
+ /// Never throws: anything unexpected means the outcome is unknown, so it's reported as RetryLater.
+ ///
+ ///
+ /// None of these calls take a cancellation token, for the same reason ProjectController used not to
+ /// pass one: FwHeadless finishes a creation whatever happens to our side of the connection, so
+ /// abandoning the wait would only stop us learning the outcome. If LexBox shuts down mid-attempt,
+ /// Quartz recovers the job and the next run picks the creation up again.
+ ///
+ public static async Task AttemptInit(FwHeadlessClient fwHeadlessClient,
+ IHgService hgService,
+ ILogger logger,
+ Guid projectId,
+ string code,
+ InitFwDataProjectInput input)
+ {
+ try
+ {
+ // FwHeadless's push is the last step of a creation, so a non-empty repo means an earlier attempt
+ // succeeded, even if we never heard back. Trust this over FwHeadless's own status, which says
+ // Created as soon as the .fwdata exists on its disk, even if FwHeadless died before pushing it.
+ if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created;
+
+ var status = await fwHeadlessClient.CreationStatus(projectId);
+ if (status?.Status == ProjectCreationStatusEnum.Creating)
+ {
+ // An earlier attempt is still running in FwHeadless (e.g. this job was interrupted and
+ // recovered); starting another would be refused, so wait for that one instead
+ status = await fwHeadlessClient.AwaitCreationFinished(projectId);
+ if (await RepoHasCommits(hgService, code)) return InitAttemptOutcome.Created;
+ return status?.Status == ProjectCreationStatusEnum.CreationFailed
+ ? InitAttemptOutcome.Failed(status.Error ?? "FwHeadless reported that the creation failed")
+ : InitAttemptOutcome.RetryLater($"Waiting for FwHeadless's creation ended with {status?.Status.ToString() ?? "no status"}");
+ }
+
+ // Clear anything an interrupted attempt left in FwHeadless's local project folder, which would
+ // otherwise break the new attempt. Throws ProjectSyncInProgressException if FwHeadless is busy
+ // with this project.
+ await fwHeadlessClient.DeleteProject(projectId);
+ var (statusCode, error) = await fwHeadlessClient.InitFwDataProject(projectId, input.WsVernacular, input.WsAnalysis, input.WsUi);
+ if (error is null) return InitAttemptOutcome.Created;
+ // FwHeadless refused to start because something else holds the project, so nothing was built
+ if (statusCode == HttpStatusCode.Conflict) return InitAttemptOutcome.RetryLater(error);
+ // FwHeadless answered with a failure, so the creation isn't running and FwHeadless has already
+ // cleaned up its own local state
+ return InitAttemptOutcome.Failed(error);
+ }
+ catch (Exception ex)
+ {
+ // Most likely FwHeadless is unreachable (e.g. restarting) or our request timed out. FwHeadless
+ // may still be creating the project, so only a later attempt can tell how this ended.
+ logger.LogWarning(ex, "Attempt to initialise project {ProjectId} ({Code}) ended without an answer", projectId, code);
+ return InitAttemptOutcome.RetryLater(ex.Message);
+ }
+ }
+
+ private static async Task RepoHasCommits(IHgService hgService, string code) =>
+ await hgService.GetTipHash(code) != HgService.AllZeroHash;
+
+ // In memory only: once the rollback has deleted the project row, this is all that's left to tell a
+ // poller why. It doesn't survive a LexBox restart, after which the poller sees "not found".
+ private static string FailureCacheKey(Guid projectId) => $"InitFwDataProjectFailure|{projectId}";
+
+ private static void RecordFailure(IMemoryCache memoryCache, Guid projectId, string? error) =>
+ memoryCache.Set(FailureCacheKey(projectId), error ?? "Creating the project from the template failed", TimeSpan.FromHours(1));
+
+ public static string? TryGetRecentFailure(IMemoryCache memoryCache, Guid projectId) =>
+ memoryCache.Get(FailureCacheKey(projectId));
+}
+
+public enum InitAttemptResult
+{
+ Created,
+ // FwHeadless definitely isn't creating the project and didn't create it
+ Failed,
+ // The outcome is unknown, e.g. FwHeadless was unreachable or is still busy
+ RetryLater,
+}
+
+public record InitAttemptOutcome(InitAttemptResult Result, string? Detail = null)
+{
+ public static InitAttemptOutcome Created { get; } = new(InitAttemptResult.Created);
+ public static InitAttemptOutcome Failed(string error) => new(InitAttemptResult.Failed, error);
+ public static InitAttemptOutcome RetryLater(string reason) => new(InitAttemptResult.RetryLater, reason);
+}
diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs
index 209e2fe849..2db9c3b5c6 100644
--- a/backend/LexBoxApi/LexBoxKernel.cs
+++ b/backend/LexBoxApi/LexBoxKernel.cs
@@ -69,7 +69,13 @@ public static void AddLexBoxApi(this IServiceCollection services,
config.Enabled = false;
});
services.AddServiceDiscovery();
- services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless"))
+ services.AddHttpClient(client =>
+ {
+ client.BaseAddress = new ("http://fwHeadless");
+ // initFwDataProject runs inline (hg init + build the project from the template + push),
+ // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom.
+ client.Timeout = TimeSpan.FromMinutes(5);
+ })
.AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config
services.AddHttpContextAccessor();
services.AddMemoryCache();
diff --git a/backend/LexBoxApi/ScheduledTasksKernel.cs b/backend/LexBoxApi/ScheduledTasksKernel.cs
index 5293d20502..5345071d68 100644
--- a/backend/LexBoxApi/ScheduledTasksKernel.cs
+++ b/backend/LexBoxApi/ScheduledTasksKernel.cs
@@ -37,6 +37,9 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur
q.AddJob(DeleteTempDirectoryJob.Key, j => j.StoreDurably());
q.AddJob(UpdateProjectMetadataJob.Key, j => j.StoreDurably());
q.AddJob(RetryEmailJob.Key, j => j.StoreDurably());
+ // RequestRecovery: rerun a creation that was interrupted by LexBox stopping, rather than leave the
+ // project marked as being initialised forever. The job is written to be safe to rerun.
+ q.AddJob(InitFwDataProjectJob.Key, j => j.StoreDurably().RequestRecovery());
q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key)
.WithIdentity("WeeklyCleanupTrigger")
//every sunday at 2am
diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs
index 4b2d8d7cde..857757fe44 100644
--- a/backend/LexBoxApi/Services/FwHeadlessClient.cs
+++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs
@@ -1,5 +1,7 @@
using System.Net;
+using System.Net.Http.Json;
using System.Text.Json;
+using LexCore.Entities;
using LexCore.Exceptions;
using LexCore.Sync;
@@ -209,4 +211,94 @@ private static async Task FailedResponse(HttpResponseMessag
}
return body;
}
+
+ ///
+ /// Initializes a newly-created (empty) project's repo with a new .fwdata configured for the
+ /// given writing systems. Runs inline (a new empty project is small). Returns a null Error on
+ /// success; on failure returns the FwHeadless status code and error body so the caller can
+ /// propagate the right status (e.g. surface a 400 rather than flattening it to 500).
+ ///
+ public async Task<(HttpStatusCode StatusCode, string? Error)> InitFwDataProject(Guid projectId,
+ IReadOnlyList wsVernacular,
+ IReadOnlyList wsAnalysis,
+ string wsUi,
+ CancellationToken cancellationToken = default)
+ {
+ var input = new InitFwDataProjectInput(wsVernacular, wsAnalysis, wsUi);
+ var response = await httpClient.PostAsJsonAsync($"/api/project/initFwDataProject?projectId={projectId}",
+ input,
+ cancellationToken);
+ if (response.IsSuccessStatusCode) return (response.StatusCode, null);
+ var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
+ logger.LogError("Failed to init fwdata project: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}",
+ response.StatusCode,
+ response.ReasonPhrase,
+ projectId,
+ responseBody);
+ // Never null on failure: AttemptInit reads a null Error as success, and a failure body can be empty
+ return (response.StatusCode, string.IsNullOrWhiteSpace(responseBody)
+ ? $"FwHeadless returned HTTP {(int)response.StatusCode}"
+ : responseBody);
+ }
+
+ ///
+ /// Current creation status for a project, or null if FwHeadless doesn't know the project or the
+ /// call failed (both are logged). Cheap and safe to poll.
+ ///
+ ///
+ /// Unlike , this one does take a cancellation token, and callers may
+ /// safely pass HttpContext.RequestAborted. Abandoning a status read costs nothing: it starts no
+ /// durable work and tells the caller nothing it has to act on. Abandoning the creation call is what
+ /// is dangerous, because the creation keeps running on the FwHeadless side either way.
+ ///
+ public async Task CreationStatus(Guid projectId, CancellationToken cancellationToken = default)
+ {
+ var response = await httpClient.GetAsync($"/api/project/creation-status?projectId={projectId}", cancellationToken);
+ if (response.IsSuccessStatusCode)
+ return await response.Content.ReadFromJsonAsync(cancellationToken);
+ // 404 is an ordinary answer ("no such project"), not a fault worth an error log.
+ if (response.StatusCode != HttpStatusCode.NotFound)
+ {
+ logger.LogError("Failed to get project creation status: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}",
+ response.StatusCode,
+ response.ReasonPhrase,
+ projectId,
+ await response.Content.ReadAsStringAsync(cancellationToken));
+ }
+ return null;
+ }
+
+ ///
+ /// Waits for an in-flight creation to finish and returns how it ended, or null if FwHeadless doesn't
+ /// know the project or the call failed. Returns TimedOutAwaitingCreation if the wait is given up on
+ /// -- by either side -- which means "still running, ask again", never "creation failed".
+ ///
+ public async Task AwaitCreationFinished(Guid projectId, CancellationToken cancellationToken = default)
+ {
+ HttpResponseMessage response;
+ try
+ {
+ response = await httpClient.GetAsync($"/api/project/await-creation-finished?projectId={projectId}", cancellationToken);
+ }
+ catch (OperationCanceledException)
+ {
+ // We gave up first, so FwHeadless's own TimedOutAwaitingCreation response is on a connection
+ // we already dropped. Report the same thing ourselves, so a caller sees one contract no
+ // matter which side stopped waiting. The creation itself is untouched and still running.
+ logger.LogInformation("Gave up awaiting creation of project {ProjectId}; it is still running", projectId);
+ return ProjectCreationStatus.TimedOutAwaitingCreation;
+ }
+
+ if (response.IsSuccessStatusCode)
+ return await response.Content.ReadFromJsonAsync(cancellationToken);
+ if (response.StatusCode != HttpStatusCode.NotFound)
+ {
+ logger.LogError("Failed to await project creation: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}",
+ response.StatusCode,
+ response.ReasonPhrase,
+ projectId,
+ await response.Content.ReadAsStringAsync(cancellationToken));
+ }
+ return null;
+ }
}
diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs
index 8aac13b26b..55d7fd3e2b 100644
--- a/backend/LexBoxApi/Services/HgService.cs
+++ b/backend/LexBoxApi/Services/HgService.cs
@@ -20,7 +20,7 @@ public partial class HgService : IHgService
{
public const string ManageClientName = "HgResumeManage";
private const string ManageSecretHeader = "X-Manage-Secret";
- private const string AllZeroHash = "0000000000000000000000000000000000000000";
+ public const string AllZeroHash = "0000000000000000000000000000000000000000";
private readonly IOptions _options;
private readonly Lazy _hgClient;
diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs
index 459f14b138..a050169c49 100644
--- a/backend/LexBoxApi/Services/ProjectService.cs
+++ b/backend/LexBoxApi/Services/ProjectService.cs
@@ -25,7 +25,9 @@ public class ProjectService(
IEmailService emailService,
FwHeadlessClient fwHeadless)
{
- public async Task CreateProject(CreateProjectInput input)
+ public async Task CreateProject(CreateProjectInput input,
+ ProjectMigrationStatus? projectOrigin = null,
+ FwDataInitStatus fwDataInitStatus = FwDataInitStatus.None)
{
await using var transaction = await dbContext.Database.BeginTransactionAsync();
var projectId = input.Id ?? Guid.NewGuid();
@@ -41,7 +43,8 @@ public async Task CreateProject(CreateProjectInput input)
Id = projectId,
Code = input.Code,
Name = input.Name,
- ProjectOrigin = ProjectMigrationStatus.Migrated,
+ ProjectOrigin = projectOrigin ?? ProjectMigrationStatus.Migrated,
+ FwDataInitStatus = fwDataInitStatus,
Description = input.Description,
Type = input.Type,
LastCommit = null,
@@ -232,6 +235,52 @@ public async Task FinishReset(string code, Stream? zipFile = null)
return project;
}
+ ///
+ /// Marks a project whose repo FwHeadless has populated from the template as ready, recording the
+ /// commit it pushed.
+ ///
+ public async Task FinishFwDataInit(Guid projectId)
+ {
+ var project = await dbContext.Projects.FindAsync(projectId);
+ if (project is null) return;
+ project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code);
+ project.FwDataInitStatus = FwDataInitStatus.None;
+ await dbContext.SaveChangesAsync();
+ }
+
+ ///
+ /// Compensating cleanup for a project whose template population failed after CreateProject:
+ /// deletes FwHeadless's local project state, then removes the repo and the project row and
+ /// invalidates the same caches as a permanent delete.
+ /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a
+ /// project that InitFwDataProject just created.
+ ///
+ public async Task CleanupFailedProjectCreation(Guid projectId, string code)
+ {
+ // do this first, because it throws if a creation or sync is in progress, which stops us from
+ // tearing down an active or already-completed creation (mirrors DeleteProjectPermanently)
+ await fwHeadless.DeleteProject(projectId);
+ try
+ {
+ // Repo before row: if deleting the repo fails, the project row survives for an admin to see and
+ // delete, instead of an orphaned repo with no row pointing at it
+ await hgService.DeleteRepoIfExists(code);
+ var project = await dbContext.Projects.FindAsync(projectId);
+ if (project is not null)
+ {
+ dbContext.Projects.Remove(project);
+ await dbContext.SaveChangesAsync();
+ }
+ }
+ finally
+ {
+ // Don't forget to add more Invalidate calls here if we add new caches
+ InvalidateProjectCodeCache(code);
+ InvalidateProjectConfidentialityCache(projectId);
+ InvalidateProjectOrgIdsCache(projectId);
+ }
+ }
+
public async ValueTask LookupProjectOrgIds(Guid projectId)
{
var cacheKey = $"ProjectOrgsForId:{projectId}";
diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs
new file mode 100644
index 0000000000..ae0bbfa03e
--- /dev/null
+++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs
@@ -0,0 +1,59 @@
+using SIL.WritingSystems;
+
+namespace LexCore.Entities;
+
+///
+/// Body of the internal FwHeadless "init fwdata project" call. The vernacular/analysis
+/// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one
+/// vernacular; analysis and UI default to "en"); FwHeadless validates them again before using them.
+///
+public record InitFwDataProjectInput(
+ IReadOnlyList WsVernacular,
+ IReadOnlyList WsAnalysis,
+ string WsUi)
+{
+ // Deliberately generous: real projects use far fewer writing systems than this, and real tags are far
+ // shorter, but private-use subtags after "-x-" may carry arbitrary metadata and we don't want to forbid
+ // any reasonable use of it.
+ public const int MaxWritingSystemsPerList = 100;
+ public const int MaxWritingSystemTagLength = 128;
+
+ ///
+ /// Checks shared by LexBoxApi (before it creates anything) and FwHeadless. Returns an error
+ /// message, or null if the list is acceptable.
+ ///
+ public static string? ValidateWritingSystemList(string kind, IReadOnlyList? tags)
+ {
+ if (tags is not { Count: > 0 })
+ return $"At least one {kind} writing system is required";
+ if (tags.Count > MaxWritingSystemsPerList)
+ return $"At most {MaxWritingSystemsPerList} {kind} writing systems are allowed";
+ if (tags.Any(string.IsNullOrWhiteSpace))
+ return $"Empty {kind} writing system code";
+ if (tags.FirstOrDefault(t => t.Length > MaxWritingSystemTagLength) is { } tooLong)
+ return $"The {kind} writing system code is longer than {MaxWritingSystemTagLength} characters: {tooLong[..32]}…";
+ if (tags.FirstOrDefault(t => !IetfLanguageTag.IsValid(t)) is { } invalid)
+ return $"Invalid {kind} writing system code: {invalid}";
+ // BCP 47 tags are case-insensitive, so "en" and "EN" are the same writing system. Tags that differ only
+ // by an implicit script ("fr", "fr-Latn") get through, because spotting those needs the SLDR, which
+ // LexBox doesn't load; ProjectLoader.NewProject merges them into one writing system.
+ if (tags.GroupBy(t => t, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1) is { } duplicate)
+ return $"Duplicate {kind} writing system: {duplicate.Key}";
+ return null;
+ }
+
+ ///
+ /// Checks for the UI writing system, shared like . Returns an
+ /// error message, or null if the tag is acceptable.
+ ///
+ public static string? ValidateUiWritingSystem(string? tag)
+ {
+ if (string.IsNullOrWhiteSpace(tag))
+ return "A UI writing system is required";
+ if (tag.Length > MaxWritingSystemTagLength)
+ return $"The UI writing system code is longer than {MaxWritingSystemTagLength} characters: {tag[..32]}…";
+ if (!IetfLanguageTag.IsValid(tag))
+ return $"Invalid UI writing system code: {tag}";
+ return null;
+ }
+}
diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs
index 85d795c720..d3a5406a28 100644
--- a/backend/LexCore/Entities/Project.cs
+++ b/backend/LexCore/Entities/Project.cs
@@ -25,6 +25,7 @@ public class Project : EntityBase
public int? RepoSizeInKb { get; set; }
public DateTimeOffset? DeletedDate { get; set; }
public ResetStatus ResetStatus { get; set; } = ResetStatus.None;
+ public FwDataInitStatus FwDataInitStatus { get; set; } = FwDataInitStatus.None;
//historical reference for if this project originated here (migrated), or came from redmine, public or private
public ProjectMigrationStatus ProjectOrigin { get; set; } = ProjectMigrationStatus.Migrated;
@@ -97,6 +98,7 @@ public enum ProjectMigrationStatus
Migrating,
PrivateRedmine,
PublicRedmine,
+ LanguageForgeNonSR,
}
public enum ResetStatus
@@ -105,6 +107,13 @@ public enum ResetStatus
InProgress = 1
}
+public enum FwDataInitStatus
+{
+ None = 0,
+ // Created by InitFwDataProject, and FwHeadless hasn't yet finished populating its repo from the template
+ InProgress = 1
+}
+
public enum ProjectType
{
Unknown = 0,
diff --git a/backend/LexCore/Entities/ProjectCode.cs b/backend/LexCore/Entities/ProjectCode.cs
index 51ac881f4f..edc50cb69f 100644
--- a/backend/LexCore/Entities/ProjectCode.cs
+++ b/backend/LexCore/Entities/ProjectCode.cs
@@ -34,10 +34,19 @@ private void AssertIsSafeRepoName(string name)
{
if (InvalidRepoNames.Contains(name))
throw new ArgumentException($"Invalid repo name: {name}.");
- if (!ProjectCodeRegex().IsMatch(name))
+ if (!IsValidCode(name))
throw new ArgumentException($"Invalid repo name: {name}.");
}
- [GeneratedRegex(Project.ProjectCodeRegex)]
+ ///
+ /// Whether the whole string matches . Use this rather than
+ /// Regex.IsMatch with that pattern: its $ also matches before a trailing newline, so "abcd"
+ /// plus a line feed would pass, and hg URLs drop the newline, so they'd point at project "abcd".
+ ///
+ public static bool IsValidCode(string? code) => code is not null && ProjectCodeRegex().IsMatch(code);
+
+ // Project.ProjectCodeRegex with \z instead of $ (see IsValidCode). That one can stay as it is for
+ // [RegularExpression], which already requires the match to cover the whole string.
+ [GeneratedRegex(@"^[a-z\d][a-z-\d]*\z")]
private static partial Regex ProjectCodeRegex();
}
diff --git a/backend/LexCore/LexCore.csproj b/backend/LexCore/LexCore.csproj
index 8a1b8a72ff..cded3b2bad 100644
--- a/backend/LexCore/LexCore.csproj
+++ b/backend/LexCore/LexCore.csproj
@@ -10,6 +10,8 @@
LexCore is a plain library (no ASP.NET framework reference), so these come as packages. -->
+
+
diff --git a/backend/LexCore/Sync/ProjectCreationStatus.cs b/backend/LexCore/Sync/ProjectCreationStatus.cs
new file mode 100644
index 0000000000..67260ec19b
--- /dev/null
+++ b/backend/LexCore/Sync/ProjectCreationStatus.cs
@@ -0,0 +1,43 @@
+using System.Text.Json.Serialization;
+
+namespace LexCore.Sync;
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum ProjectCreationStatusEnum
+{
+ /// No creation is in flight and no project data exists in FwHeadless for this project.
+ NotCreated,
+ /// A creation is running right now.
+ Creating,
+ /// Project data exists in FwHeadless, i.e. a creation finished successfully.
+ Created,
+ /// A creation was attempted and failed. says why.
+ CreationFailed,
+ ///
+ /// The caller's await gave up before creation finished. The creation itself is unaffected and is
+ /// still running -- poll again rather than treating this as a failure.
+ ///
+ TimedOutAwaitingCreation,
+}
+
+///
+/// Status of a FwHeadless project creation, for callers that want to poll (or long-poll) instead of
+/// holding a request open for the whole operation. Mirrors , which does
+/// the same job for merges.
+///
+public record ProjectCreationStatus(
+ [property: JsonConverter(typeof(JsonStringEnumConverter))]
+ ProjectCreationStatusEnum Status,
+ string? Error = null)
+{
+ public static ProjectCreationStatus NotCreated => new(ProjectCreationStatusEnum.NotCreated);
+ public static ProjectCreationStatus Creating => new(ProjectCreationStatusEnum.Creating);
+ public static ProjectCreationStatus Created => new(ProjectCreationStatusEnum.Created);
+ public static ProjectCreationStatus TimedOutAwaitingCreation => new(ProjectCreationStatusEnum.TimedOutAwaitingCreation);
+
+ public static ProjectCreationStatus Failed(string error) => new(ProjectCreationStatusEnum.CreationFailed, error);
+
+ /// True once creation has stopped running, whether it succeeded or failed.
+ [JsonIgnore]
+ public bool IsFinished => Status is ProjectCreationStatusEnum.Created or ProjectCreationStatusEnum.CreationFailed;
+}
diff --git a/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs
new file mode 100644
index 0000000000..9795c46fde
--- /dev/null
+++ b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.Designer.cs
@@ -0,0 +1,1425 @@
+//
+using System;
+using System.Collections.Generic;
+using LexData;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace LexData.Migrations
+{
+ [DbContext(typeof(LexBoxDbContext))]
+ [Migration("20261109090102_AddProjectFwDataInitStatus")]
+ partial class AddProjectFwDataInitStatus
+ {
+ ///
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "und-u-ks-level2,und-u-ks-level2,icu,False")
+ .HasAnnotation("ProductVersion", "10.0.13")
+ .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+ NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("TriggerName")
+ .HasColumnType("text")
+ .HasColumnName("trigger_name");
+
+ b.Property("TriggerGroup")
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.Property("BlobData")
+ .HasColumnType("bytea")
+ .HasColumnName("blob_data");
+
+ b.HasKey("SchedulerName", "TriggerName", "TriggerGroup");
+
+ b.ToTable("qrtz_blob_triggers", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCalendar", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("CalendarName")
+ .HasColumnType("text")
+ .HasColumnName("calendar_name");
+
+ b.Property("Calendar")
+ .IsRequired()
+ .HasColumnType("bytea")
+ .HasColumnName("calendar");
+
+ b.HasKey("SchedulerName", "CalendarName");
+
+ b.ToTable("qrtz_calendars", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("TriggerName")
+ .HasColumnType("text")
+ .HasColumnName("trigger_name");
+
+ b.Property("TriggerGroup")
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.Property("CronExpression")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("cron_expression");
+
+ b.Property("TimeZoneId")
+ .HasColumnType("text")
+ .HasColumnName("time_zone_id");
+
+ b.HasKey("SchedulerName", "TriggerName", "TriggerGroup");
+
+ b.ToTable("qrtz_cron_triggers", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzFiredTrigger", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("EntryId")
+ .HasColumnType("text")
+ .HasColumnName("entry_id");
+
+ b.Property("FiredTime")
+ .HasColumnType("bigint")
+ .HasColumnName("fired_time");
+
+ b.Property("InstanceName")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("instance_name");
+
+ b.Property("IsNonConcurrent")
+ .HasColumnType("bool")
+ .HasColumnName("is_nonconcurrent");
+
+ b.Property("JobGroup")
+ .HasColumnType("text")
+ .HasColumnName("job_group");
+
+ b.Property("JobName")
+ .HasColumnType("text")
+ .HasColumnName("job_name");
+
+ b.Property("Priority")
+ .HasColumnType("integer")
+ .HasColumnName("priority");
+
+ b.Property("RequestsRecovery")
+ .HasColumnType("bool")
+ .HasColumnName("requests_recovery");
+
+ b.Property("ScheduledTime")
+ .HasColumnType("bigint")
+ .HasColumnName("sched_time");
+
+ b.Property("State")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("state");
+
+ b.Property("TriggerGroup")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.Property("TriggerName")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("trigger_name");
+
+ b.HasKey("SchedulerName", "EntryId");
+
+ b.HasIndex("InstanceName")
+ .HasDatabaseName("idx_qrtz_ft_trig_inst_name");
+
+ b.HasIndex("JobGroup")
+ .HasDatabaseName("idx_qrtz_ft_job_group");
+
+ b.HasIndex("JobName")
+ .HasDatabaseName("idx_qrtz_ft_job_name");
+
+ b.HasIndex("RequestsRecovery")
+ .HasDatabaseName("idx_qrtz_ft_job_req_recovery");
+
+ b.HasIndex("TriggerGroup")
+ .HasDatabaseName("idx_qrtz_ft_trig_group");
+
+ b.HasIndex("TriggerName")
+ .HasDatabaseName("idx_qrtz_ft_trig_name");
+
+ b.HasIndex("SchedulerName", "TriggerName", "TriggerGroup")
+ .HasDatabaseName("idx_qrtz_ft_trig_nm_gp");
+
+ b.ToTable("qrtz_fired_triggers", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("JobName")
+ .HasColumnType("text")
+ .HasColumnName("job_name");
+
+ b.Property("JobGroup")
+ .HasColumnType("text")
+ .HasColumnName("job_group");
+
+ b.Property("Description")
+ .HasColumnType("text")
+ .HasColumnName("description");
+
+ b.Property("IsDurable")
+ .HasColumnType("bool")
+ .HasColumnName("is_durable");
+
+ b.Property("IsNonConcurrent")
+ .HasColumnType("bool")
+ .HasColumnName("is_nonconcurrent");
+
+ b.Property("IsUpdateData")
+ .HasColumnType("bool")
+ .HasColumnName("is_update_data");
+
+ b.Property("JobClassName")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("job_class_name");
+
+ b.Property("JobData")
+ .HasColumnType("bytea")
+ .HasColumnName("job_data");
+
+ b.Property("RequestsRecovery")
+ .HasColumnType("bool")
+ .HasColumnName("requests_recovery");
+
+ b.HasKey("SchedulerName", "JobName", "JobGroup");
+
+ b.HasIndex("RequestsRecovery")
+ .HasDatabaseName("idx_qrtz_j_req_recovery");
+
+ b.ToTable("qrtz_job_details", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzLock", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("LockName")
+ .HasColumnType("text")
+ .HasColumnName("lock_name");
+
+ b.HasKey("SchedulerName", "LockName");
+
+ b.ToTable("qrtz_locks", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzPausedTriggerGroup", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("TriggerGroup")
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.HasKey("SchedulerName", "TriggerGroup");
+
+ b.ToTable("qrtz_paused_trigger_grps", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSchedulerState", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("InstanceName")
+ .HasColumnType("text")
+ .HasColumnName("instance_name");
+
+ b.Property("CheckInInterval")
+ .HasColumnType("bigint")
+ .HasColumnName("checkin_interval");
+
+ b.Property("LastCheckInTime")
+ .HasColumnType("bigint")
+ .HasColumnName("last_checkin_time");
+
+ b.HasKey("SchedulerName", "InstanceName");
+
+ b.ToTable("qrtz_scheduler_state", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("TriggerName")
+ .HasColumnType("text")
+ .HasColumnName("trigger_name");
+
+ b.Property("TriggerGroup")
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.Property("BooleanProperty1")
+ .HasColumnType("bool")
+ .HasColumnName("bool_prop_1");
+
+ b.Property("BooleanProperty2")
+ .HasColumnType("bool")
+ .HasColumnName("bool_prop_2");
+
+ b.Property("DecimalProperty1")
+ .HasColumnType("numeric")
+ .HasColumnName("dec_prop_1");
+
+ b.Property("DecimalProperty2")
+ .HasColumnType("numeric")
+ .HasColumnName("dec_prop_2");
+
+ b.Property("IntegerProperty1")
+ .HasColumnType("integer")
+ .HasColumnName("int_prop_1");
+
+ b.Property("IntegerProperty2")
+ .HasColumnType("integer")
+ .HasColumnName("int_prop_2");
+
+ b.Property("LongProperty1")
+ .HasColumnType("bigint")
+ .HasColumnName("long_prop_1");
+
+ b.Property("LongProperty2")
+ .HasColumnType("bigint")
+ .HasColumnName("long_prop_2");
+
+ b.Property("StringProperty1")
+ .HasColumnType("text")
+ .HasColumnName("str_prop_1");
+
+ b.Property("StringProperty2")
+ .HasColumnType("text")
+ .HasColumnName("str_prop_2");
+
+ b.Property("StringProperty3")
+ .HasColumnType("text")
+ .HasColumnName("str_prop_3");
+
+ b.Property("TimeZoneId")
+ .HasColumnType("text")
+ .HasColumnName("time_zone_id");
+
+ b.HasKey("SchedulerName", "TriggerName", "TriggerGroup");
+
+ b.ToTable("qrtz_simprop_triggers", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("TriggerName")
+ .HasColumnType("text")
+ .HasColumnName("trigger_name");
+
+ b.Property("TriggerGroup")
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.Property("RepeatCount")
+ .HasColumnType("bigint")
+ .HasColumnName("repeat_count");
+
+ b.Property("RepeatInterval")
+ .HasColumnType("bigint")
+ .HasColumnName("repeat_interval");
+
+ b.Property("TimesTriggered")
+ .HasColumnType("bigint")
+ .HasColumnName("times_triggered");
+
+ b.HasKey("SchedulerName", "TriggerName", "TriggerGroup");
+
+ b.ToTable("qrtz_simple_triggers", "quartz");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b =>
+ {
+ b.Property("SchedulerName")
+ .HasColumnType("text")
+ .HasColumnName("sched_name");
+
+ b.Property("TriggerName")
+ .HasColumnType("text")
+ .HasColumnName("trigger_name");
+
+ b.Property("TriggerGroup")
+ .HasColumnType("text")
+ .HasColumnName("trigger_group");
+
+ b.Property("CalendarName")
+ .HasColumnType("text")
+ .HasColumnName("calendar_name");
+
+ b.Property("Description")
+ .HasColumnType("text")
+ .HasColumnName("description");
+
+ b.Property("EndTime")
+ .HasColumnType("bigint")
+ .HasColumnName("end_time");
+
+ b.Property("JobData")
+ .HasColumnType("bytea")
+ .HasColumnName("job_data");
+
+ b.Property("JobGroup")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("job_group");
+
+ b.Property("JobName")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("job_name");
+
+ b.Property("MisfireInstruction")
+ .HasColumnType("smallint")
+ .HasColumnName("misfire_instr");
+
+ b.Property("MisfireOriginalFireTime")
+ .HasColumnType("bigint")
+ .HasColumnName("misfire_orig_fire_time");
+
+ b.Property("NextFireTime")
+ .HasColumnType("bigint")
+ .HasColumnName("next_fire_time");
+
+ b.Property("PreviousFireTime")
+ .HasColumnType("bigint")
+ .HasColumnName("prev_fire_time");
+
+ b.Property("Priority")
+ .HasColumnType("integer")
+ .HasColumnName("priority");
+
+ b.Property("StartTime")
+ .HasColumnType("bigint")
+ .HasColumnName("start_time");
+
+ b.Property("TriggerState")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("trigger_state");
+
+ b.Property("TriggerType")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("trigger_type");
+
+ b.HasKey("SchedulerName", "TriggerName", "TriggerGroup");
+
+ b.HasIndex("NextFireTime")
+ .HasDatabaseName("idx_qrtz_t_next_fire_time");
+
+ b.HasIndex("TriggerState")
+ .HasDatabaseName("idx_qrtz_t_state");
+
+ b.HasIndex("NextFireTime", "TriggerState")
+ .HasDatabaseName("idx_qrtz_t_nft_st");
+
+ b.HasIndex("SchedulerName", "JobName", "JobGroup");
+
+ b.ToTable("qrtz_triggers", "quartz");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.DraftProject", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("Code")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("Description")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("IsConfidential")
+ .HasColumnType("boolean");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("OrgId")
+ .HasColumnType("uuid");
+
+ b.Property("ProjectManagerId")
+ .HasColumnType("uuid");
+
+ b.Property("RetentionPolicy")
+ .HasColumnType("integer");
+
+ b.Property("Type")
+ .HasColumnType("integer");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Code")
+ .IsUnique();
+
+ b.HasIndex("ProjectManagerId");
+
+ b.ToTable("DraftProjects");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b =>
+ {
+ b.Property("ProjectId")
+ .HasColumnType("uuid");
+
+ b.Property("FlexModelVersion")
+ .HasColumnType("integer");
+
+ b.Property("LangProjectId")
+ .HasColumnType("uuid");
+
+ b.Property("LexEntryCount")
+ .HasColumnType("integer");
+
+ b.HasKey("ProjectId");
+
+ b.ToTable("FlexProjectMetadata");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.MediaFile", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CreatedDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Filename")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("Metadata")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("text")
+ .HasDefaultValueSql("'{}'");
+
+ b.Property("ProjectId")
+ .HasColumnType("uuid");
+
+ b.Property("Revision")
+ .HasColumnType("integer");
+
+ b.Property("UpdatedDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ProjectId");
+
+ b.ToTable("Files");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.OrgMember", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("OrgId")
+ .HasColumnType("uuid");
+
+ b.Property("Role")
+ .HasColumnType("integer");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("UserId")
+ .HasColumnType("uuid");
+
+ b.HasKey("Id");
+
+ b.HasIndex("OrgId");
+
+ b.HasIndex("UserId", "OrgId")
+ .IsUnique();
+
+ b.ToTable("OrgMembers", (string)null);
+ });
+
+ modelBuilder.Entity("LexCore.Entities.OrgProjects", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("OrgId")
+ .HasColumnType("uuid");
+
+ b.Property("ProjectId")
+ .HasColumnType("uuid");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ProjectId");
+
+ b.HasIndex("OrgId", "ProjectId")
+ .IsUnique();
+
+ b.ToTable("OrgProjects");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.Organization", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text")
+ .UseCollation("case_insensitive");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Name")
+ .IsUnique();
+
+ b.ToTable("Orgs", (string)null);
+ });
+
+ modelBuilder.Entity("LexCore.Entities.Project", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("Code")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("DeletedDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Description")
+ .HasColumnType("text");
+
+ b.Property("FwDataInitStatus")
+ .HasColumnType("integer");
+
+ b.Property("IsConfidential")
+ .HasColumnType("boolean");
+
+ b.Property("LastCommit")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("MigratedDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("ParentId")
+ .HasColumnType("uuid");
+
+ b.Property("ProjectOrigin")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer")
+ .HasDefaultValue(1);
+
+ b.Property("RepoSizeInKb")
+ .HasColumnType("integer");
+
+ b.Property("ResetStatus")
+ .HasColumnType("integer");
+
+ b.Property("RetentionPolicy")
+ .HasColumnType("integer");
+
+ b.Property("Type")
+ .HasColumnType("integer");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Code")
+ .IsUnique();
+
+ b.HasIndex("ParentId");
+
+ b.ToTable("Projects");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.ProjectUsers", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("ProjectId")
+ .HasColumnType("uuid");
+
+ b.Property("Role")
+ .HasColumnType("integer");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("UserId")
+ .HasColumnType("uuid");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ProjectId");
+
+ b.HasIndex("UserId", "ProjectId")
+ .IsUnique();
+
+ b.ToTable("ProjectUsers");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.User", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CanCreateProjects")
+ .HasColumnType("boolean");
+
+ b.Property("CreatedById")
+ .HasColumnType("uuid");
+
+ b.Property("CreatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("Email")
+ .HasColumnType("text")
+ .UseCollation("case_insensitive");
+
+ b.Property("EmailVerified")
+ .HasColumnType("boolean");
+
+ b.Property>("FeatureFlags")
+ .HasColumnType("text[]");
+
+ b.Property("GoogleId")
+ .HasColumnType("text");
+
+ b.Property("IsAdmin")
+ .HasColumnType("boolean");
+
+ b.Property("LastActive")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("LocalizationCode")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("text")
+ .HasDefaultValue("en");
+
+ b.Property("Locked")
+ .HasColumnType("boolean");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("OptedOutOfAnalytics")
+ .HasColumnType("boolean");
+
+ b.Property("PasswordHash")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("PasswordStrength")
+ .HasColumnType("integer");
+
+ b.Property("Salt")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("UpdatedDate")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("timestamp with time zone")
+ .HasDefaultValueSql("now()");
+
+ b.Property("Username")
+ .HasColumnType("text")
+ .UseCollation("case_insensitive");
+
+ b.HasKey("Id");
+
+ b.HasIndex("CreatedById");
+
+ b.HasIndex("Email")
+ .IsUnique();
+
+ b.HasIndex("Username")
+ .IsUnique();
+
+ b.ToTable("Users");
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("text");
+
+ b.Property("ApplicationType")
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("ClientId")
+ .HasMaxLength(100)
+ .HasColumnType("character varying(100)");
+
+ b.Property("ClientSecret")
+ .HasColumnType("text");
+
+ b.Property("ClientType")
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("ConcurrencyToken")
+ .IsConcurrencyToken()
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("ConsentType")
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("DisplayName")
+ .HasColumnType("text");
+
+ b.Property("DisplayNames")
+ .HasColumnType("text");
+
+ b.Property("JsonWebKeySet")
+ .HasColumnType("text");
+
+ b.Property("Permissions")
+ .HasColumnType("text");
+
+ b.Property("PostLogoutRedirectUris")
+ .HasColumnType("text");
+
+ b.Property("Properties")
+ .HasColumnType("text");
+
+ b.Property("RedirectUris")
+ .HasColumnType("text");
+
+ b.Property("Requirements")
+ .HasColumnType("text");
+
+ b.Property("Settings")
+ .HasColumnType("text");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ClientId")
+ .IsUnique();
+
+ b.ToTable("OpenIddictApplications", (string)null);
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("text");
+
+ b.Property("ApplicationId")
+ .HasColumnType("text");
+
+ b.Property("ConcurrencyToken")
+ .IsConcurrencyToken()
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("CreationDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Properties")
+ .HasColumnType("text");
+
+ b.Property("Scopes")
+ .HasColumnType("text");
+
+ b.Property("Status")
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("Subject")
+ .HasMaxLength(400)
+ .HasColumnType("character varying(400)");
+
+ b.Property("Type")
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ApplicationId", "Status", "Subject", "Type");
+
+ b.ToTable("OpenIddictAuthorizations", (string)null);
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreScope", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("text");
+
+ b.Property("ConcurrencyToken")
+ .IsConcurrencyToken()
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("Description")
+ .HasColumnType("text");
+
+ b.Property("Descriptions")
+ .HasColumnType("text");
+
+ b.Property("DisplayName")
+ .HasColumnType("text");
+
+ b.Property("DisplayNames")
+ .HasColumnType("text");
+
+ b.Property("Name")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)");
+
+ b.Property("Properties")
+ .HasColumnType("text");
+
+ b.Property("Resources")
+ .HasColumnType("text");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Name")
+ .IsUnique();
+
+ b.ToTable("OpenIddictScopes", (string)null);
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("text");
+
+ b.Property("ApplicationId")
+ .HasColumnType("text");
+
+ b.Property("AuthorizationId")
+ .HasColumnType("text");
+
+ b.Property("ConcurrencyToken")
+ .IsConcurrencyToken()
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("CreationDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("ExpirationDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Payload")
+ .HasColumnType("text");
+
+ b.Property("Properties")
+ .HasColumnType("text");
+
+ b.Property("RedemptionDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("ReferenceId")
+ .HasMaxLength(100)
+ .HasColumnType("character varying(100)");
+
+ b.Property("Status")
+ .HasMaxLength(50)
+ .HasColumnType("character varying(50)");
+
+ b.Property("Subject")
+ .HasMaxLength(400)
+ .HasColumnType("character varying(400)");
+
+ b.Property("Type")
+ .HasMaxLength(150)
+ .HasColumnType("character varying(150)");
+
+ b.HasKey("Id");
+
+ b.HasIndex("AuthorizationId");
+
+ b.HasIndex("ReferenceId")
+ .IsUnique();
+
+ b.HasIndex("ApplicationId", "Status", "Subject", "Type");
+
+ b.ToTable("OpenIddictTokens", (string)null);
+ });
+
+ modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("ChangeEntities")
+ .HasColumnType("jsonb");
+
+ b.Property("ClientId")
+ .HasColumnType("uuid");
+
+ b.Property("Metadata")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("ProjectId")
+ .HasColumnType("uuid");
+
+ b.ComplexProperty(typeof(Dictionary), "HybridDateTime", "SIL.Harmony.Core.ServerCommit.HybridDateTime#HybridDateTime", b1 =>
+ {
+ b1.IsRequired();
+
+ b1.Property("Counter")
+ .HasColumnType("bigint");
+
+ b1.Property("DateTime")
+ .HasColumnType("timestamp with time zone");
+ });
+
+ b.HasKey("Id");
+
+ b.HasIndex("ProjectId");
+
+ b.ToTable("CrdtCommits", (string)null);
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzBlobTrigger", b =>
+ {
+ b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger")
+ .WithMany("BlobTriggers")
+ .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Trigger");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzCronTrigger", b =>
+ {
+ b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger")
+ .WithMany("CronTriggers")
+ .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Trigger");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimplePropertyTrigger", b =>
+ {
+ b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger")
+ .WithMany("SimplePropertyTriggers")
+ .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Trigger");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzSimpleTrigger", b =>
+ {
+ b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", "Trigger")
+ .WithMany("SimpleTriggers")
+ .HasForeignKey("SchedulerName", "TriggerName", "TriggerGroup")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Trigger");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b =>
+ {
+ b.HasOne("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", "JobDetail")
+ .WithMany("Triggers")
+ .HasForeignKey("SchedulerName", "JobName", "JobGroup")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("JobDetail");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.DraftProject", b =>
+ {
+ b.HasOne("LexCore.Entities.User", "ProjectManager")
+ .WithMany()
+ .HasForeignKey("ProjectManagerId")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.Navigation("ProjectManager");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.FlexProjectMetadata", b =>
+ {
+ b.HasOne("LexCore.Entities.Project", null)
+ .WithOne("FlexProjectMetadata")
+ .HasForeignKey("LexCore.Entities.FlexProjectMetadata", "ProjectId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.OwnsOne("LexCore.Entities.ProjectWritingSystems", "WritingSystems", b1 =>
+ {
+ b1.Property("FlexProjectMetadataProjectId");
+
+ b1.HasKey("FlexProjectMetadataProjectId");
+
+ b1.ToTable("FlexProjectMetadata");
+
+ b1
+ .ToJson("WritingSystems")
+ .HasColumnType("jsonb");
+
+ b1.WithOwner()
+ .HasForeignKey("FlexProjectMetadataProjectId");
+
+ b1.OwnsMany("LexCore.Entities.FLExWsId", "AnalysisWss", b2 =>
+ {
+ b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId");
+
+ b2.Property("__synthesizedOrdinal")
+ .ValueGeneratedOnAdd();
+
+ b2.Property("IsActive");
+
+ b2.Property("IsDefault");
+
+ b2.Property("Tag")
+ .IsRequired();
+
+ b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal");
+
+ b2.ToTable("FlexProjectMetadata");
+
+ b2.WithOwner()
+ .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId");
+ });
+
+ b1.OwnsMany("LexCore.Entities.FLExWsId", "VernacularWss", b2 =>
+ {
+ b2.Property("ProjectWritingSystemsFlexProjectMetadataProjectId");
+
+ b2.Property("__synthesizedOrdinal")
+ .ValueGeneratedOnAdd();
+
+ b2.Property("IsActive");
+
+ b2.Property("IsDefault");
+
+ b2.Property("Tag")
+ .IsRequired();
+
+ b2.HasKey("ProjectWritingSystemsFlexProjectMetadataProjectId", "__synthesizedOrdinal");
+
+ b2.ToTable("FlexProjectMetadata");
+
+ b2.WithOwner()
+ .HasForeignKey("ProjectWritingSystemsFlexProjectMetadataProjectId");
+ });
+
+ b1.Navigation("AnalysisWss");
+
+ b1.Navigation("VernacularWss");
+ });
+
+ b.Navigation("WritingSystems");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.MediaFile", b =>
+ {
+ b.HasOne("LexCore.Entities.Project", null)
+ .WithMany()
+ .HasForeignKey("ProjectId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+ });
+
+ modelBuilder.Entity("LexCore.Entities.OrgMember", b =>
+ {
+ b.HasOne("LexCore.Entities.Organization", "Organization")
+ .WithMany("Members")
+ .HasForeignKey("OrgId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.HasOne("LexCore.Entities.User", "User")
+ .WithMany("Organizations")
+ .HasForeignKey("UserId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Organization");
+
+ b.Navigation("User");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.OrgProjects", b =>
+ {
+ b.HasOne("LexCore.Entities.Organization", "Org")
+ .WithMany()
+ .HasForeignKey("OrgId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.HasOne("LexCore.Entities.Project", "Project")
+ .WithMany()
+ .HasForeignKey("ProjectId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Org");
+
+ b.Navigation("Project");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.Project", b =>
+ {
+ b.HasOne("LexCore.Entities.Project", null)
+ .WithMany()
+ .HasForeignKey("ParentId");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.ProjectUsers", b =>
+ {
+ b.HasOne("LexCore.Entities.Project", "Project")
+ .WithMany("Users")
+ .HasForeignKey("ProjectId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.HasOne("LexCore.Entities.User", "User")
+ .WithMany("Projects")
+ .HasForeignKey("UserId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+
+ b.Navigation("Project");
+
+ b.Navigation("User");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.User", b =>
+ {
+ b.HasOne("LexCore.Entities.User", "CreatedBy")
+ .WithMany("UsersICreated")
+ .HasForeignKey("CreatedById")
+ .OnDelete(DeleteBehavior.Restrict);
+
+ b.Navigation("CreatedBy");
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b =>
+ {
+ b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application")
+ .WithMany("Authorizations")
+ .HasForeignKey("ApplicationId");
+
+ b.Navigation("Application");
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreToken", b =>
+ {
+ b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", "Application")
+ .WithMany("Tokens")
+ .HasForeignKey("ApplicationId");
+
+ b.HasOne("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", "Authorization")
+ .WithMany("Tokens")
+ .HasForeignKey("AuthorizationId");
+
+ b.Navigation("Application");
+
+ b.Navigation("Authorization");
+ });
+
+ modelBuilder.Entity("SIL.Harmony.Core.ServerCommit", b =>
+ {
+ b.HasOne("LexCore.Entities.Project", null)
+ .WithMany("HarmonyCommits")
+ .HasForeignKey("ProjectId")
+ .OnDelete(DeleteBehavior.Cascade)
+ .IsRequired();
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzJobDetail", b =>
+ {
+ b.Navigation("Triggers");
+ });
+
+ modelBuilder.Entity("AppAny.Quartz.EntityFrameworkCore.Migrations.QuartzTrigger", b =>
+ {
+ b.Navigation("BlobTriggers");
+
+ b.Navigation("CronTriggers");
+
+ b.Navigation("SimplePropertyTriggers");
+
+ b.Navigation("SimpleTriggers");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.Organization", b =>
+ {
+ b.Navigation("Members");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.Project", b =>
+ {
+ b.Navigation("FlexProjectMetadata");
+
+ b.Navigation("HarmonyCommits");
+
+ b.Navigation("Users");
+ });
+
+ modelBuilder.Entity("LexCore.Entities.User", b =>
+ {
+ b.Navigation("Organizations");
+
+ b.Navigation("Projects");
+
+ b.Navigation("UsersICreated");
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreApplication", b =>
+ {
+ b.Navigation("Authorizations");
+
+ b.Navigation("Tokens");
+ });
+
+ modelBuilder.Entity("OpenIddict.EntityFrameworkCore.Models.OpenIddictEntityFrameworkCoreAuthorization", b =>
+ {
+ b.Navigation("Tokens");
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
diff --git a/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs
new file mode 100644
index 0000000000..7fd0ffdad8
--- /dev/null
+++ b/backend/LexData/Migrations/20261109090102_AddProjectFwDataInitStatus.cs
@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace LexData.Migrations
+{
+ ///
+ public partial class AddProjectFwDataInitStatus : Migration
+ {
+ ///
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AddColumn(
+ name: "FwDataInitStatus",
+ table: "Projects",
+ type: "integer",
+ nullable: false,
+ defaultValue: 0);
+ }
+
+ ///
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropColumn(
+ name: "FwDataInitStatus",
+ table: "Projects");
+ }
+ }
+}
diff --git a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs
index a8a113dd7c..6678f08f20 100644
--- a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs
+++ b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs
@@ -694,6 +694,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("Description")
.HasColumnType("text");
+ b.Property("FwDataInitStatus")
+ .HasColumnType("integer");
+
b.Property("IsConfidential")
.HasColumnType("boolean");
diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs
new file mode 100644
index 0000000000..bb8b796ec8
--- /dev/null
+++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs
@@ -0,0 +1,131 @@
+using FwHeadless.Services;
+using LexCore.Sync;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Testing.FwHeadless;
+
+///
+/// Unit tests for the per-project creation reservation that stops a sync from racing a project that
+/// is still being created from a template (and stops two concurrent creations of the same project).
+///
+public class SyncHostedServiceCreationReservationTests
+{
+ /// Bounded so a regression that never completes the wait fails the test instead of hanging it.
+ private static CancellationToken TestTimeout => new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
+
+ private static SyncHostedService NewService() =>
+ new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions()));
+
+ [Fact]
+ public void TryStartProjectCreation_blocks_a_second_concurrent_creation_and_is_reusable_after_release()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+
+ svc.TryStartProjectCreation(projectId).Should().BeTrue();
+ svc.TryStartProjectCreation(projectId).Should().BeFalse("a creation is already in flight for this project");
+ svc.IsJobQueuedOrRunning(projectId).Should().BeTrue();
+
+ svc.EndProjectCreation(projectId, ProjectCreationStatus.Created);
+ svc.IsJobQueuedOrRunning(projectId).Should().BeFalse();
+ svc.TryStartProjectCreation(projectId).Should().BeTrue("the reservation is released and reusable");
+ }
+
+ [Fact]
+ public void QueueJob_is_refused_while_a_project_is_being_created()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+
+ svc.TryStartProjectCreation(projectId).Should().BeTrue();
+ svc.QueueJob(projectId).Should().Be(QueueJobResult.ProjectBeingCreated, "a sync must not race a project that's still being created");
+
+ svc.EndProjectCreation(projectId, ProjectCreationStatus.Created);
+ svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued, "syncing is allowed once creation has finished");
+ }
+
+ [Fact]
+ public void A_queued_sync_blocks_a_creation()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+
+ svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued);
+ svc.QueueJob(projectId).Should().Be(QueueJobResult.AlreadyQueued);
+ svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project");
+ }
+
+ [Fact]
+ public void QueueJob_is_refused_while_a_sync_is_running()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+
+ svc.QueueJob(projectId).Should().Be(QueueJobResult.Queued);
+ svc.MarkSyncStarted(projectId);
+
+ svc.QueueJob(projectId).Should().Be(QueueJobResult.SyncAlreadyRunning,
+ "the running sync may have missed the caller's changes, so it must not be reported as covering them");
+ svc.IsJobQueuedOrRunning(projectId).Should().BeTrue();
+ svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is running for this project");
+ }
+
+ [Fact]
+ public async Task AwaitCreationFinished_returns_the_outcome_the_creation_reported()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+ svc.TryStartProjectCreation(projectId).Should().BeTrue();
+
+ var awaiting = svc.AwaitCreationFinished(projectId, TestTimeout);
+ awaiting.IsCompleted.Should().BeFalse("the creation is still in flight");
+
+ svc.EndProjectCreation(projectId, ProjectCreationStatus.Failed("the push failed"));
+
+ var result = await awaiting;
+ result.Should().NotBeNull();
+ result!.Status.Should().Be(ProjectCreationStatusEnum.CreationFailed);
+ result.Error.Should().Be("the push failed");
+ }
+
+ [Fact]
+ public async Task AwaitCreationFinished_returns_a_result_that_finished_before_the_caller_asked()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+
+ svc.TryStartProjectCreation(projectId).Should().BeTrue();
+ svc.EndProjectCreation(projectId, ProjectCreationStatus.Created);
+
+ // The caller shows up late (e.g. its own request timed out and it came back to ask). It must
+ // still learn the outcome rather than being told nothing ever happened.
+ var result = await svc.AwaitCreationFinished(projectId, TestTimeout);
+ result.Should().NotBeNull();
+ result!.Status.Should().Be(ProjectCreationStatusEnum.Created);
+ }
+
+ [Fact]
+ public async Task AwaitCreationFinished_returns_null_when_no_creation_is_known()
+ {
+ var svc = NewService();
+
+ var result = await svc.AwaitCreationFinished(Guid.NewGuid(), TestTimeout);
+ result.Should().BeNull("the caller should fall back to deriving the status from the project itself");
+ }
+
+ [Fact]
+ public async Task AwaitCreationFinished_honours_the_caller_token_without_ending_the_creation()
+ {
+ var svc = NewService();
+ var projectId = Guid.NewGuid();
+ svc.TryStartProjectCreation(projectId).Should().BeTrue();
+
+ using var cts = new CancellationTokenSource();
+ var awaiting = svc.AwaitCreationFinished(projectId, cts.Token);
+ await cts.CancelAsync();
+
+ await Assert.ThrowsAnyAsync(() => awaiting);
+ svc.IsProjectBeingCreated(projectId).Should().BeTrue("giving up on the wait must not cancel the creation");
+ }
+}
diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs
new file mode 100644
index 0000000000..97e9fe0cd9
--- /dev/null
+++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs
@@ -0,0 +1,189 @@
+using LexBoxApi.Controllers;
+using LexCore.Entities;
+using LexCore.ServiceInterfaces;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+
+namespace Testing.LexBoxApi;
+
+///
+/// Unit tests for ProjectController.InitFwDataProject input validation. The 400 branches run before
+/// any injected service is touched, so the controller can be built with null dependencies; only a
+/// ProblemDetailsFactory (from AddControllers) is needed for Problem() to render.
+///
+public class InitFwDataProjectValidationTests
+{
+ private static ProjectController NewController(IPermissionService? permissionService = null)
+ {
+ var services = new ServiceCollection();
+ services.AddLogging();
+ services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves
+ return new ProjectController(null!, null!, null!, permissionService!, null!, NullLogger.Instance)
+ {
+ ControllerContext = new ControllerContext
+ {
+ HttpContext = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() }
+ }
+ };
+ }
+
+ [Fact]
+ public async Task Rejects_when_no_vernacular_writing_system_is_supplied()
+ {
+ var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: []);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("vernacular");
+ }
+
+ // The controller's services are all null, so reaching the 400 proves nothing was created first
+ [Fact]
+ public async Task Rejects_a_duplicate_vernacular_writing_system_before_creating_anything()
+ {
+ var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr", "fr"]);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("Duplicate vernacular writing system");
+ }
+
+ [Fact]
+ public async Task Rejects_an_invalid_vernacular_writing_system_before_creating_anything()
+ {
+ var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["not_a_tag"]);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Be("Invalid vernacular writing system code: not_a_tag");
+ }
+
+ [Fact]
+ public async Task Rejects_too_many_analysis_writing_systems_before_creating_anything()
+ {
+ var analysis = Enumerable.Range(0, InitFwDataProjectInput.MaxWritingSystemsPerList + 1).Select(i => $"qaa-x-ws{i}").ToArray();
+ var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsAnalysis: analysis);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("analysis writing systems are allowed");
+ }
+
+ [Fact]
+ public async Task Rejects_a_too_long_UI_writing_system_before_creating_anything()
+ {
+ var wsUi = "qaa-x-" + new string('a', InitFwDataProjectInput.MaxWritingSystemTagLength);
+ var result = await NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], wsUi: wsUi);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("UI writing system code is longer than");
+ }
+
+ [Theory]
+ [InlineData("Bad Code!")]
+ // A trailing newline passes a Regex.IsMatch with $, and the hg URL would then name project "abcd"
+ [InlineData("abcd\n")]
+ public async Task Rejects_an_invalid_project_code(string code)
+ {
+ var result = await NewController().InitFwDataProject(code, name: "My project", wsVernacular: ["fr"]);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("Invalid project code");
+ }
+
+ [Theory]
+ [InlineData(null)]
+ [InlineData("")]
+ [InlineData(" ")]
+ public async Task Rejects_a_missing_project_name(string? name)
+ {
+ var result = await NewController().InitFwDataProject("myproj", name: name!, wsVernacular: ["fr"]);
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("A project name is required");
+ }
+
+ /// A permission service that either passes or fails the admin assertion.
+ private static IPermissionService PermissionService(bool isAdmin)
+ {
+ var mock = new Mock();
+ if (!isAdmin) mock.Setup(p => p.AssertIsAdmin()).Throws();
+ return mock.Object;
+ }
+
+ [Fact]
+ public async Task Rejects_a_project_origin_that_is_not_a_migration_status()
+ {
+ var result = await NewController(PermissionService(isAdmin: true))
+ .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin");
+ var problem = result.Result.Should().BeOfType()
+ .Which.Value.Should().BeOfType().Which;
+ problem.Status.Should().Be(StatusCodes.Status400BadRequest);
+ problem.Detail.Should().Contain("Invalid project origin");
+ }
+
+ [Fact]
+ public async Task Rejects_a_project_origin_from_a_non_admin()
+ {
+ // The admin check must happen before the value is even parsed, and must outlive [AdminRequired],
+ // which is expected to be relaxed for this endpoint later.
+ var act = () => NewController(PermissionService(isAdmin: false))
+ .InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "Migrated");
+ await act.Should().ThrowAsync();
+ }
+
+ [Fact]
+ public async Task Ignores_an_empty_project_origin_without_requiring_admin()
+ {
+ // An empty value isn't "specified", so it needs no admin check; a null permission service
+ // proves AssertIsAdmin() was never called. Fails later (null project service) than validation.
+ var act = () => NewController().InitFwDataProject("myproj", name: "My project", wsVernacular: ["fr"], projectOrigin: "");
+ await act.Should().ThrowAsync();
+ }
+
+ [Theory]
+ [InlineData("Migrated", ProjectMigrationStatus.Migrated)]
+ [InlineData("migrating", ProjectMigrationStatus.Migrating)]
+ [InlineData("PUBLICREDMINE", ProjectMigrationStatus.PublicRedmine)]
+ [InlineData("languageforgenonsr", ProjectMigrationStatus.LanguageForgeNonSR)]
+ public void Project_origin_parses_case_insensitively(string input, ProjectMigrationStatus expected)
+ {
+ ProjectController.TryParseProjectOrigin(input, out var origin).Should().BeTrue();
+ origin.Should().Be(expected);
+ }
+
+ [Theory]
+ [InlineData("NotAnOrigin")]
+ [InlineData("1")] // Enum.TryParse accepts raw numbers; the API takes names only
+ [InlineData("42")]
+ [InlineData("Migrated,Migrating")] // ...and comma-separated lists, which OR into a defined value here
+ [InlineData("")]
+ public void Project_origin_rejects_values_that_are_not_named_members(string input)
+ {
+ ProjectController.TryParseProjectOrigin(input, out _).Should().BeFalse();
+ }
+
+ [Fact]
+ public void Analysis_writing_systems_default_to_english_when_null()
+ {
+ ProjectController.AnalysisWritingSystemsOrDefault(null).Should().Equal("en");
+ }
+
+ [Fact]
+ public void Analysis_writing_systems_default_to_english_when_empty()
+ {
+ ProjectController.AnalysisWritingSystemsOrDefault([]).Should().Equal("en");
+ }
+
+ [Fact]
+ public void Analysis_writing_systems_are_kept_when_supplied()
+ {
+ ProjectController.AnalysisWritingSystemsOrDefault(["fr", "es"]).Should().Equal("fr", "es");
+ }
+}
diff --git a/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs
new file mode 100644
index 0000000000..e34e61b1f1
--- /dev/null
+++ b/backend/Testing/LexBoxApi/Jobs/InitFwDataProjectJobDbTests.cs
@@ -0,0 +1,60 @@
+using LexBoxApi.Jobs;
+using LexBoxApi.Services;
+using LexBoxApi.Services.Email;
+using LexCore.Config;
+using LexCore.Entities;
+using LexCore.ServiceInterfaces;
+using LexData;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Moq;
+using Quartz;
+using Testing.Fixtures;
+
+namespace Testing.LexBoxApi.Jobs;
+
+[Collection(nameof(TestingServicesFixture))]
+[Trait("Category", "RequiresDb")]
+public class InitFwDataProjectJobDbTests(TestingServicesFixture testing)
+{
+ [Fact]
+ public async Task A_failure_marking_the_project_ready_is_retried()
+ {
+ var serviceProvider = testing.ConfigureServices();
+ var dbContext = serviceProvider.GetRequiredService();
+ var hgService = new Mock