diff --git a/docker-compose.yml b/docker-compose.yml index 35d8145697..33f2807515 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,9 @@ services: # needed this volume mapping so changes to partials would be reflected in running app. - ./src/angular-app:/var/www/src/angular-app + # To be able to run Python migration scripts from inside the container in dev mode + - ./scripts:/var/www/scripts + lfmerge: build: context: docker/lfmerge diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index d803423066..f4c56674bb 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -55,6 +55,12 @@ COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait # DEVELOPMENT BASE IMAGE FROM sillsdev/web-languageforge:base-php AS development-base RUN install-php-extensions xdebug-^3.1 +# Python for scripts/copy_project_for_migration.py, which runs in this container +# because it needs Mongo, the project assets volume and the LFMerge sendreceive +# volume. Development only; the production image has no need for it. +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 \ + && apt-get clean && rm -rf /var/lib/apt/lists/* COPY docker/app/docker-php-ext-xdebug.ini /usr/local/etc/php/conf.d RUN mv $PHP_INI_DIR/php.ini-development $PHP_INI_DIR/php.ini COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait diff --git a/docker/lfmerge/Dockerfile b/docker/lfmerge/Dockerfile index 2e47bf37f7..00110f7700 100644 --- a/docker/lfmerge/Dockerfile +++ b/docker/lfmerge/Dockerfile @@ -1,2 +1,2 @@ -FROM ghcr.io/sillsdev/lfmerge:2.0.142 +FROM ghcr.io/sillsdev/lfmerge:2.0.143 # Do not add anything to this Dockerfile, it should stay empty diff --git a/scripts/copy_project_for_migration.py b/scripts/copy_project_for_migration.py new file mode 100755 index 0000000000..05b2cfcde0 --- /dev/null +++ b/scripts/copy_project_for_migration.py @@ -0,0 +1,3438 @@ +#!/usr/bin/env python3 +"""Migrate a non-Send/Receive Language Forge project into the Send/Receive system. + +Given the projectCode of an existing project, this creates a complete copy whose +projectCode is the original with "-migrate" appended, points the copy's +sendReceiveProjectIdentifier at the LexBox-normalised form of that code, and +drives it through a full migration into LexBox. + +The original project is never modified. + +The source project must NOT already be a Send/Receive project: if it has a +non-empty sendReceiveProjectIdentifier the script exits with an error. Such a +project is already synced with a repository, and copying it would leave two LF +projects pointing into the same Send/Receive history. + +MIGRATION SEQUENCE + 1. Copy the sf_ database to sf_-migrate. + 2. Rewrite the copy's numeric option list keys as strings, and the two + numeric lexicon references that point at them. See OPTION LIST KEY + COERCION below. This touches the copy only. + 3. Give a fresh UUID4 to any lexicon entry in the copy whose guid is an + empty string. See EMPTY ENTRY GUIDS below. This touches the copy only. + 4. Stage the on-disk assets as real directories under the copy's own assets + folder. (They are converted to symlinks in step 8.) + 5. Insert the `projects` document: new _id, new projectCode, and + sendReceiveProjectIdentifier set to the normalised form of that code + (see send_receive_identifier -- underscores become hyphens and CamelCase + is split, because LexBox project codes must be lowercase). + 6. init_lexbox_project() logs in to LexBox (POST /api/login), calls + POST /api/project/initFwDataProject with the writing systems taken from + the LF project config. That endpoint answers 202 at once and builds the + project in the background, so the script then polls + GET /api/project/fwDataProjectCreationStatus/{projectId} until LexBox + reports it Created. Every writing system the project knows about + must be covered by the vernacular or analysis list; any that is not is + warned about and sent as analysis. + 7. Remove any webwork clone and LFMerge state file an earlier attempt left + behind (see clear_send_receive_state -- the LexBox project just created + has a new root changeset, so an older clone is unrelated to it), then run + the first Send/Receive. LFMerge clones the repository into + webwork// and pushes the Mongo data (lexicon, comments, + option lists) up to LexBox. Then wait for it to finish. + 8. Move any asset not already present in + webwork//LinkedFiles/{AudioVisual,Pictures} into that tree, + and replace the copy's assets//sf_/{audio,pictures} + directories with symlinks pointing there -- the layout a real + Send/Receive project has (LexProjectModel::createAssetsFolders). + 9. Second Send/Receive, to push those assets up to LexBox. + + Steps 7 and 9 wait on the LFMerge state file, + sendreceive/state/.state, so the phases stay ordered. + +WHERE TO RUN THIS + Inside the `app` container, which is where the Mongo host, the project assets + volume and the LFMerge sendreceive volume are all reachable: + + docker compose exec app python3 /var/www/scripts/copy_project_for_migration.py + + The base image (php:7.4-apache) ships no Python, so the development-base + stage of docker/app/Dockerfile installs python3. If `python3: not found`, + rebuild the image with `make build`. + + No pip packages are needed. This script deliberately uses only the standard + library plus the mongosh/mongodump/mongorestore binaries that the app image + already installs (see docker/base-php/Dockerfile). + + Note that the app and lfmerge containers share both the assets volume and the + sendreceive volume (docker-compose.yml), so the files this script writes are + the same files LFMerge reads. + +WHAT GETS COPIED + * the `projects` document in the main database + * the project's own database, sf_, with all collections and + indexes (lexicon, lexiconComments, optionlists, activity, messages) + * the on-disk assets, assets//sf_/{audio,pictures} + * membership: the new project's _id is added to each member's + users.projects array + +OPTION LIST KEY COERCION + Option list item keys are meant to be strings, but some projects store them + as BSON int32: in the 2026-07-06 corpus, 39,759 such keys across 53 projects, + nearly all in the `anthro-code` list (reached from config as + senses.anthropologyCategories), the rest in `domain-type` and + `pers-feature-value`. Step 2 rewrites every one of them in the copy as the + string of that integer, which is exactly what each item's `abbreviation` + already holds. Coercion cannot collide: no coerced key equals an existing + string key in the same list, and no two integers coerce to the same string. + + The lexicon almost always already references these items by string ("27", + "130"), so coercing the lists is what brings the two sides into agreement. + Two senses in the whole corpus are the exception -- they reference by + integer, and would be left dangling by a list-only coercion, so they are + converted too. See SPECIAL_LEXICON_INT_REFERENCES; those two senses are the + only lexicon documents this script modifies for the sake of option lists. + The other lexicon change it makes is described next. + +EMPTY ENTRY GUIDS + Five projects in the 2026-07-06 corpus have exactly one lexicon entry whose + guid is an empty string. That is the whole of the problem: no nulls, no + missing guid fields, no malformed values, and entries only -- senses and + examples never exhibit it. Step 3 therefore matches `guid: ""` exactly and + leaves every other guid alone, including one that is a valid UUID of some + version other than 4, which LfMerge parses happily. + + An empty guid does not make LfMerge fail, which is why it has gone + unnoticed. LfMerge reads the field with Guid.TryParse + (MongoConnection.CanParseGuid), so the entry is quietly dropped from the + ObjectId->Guid map that comments are attached by, and + ConvertMongoToLcmLexicon then falls back to `lfEntry.Guid ?? Guid.Empty`. + The entry would be created in FieldWorks under the all-zeros GUID, and a + second entry in the same state would find that object and overwrite it + rather than create its own -- so entries would silently merge instead of + erroring. + +WHAT IS DELIBERATELY NOT TOUCHED + * The SOURCE project, in every respect. All coercion happens in the copy, + after the database has been duplicated. + * Every other lexicon value. The only lexicon writes are the two senses + named in SPECIAL_LEXICON_INT_REFERENCES and the empty entry guids of + step 3; any other numeric option list reference is reported as a warning + and left exactly as it is. + * Guids on senses and examples. They are nullable in the same way, but the + empty-string problem does not arise there, so they are not scanned. + * `abbreviation` and `value` on option list items, which are already strings. + * Sync state is copied verbatim: lastSyncedDate and the per-entry dirtySR + flags are left exactly as the original has them. Because the source has + never synced, that means lastSyncedDate is still its default of epoch 0 -- + which is what you want for a first push into an empty repository. + * `sendReceiveProject` ({name, repository, role}) is not populated. The + source has no Send/Receive configuration to copy, so the copy ends up with + sendReceiveProjectIdentifier set but no repository details. LFMerge does + not need them -- it works from the identifier -- but the LF UI displays + them, so init_lexbox_project() is the place to record them if wanted. + * `userrelation` unread-activity documents are not copied. +""" + +from __future__ import annotations + +import argparse +import http.cookiejar +import json +import os +import re +import shutil +import ssl +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +import uuid +from typing import Any, Callable +from urllib.parse import quote_plus, urlencode, urlsplit, urlunsplit +from xml.sax.saxutils import escape as xml_escape + +MIGRATE_SUFFIX = "-migrate" + +# Defaults match the `app` service environment in docker-compose.yml, and the +# hardcoded LFMerge paths in SendReceiveCommands.php. +DEFAULT_MONGODB_CONN = "mongodb://lf-db:27017" +DEFAULT_DATABASE = "scriptureforge" +DEFAULT_ASSETS_ROOT = "/var/www/html/assets" +DEFAULT_SENDRECEIVE_BASE = "/var/lib/languageforge/lexicon/sendreceive" + +# (assets subfolder, LinkedFiles subfolder) -- LexProjectModel::createAssetsFolders +ASSET_LINKED_FILES = (("audio", "AudioVisual"), ("pictures", "Pictures")) + +# The only two lexicon values in the whole corpus that reference an option list +# item by a NUMBER rather than by a string. Everywhere else the lexicon already +# uses strings ("27", "130", "824.1"), so coercing the option lists on its own +# lines the two sides up -- but these two would be left pointing at a key that +# no longer exists, so they are coerced with them. +# +# Keyed by SOURCE projectCode (the copy's code is that plus --suffix, so the +# source is what identifies the data). The value is the field to look in and the +# exact `values` list that must be present, so a sense is only rewritten when +# both the project and the list match. Anything else numeric is reported and +# left alone. Found by scanning the 2026-07-06 mongodump: +# +# test-project-d entry 58bfb4f78b2147101c74ece2 [18] +# vocabularium_totius_latinitatis entry 61de9ccb499649193b5d5ce5 [104] +# +# Both are anthropologyCategories, whose list is anthro-code. Both projects are +# tiny (65 and 3 entries), which is why nobody hit this before. +SPECIAL_LEXICON_INT_REFERENCES: dict[str, tuple[str, list[int]]] = { + "test-project-d": ("anthropologyCategories", [18]), + "vocabularium_totius_latinitatis": ("anthropologyCategories", [104]), +} + +# SRState values, from src/angular-app/bellows/shared/model/send-receive-state.model.ts +SR_IN_PROGRESS = frozenset({"CLONING", "SYNCING", "PENDING", "LF_CLONING"}) +SR_DONE = frozenset({"IDLE", "SYNCED"}) +SR_FAILED = frozenset({"HOLD", "ERROR"}) + +WEB_USER = "www-data" + +# LexBox. The JWT is delivered as this cookie, whose value literally IS the JWT +# (AuthKernel.cs:229 overrides the cookie ticket format to write a JWT). +# In development LexBox runs outside this Docker Compose environment, reached +# through the "admin.localhost:host-gateway" extra_hosts entry that the app and +# lfmerge services already declare, over plain HTTP. Production is lexbox.org. +DEFAULT_LEXBOX_HOST = "http://admin.localhost" +DEFAULT_LEXBOX_USER = "admin" +DEFAULT_LEXBOX_PASSWORD = "pass" +LEXBOX_AUTH_COOKIE = ".LexBoxAuth" +# Project.cs:12 -- LexBox rejects codes that do not match, and codes under 4 chars. +LEXBOX_CODE_RE = re.compile(r"^[a-z\d][a-z\-\d]*$") +LEXBOX_CODE_MIN_LENGTH = 4 +# liblcm seeds a new project's anthropology list (the OCM outline) from a file shipped with +# SIL.LCModel and writes each label into the project's DEFAULT ANALYSIS writing system -- which is +# simply the first entry of wsAnalysis. The OCM file has no labels for an unlisted-language ("qaa") +# writing system, so one of those in first position makes CreateNewLangProj die inside +# InitializeAnthroList with +# System.ArgumentOutOfRangeException ... (Parameter 'ws') +# and the project is never created. Every OCM entry does have an `en` label, so when the list leads +# with a qaa tag, `en` is moved (or added) to the front and the rest keeps its order behind it. +# Any other leading tag is left alone: reordering a working project's writing systems for no reason +# would change which one FieldWorks treats as its default analysis writing system. +ANTHRO_DEFAULT_ANALYSIS_WS = "en" +UNLISTED_LANGUAGE_PREFIX = "qaa" + +# A writing system declared in both the vernacular and the analysis list gets taken out of one of +# them, so that each one has a single role in FieldWorks. Which one it leaves is decided by where +# its data actually is, not by which field the config happens to declare it in -- the config +# declaration can rest on a single stray value. One project declares "pt" on its lexeme field +# because one entry of 1248 has the vernacular headword duplicated into its "pt" alternative, while "pt" +# holds 1239 glosses, 1121 definitions and 1096 example translations. Going by the declaration would +# have made Portuguese vernacular-only and left the project with "en", which has three values in it. +# +# This writing system is never removed from the ANALYSIS list, whichever way its data leans. It is +# "en" for the same reason ANTHRO_DEFAULT_ANALYSIS_WS is: liblcm writes the anthropology list in the +# default analysis writing system and "en" is the only one the OCM file is guaranteed to have labels +# for, so a project left with no "en" in analysis can fail to be created at all. It can still be +# removed from the vernacular list, which is the usual outcome when a project glosses in English. +VERNACULAR_ANALYSIS_OVERLAP_EXEMPTION = "en" + +# Custom fields. LF names them by the level they live at -- LfMerge's MagicStrings has +# customField_entry / customField_senses / customField_example -- and the config nests them under +# entry.fields, senses.fields and senses.fields.examples.fields to match. +CUSTOM_FIELD_PREFIX = "customField_" + +# What a future LexBox API would need to create the field in FieldWorks. This is the inverse of +# LfMerge's CellarPropertyTypeToLfCustomFieldType (ConvertLcmToMongoCustomField.cs). That map is not +# injective -- MultiString and MultiUnicode both come out as multitext -- so the inverse picks +# MultiUnicode, which is what LF round-trips. +LCM_TYPE_FOR_LF_TYPE = { + "multitext": "MultiUnicode", + "optionlist": "ReferenceAtomic", + "multioptionlist": "ReferenceCollection", + "multiparagraph": "OwningAtomic", +} + +# Shapes LF writes into the lexicon, used to guess the type of a field the config never declared. +# A guess is not a declaration, so anything derived this way is reported as one. +# shape -> (LF type, LCM type) +LF_TYPE_FOR_VALUE_SHAPE = { + "multitext": ("multitext", "MultiUnicode"), + "single-listref": ("optionlist", "ReferenceAtomic"), + "multi-listref": ("multioptionlist", "ReferenceCollection"), + "paragraphs": ("multiparagraph", "OwningAtomic"), + # LF has no Date or Number field type; it stores those as a multitext under a fake writing + # system (LfMerge's MagicStrings.LanguageCodeForGenDateFields / ...ForIntFields). No project in + # the migration backlog uses either, but the shape is recognised so one would not be mistaken + # for text. + "date": ("date (fake ws qaa-Qaad)", "GenDate"), + "number": ("number (fake ws qaa-Zmth)", "Integer"), +} + +# Option lists FieldWorks already has, from LfMerge's MagicStrings.LcmOptionlistNames. A custom +# field pointing at anything else needs its possibility list created before the field can be. +STANDARD_LF_OPTION_LIST_CODES = frozenset({ + "grammatical-info", "semantic-domain-ddp4", "domain-type", "environments", "location", + "usage-type", "reversal-type", "sense-type", "anthro-code", "status", "etymology", + "lexical-relation", "note-type", "paradigm", "users", "translation-type", + "from-part-of-speech", "morph-type", "noun-slot", "verb-slot", "stative-slot", + "noun-infl-class", "verb-infl-class", +}) + +# Custom field names that FieldWorks may or may not treat as the same field. FLEx is inconsistent +# about two names differing only by case -- sometimes one field, sometimes two -- so the migration +# gives one of them a distinct name before it ever reaches FieldWorks. +# +# One project has "Class Marker" and "Class marker" at the entry level, and they are NOT +# duplicates. 830 entries fill only the first, 1279 only the second, and the one entry that fills +# both holds the same value in each, so nothing is lost either way -- but they hold different +# conventions. "Class Marker" is the older field, populated in 2014, and every one of its 832 +# values is a noun class morpheme ("mu", "ki") or the "##" placeholder, never a class number; +# "Class marker" is the 2016 field and mixes 1108 morphemes with 172 class numbers. Renaming rather +# than merging keeps the two apart so the distinction stays recoverable. +# +# Another project has "Morphophonemic Prop" and "MorphoPhonemic Prop" at the sense level, and here +# the two are not comparable at all. The lowercase-p one holds real data: 167 senses, all in "en", +# carrying a controlled vocabulary of vernacular morphophonemic properties ("underlong", "PMlowers", +# "RECIP; PMlowers; Mlowers; foreshortens") across 141 distinct values, nearly all written in 2014. +# The capital-P one has three senses, all in "es", holding ".", "x" and "do not load for CADA" -- +# a scratch field somebody made by mistyping the capitalisation, used as a marker and abandoned. +# So it is renamed out of the way rather than merged, which would fold three junk values into a +# clean linguistic field. No sense fills both, and the two use different writing systems, so a +# merge would in fact have been lossless; it just would not have been sensible. +# +# Keyed by field NAME rather than by project, since the project will not necessarily carry the same +# code or name in production as it does in the test snapshot. Nothing here is level-specific: a +# name is renamed wherever it clashes, and "Class Marker" happens to be an entry field while +# "MorphoPhonemic Prop" is a sense field. +CASE_CLASH_CUSTOM_FIELD_RENAMES = { + "Class Marker": "Class Marker Morpheme", + "MorphoPhonemic Prop": "MorphoPhonemic Prop Unused", +} + +# The LCM class each level's fields belong to, for the same future API. +LCM_CLASS_FOR_LEVEL = {"entry": "LexEntry", "senses": "LexSense", "examples": "LexExampleSentence"} + +# --- FLExProject.CustomProperties ------------------------------------------ # +# FieldWorks keeps its custom field definitions in FLExProject.CustomProperties, the one file in +# revision 0 of a FLEx Mercurial repository. LfMerge will not create them itself -- it logs +# "Custom field X from LF skipped, because we're not yet creating new custom fields in LCM" -- so +# the definitions have to be in the repository BEFORE the first Send/Receive, which is the only +# pass that would have carried the data. That is why a project with custom fields stops short of +# Send/Receive and prints this file for a human to commit. +# +# The format is copied from real Chorus repositories, not invented: CRLF line endings, tab +# indentation, one attribute per line in alphabetical order, elements sorted by class then name, +# and no trailing newline after . +CUSTOM_PROPERTIES_EOL = "\r\n" +CUSTOM_PROPERTIES_HEADER = '' +CUSTOM_PROPERTIES_FILENAME = "FLExProject.CustomProperties" + +# FieldWorks "magic" writing system selectors, all four of which occur here. The choice is only +# about which writing systems FLEx offers in the field; it does not change the field's type, which +# is always MultiUnicode because LF never had a single-string custom field. +# +# -1 and -2 name one writing system each, and are worth reaching for: a great many custom fields +# hold data in exactly one writing system, and a field showing one row rather than one row per +# writing system in a list is what that data actually is. +# +# The combined selectors matter because a field is not obliged to stay on one side of the project's +# vernacular/analysis divide. One project's "Top 10 list" holds Thai in 314 entries and English +# in 223; -4 alone would leave the English out of the field entirely. +WS_SELECTOR_FIRST_ANALYSIS = -1 +WS_SELECTOR_FIRST_VERNACULAR = -2 +WS_SELECTOR_ALL_ANALYSIS = -3 +WS_SELECTOR_ALL_VERNACULAR = -4 +WS_SELECTOR_ANALYSIS_THEN_VERNACULAR = -5 +WS_SELECTOR_VERNACULAR_THEN_ANALYSIS = -6 + +# destclass for the types that need one: 7 is CmPossibility, what a list reference points at, and +# 14 is StText, what a multiparagraph owns. +LCM_DEST_CLASS_FOR_TYPE = { + "ReferenceAtomic": 7, + "ReferenceCollection": 7, + "OwningAtomic": 14, +} + +# listRoot GUIDs of the possibility lists in a project FwHeadless creates, read from the +# NewLangProj.fwdata template it builds every project from. These are NOT universal -- a project +# made by an older FieldWorks has different GUIDs for the same lists, which is why they come from +# the template and not from some existing project. +# +# Only lists that are BOTH in the template and mapped to an LF list code by LfMerge +# (ConvertLcmToMongoCustomField.GuidToListCode) are here. grammatical-info and +# semantic-domain-ddp4 are mapped by LfMerge but PartsOfSpeech and SemanticDomainList are absent +# from the template, so a fresh project has no GUID to point at; any other list arrives as its +# NAME rather than a code, and has to be created by hand first. +LCM_LIST_ROOT_GUIDS = { + "anthro-code": "d87ba7c6-ea5e-11de-9dd2-0013722f8dec", + "domain-type": "b0a1eb98-ea5e-11de-888e-0013722f8dec", + "location": "d887937e-ea5e-11de-985e-0013722f8dec", + "sense-type": "af32c340-ea5e-11de-9040-0013722f8dec", + "status": "d86632ba-ea5e-11de-8a58-0013722f8dec", + "usage-type": "af542404-ea5e-11de-8bde-0013722f8dec", +} +LIST_ROOT_PLACEHOLDER = "FILL-IN-LIST-ROOT-GUID" + +# initFwDataProject builds a template .fwdata and pushes it, in a background job; +# this is how long to wait for that job to finish. It is NOT idempotent, so it +# is never retried. +DEFAULT_LEXBOX_TIMEOUT_SECONDS = 900 # 15 minutes + +# How long to wait for lfmergeqm to dequeue our request before giving up, and +# how long a whole Send/Receive may take. +PICKUP_TIMEOUT_SECONDS = 90 +DEFAULT_SR_TIMEOUT_SECONDS = 1800 +POLL_INTERVAL_SECONDS = 2.0 + + +# --------------------------------------------------------------------------- # +# configuration +# --------------------------------------------------------------------------- # + + +class Config: + """Connection and path settings, read from the environment.""" + + def __init__(self) -> None: + self.conn = os.environ.get("MONGODB_CONN", DEFAULT_MONGODB_CONN) + self.user = os.environ.get("MONGODB_USER", "") + self.password = os.environ.get("MONGODB_PASS", "") + self.authsource = os.environ.get("MONGODB_AUTHSOURCE", "admin") + self.database = os.environ.get("DATABASE", DEFAULT_DATABASE) + self.assets_root = os.environ.get("ASSETS_ROOT", DEFAULT_ASSETS_ROOT) + self.sendreceive_base = os.environ.get("LFMERGE_BASE_DIR", DEFAULT_SENDRECEIVE_BASE) + + @property + def syncqueue_path(self) -> str: + return os.path.join(self.sendreceive_base, "syncqueue") + + def queue_file(self, project_code: str) -> str: + # LfMerge's Queue.cs reads the FILENAME as the project code. + return os.path.join(self.syncqueue_path, project_code) + + def state_file(self, project_code: str) -> str: + # SendReceiveCommands.php:213 lowercases the code for the state file. + return os.path.join(self.sendreceive_base, "state", project_code.lower() + ".state") + + def webwork_path(self, project_code: str) -> str: + # LexProjectModel::getSendReceiveWorkFolder() lowercases the code. + return os.path.join(self.sendreceive_base, "webwork", project_code.lower()) + + def linked_files_path(self, project_code: str) -> str: + return os.path.join(self.webwork_path(project_code), "LinkedFiles") + + def assets_path(self, app_name: str, db_name: str) -> str: + return os.path.join(self.assets_root, app_name, db_name) + + def uri(self, database: str | None = None) -> str: + """Build a mongodb:// URI with credentials embedded. + + The Mongo CLI tools refuse to combine --uri with --username/--db, so + everything has to go into the URI itself. + """ + parts = urlsplit(self.conn) + netloc = parts.netloc + if "@" in netloc: # credentials already supplied in MONGODB_CONN + host = netloc + else: + host = netloc + if self.user: + host = f"{quote_plus(self.user)}:{quote_plus(self.password)}@{netloc}" + # The leading "/" is required before any query string, so always emit it. + path = "/" + (database or "") + query = parts.query + if self.authsource and "authSource" not in query: + query = f"{query}&authSource={self.authsource}" if query else f"authSource={self.authsource}" + return urlunsplit((parts.scheme or "mongodb", host, path, query, "")) + + def safe_uri(self, database: str | None = None) -> str: + """The URI with the password masked, for logging.""" + uri = self.uri(database) + if self.password: + uri = uri.replace(quote_plus(self.password), "****") + return uri + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # + + +def log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +class Fatal(Exception): + """An error that should abort the run (and trigger rollback).""" + + +def database_name(project_code: str) -> str: + """Replicate ProjectModel::databaseName() exactly (ProjectModel.php:115).""" + return "sf_" + project_code.lower().replace(" ", "_") + + +def send_receive_identifier(project_code: str) -> str: + """Derive a Send/Receive (LexBox) project code from a projectCode. + + Underscores become hyphens. An uppercase letter immediately following a + lowercase letter becomes a hyphen plus its lowercase form, which splits + CamelCase into words. Every other uppercase letter simply becomes lowercase, + so a run of capitals is not broken up. + + TitleCaseProject -> title-case-project + FOO_BAR -> foo-bar + + LexBox project codes must match ^[a-z\\d][a-z-\\d]*$, so this makes the + common Language Forge naming styles usable. It does not fix everything -- + spaces and other punctuation survive and are caught by validation. + """ + result = [] + for index, character in enumerate(project_code): + if character == "_": + result.append("-") + elif character.isupper(): + # "immediately follows a lowercase letter" is judged against the + # original string, before underscores were rewritten. + if index > 0 and project_code[index - 1].islower(): + result.append("-") + result.append(character.lower()) + else: + result.append(character) + return "".join(result) + + +def require_tool(name: str) -> None: + if shutil.which(name) is None: + raise Fatal( + f"Required tool '{name}' was not found on PATH. " + "This script is meant to run inside the app container, which provides " + "mongosh and mongodb-org-tools." + ) + + +def run(cmd: list[str], *, what: str) -> str: + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise Fatal(f"{what} failed (exit {result.returncode}):\n{result.stderr.strip()}") + return result.stdout + + +def mongosh_json(config: Config, js: str, *, what: str) -> Any: + """Run a mongosh script and parse the single JSON value it prints. + + The script is passed via a temp file so there is no shell quoting to get + wrong, and values are interpolated with json.dumps. + """ + require_tool("mongosh") + with tempfile.NamedTemporaryFile("w", suffix=".js", delete=False) as handle: + handle.write(js) + script_path = handle.name + try: + output = run(["mongosh", config.uri(), "--quiet", "--file", script_path], what=what) + finally: + os.unlink(script_path) + + for line in reversed(output.splitlines()): + line = line.strip() + if line: + try: + return json.loads(line) + except json.JSONDecodeError as exc: + raise Fatal(f"{what}: could not parse mongosh output {line!r}: {exc}") from exc + raise Fatal(f"{what}: mongosh produced no output") + + +def chown_to_web_user(path: str, *, recursive: bool = True) -> None: + """Best-effort chown to www-data, so LFMerge and Apache can use the files. + + LFMerge runs lfmergeqm as www-data, so anything this script drops into the + sendreceive tree has to be readable and writable by that user. + """ + try: + import grp + import pwd + + uid = pwd.getpwnam(WEB_USER).pw_uid + gid = grp.getgrnam(WEB_USER).gr_gid + except (ImportError, KeyError): + return + + def attempt(target: str) -> None: + try: + os.chown(target, uid, gid, follow_symlinks=False) + except (PermissionError, FileNotFoundError, NotImplementedError, TypeError): + try: + os.chown(target, uid, gid) + except (PermissionError, FileNotFoundError): + pass + + attempt(path) + if recursive and os.path.isdir(path) and not os.path.islink(path): + for root, dirnames, filenames in os.walk(path): + for name in dirnames + filenames: + attempt(os.path.join(root, name)) + + +# --------------------------------------------------------------------------- # +# mongo steps +# --------------------------------------------------------------------------- # + + +def fetch_project(config: Config, project_code: str) -> dict | None: + js = f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const doc = main.projects.findOne({{ projectCode: {json.dumps(project_code)} }}); +print(EJSON.stringify(doc === null ? {{ found: false }} : {{ + found: true, + id: doc._id.toString(), + projectCode: doc.projectCode, + projectName: doc.projectName || "", + appName: doc.appName || "", + siteName: doc.siteName || "", + isArchived: doc.isArchived === true, + sendReceiveProjectIdentifier: doc.sendReceiveProjectIdentifier || "", + memberCount: Object.keys(doc.users || {{}}).length, +}})); +""" + result = mongosh_json(config, js, what="Reading project record") + return result if result.get("found") else None + + +def dedup(tags: list[str]) -> list[str]: + """Drop duplicates and blanks, preserving first-seen order.""" + seen: set[str] = set() + result = [] + for tag in tags: + tag = (tag or "").strip() + if tag and tag not in seen: + seen.add(tag) + result.append(tag) + return result + + +def fetch_writing_systems(config: Config, project_code: str) -> dict[str, list[str]]: + """Read the ordered vernacular and analysis writing system tags from the config. + + Language Forge has no vernacular/analysis flag anywhere -- the distinction is + conventional, encoded in which config field a tag appears under: + + vernacular: config.entry.fields.lexeme.inputSystems (index 0 is the + project's primary language, per LexProjectModel.php:38) + plus config.entry.fields.citationForm.inputSystems + analysis: config.entry.fields.senses.fields.definition.inputSystems + plus ...senses.fields.gloss.inputSystems + + The lists are ordered BSON arrays of hyphenated BCP-47 tags and are passed + through verbatim. The secondary fields are unioned in because a project may + populate gloss but not definition (or citationForm but not lexeme), and a + writing system missing from the FieldWorks project could not receive data. + Order is preserved, primary field first. + + Also returns the complete set of writing systems the project knows about, so + the caller can check that vernacular + analysis covers all of them: + + config_all: every inputSystems entry anywhere under config.entry, found + by walking the whole field tree (fields can nest, e.g. + senses.examples.sentence). Ordered by fieldOrder where the + config provides one, so the order is the config's own. + declared: the keys of project.inputSystems, which is the project's map + of writing system definitions (InputSystem.php). This is a + BSON document, so its order is insertion order and is not + meaningful -- it is only used to catch writing systems that + no config field references. + """ + js = f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const doc = main.projects.findOne( + {{ projectCode: {json.dumps(project_code)} }}, + {{ config: 1, inputSystems: 1 }} +); +const entry = (((doc || {{}}).config || {{}}).entry) || {{}}; +const fields = entry.fields || {{}}; +const senses = ((fields.senses || {{}}).fields) || {{}}; + +function tags(field) {{ + return (field && Array.isArray(field.inputSystems)) ? field.inputSystems : []; +}} + +// Walk the entire config field tree, collecting every inputSystems entry. +// fieldOrder is the order the user sees, so honour it and append any field not +// listed there; nested field lists (senses, examples) are recursed into. +const configAll = []; +function walk(fieldList) {{ + if (!fieldList || typeof fieldList !== "object") return; + const children = fieldList.fields || {{}}; + const order = Array.isArray(fieldList.fieldOrder) ? fieldList.fieldOrder : []; + const names = []; + order.forEach(function (name) {{ + if (children[name] && names.indexOf(name) === -1) names.push(name); + }}); + Object.keys(children).forEach(function (name) {{ + if (names.indexOf(name) === -1) names.push(name); + }}); + names.forEach(function (name) {{ + const child = children[name]; + tags(child).forEach(function (tag) {{ configAll.push(tag); }}); + if (child && child.fields) walk(child); + }}); +}} +walk(entry); + +print(EJSON.stringify({{ + lexeme: tags(fields.lexeme), + citationForm: tags(fields.citationForm), + definition: tags(senses.definition), + gloss: tags(senses.gloss), + configAll: configAll, + declared: Object.keys((doc || {{}}).inputSystems || {{}}), +}})); +""" + found = mongosh_json(config, js, what="Reading writing systems from project config") + return { + "lexeme": list(found["lexeme"]), + "citationForm": list(found["citationForm"]), + "definition": list(found["definition"]), + "gloss": list(found["gloss"]), + "config_all": list(found["configAll"]), + "declared": list(found["declared"]), + } + + +def fetch_custom_fields(config: Config, project_code: str) -> dict[str, dict]: + """The custom fields the project config declares, keyed by LF field name. + + Each value carries the level the field lives at, the LF `type` discriminator, the label, the + writing systems the field uses and, for list-backed fields, the option list code. + + The config is authoritative for a field's TYPE, but not for whether it is used: 8 fields in the + 2026-07-06 corpus are declared and never populated, and 32 are populated without being declared. + scan_custom_field_usage() supplies the other half. + """ + js = f""" +const mainDb = {json.dumps(config.database)}; +const projectCode = {json.dumps(project_code)}; +const prefix = {json.dumps(CUSTOM_FIELD_PREFIX)}; +""" + """ +const main = db.getSiblingDB(mainDb); +const doc = main.projects.findOne({ projectCode: projectCode }, { config: 1 }); +const entry = (((doc || {}).config || {}).entry) || {}; + +const found = {}; + +// The three levels are spelled out rather than walked generically, so that the level recorded for +// each field is exactly the one its name is prefixed with. +function collect(fieldList, level) { + const fields = (fieldList || {}).fields || {}; + Object.keys(fields).forEach(function (name) { + if (name.indexOf(prefix) !== 0) return; + const field = fields[name] || {}; + found[name] = { + level: level, + type: field.type || null, + label: field.label || null, + inputSystems: Array.isArray(field.inputSystems) ? field.inputSystems : [], + listCode: field.listCode || null + }; + }); + return fields; +} + +const entryFields = collect(entry, "entry"); +const senseFields = collect(entryFields.senses, "senses"); +collect(senseFields.examples, "examples"); + +print(EJSON.stringify(found)); +""" + return mongosh_json(config, js, what=f"Reading custom field config for {project_code}") + + +def scan_custom_field_usage(config: Config, db_name: str) -> dict[str, dict]: + """How each custom field is actually used in db_name.lexicon, keyed by LF field name. + + Reports, per field: the level it was seen at, how many entries carry a value, how many values + in total, the value shapes observed (so a field the config does not declare can still be typed), + the writing systems its values use, and how many entries carry data in each of them. + + The per-writing-system counts are what a human uses to check a guess: a field whose data is + almost all in the vernacular writing system is a vernacular field whatever the config says. + + Shape detection mirrors what LF writes: a multitext is {ws: {value: ...}}, a single option list + reference is {value: ...}, a multiple one is {values: [...]}. + """ + js = f""" +const targetName = {json.dumps(db_name)}; +""" + """ +const target = db.getSiblingDB(targetName); +const found = {}; + +function isEmpty(value) { + if (value === null || value === undefined) return true; + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "string") return value === ""; + if (typeof value === "object") { + if (Object.prototype.hasOwnProperty.call(value, "values")) return (value.values || []).length === 0; + if (Object.prototype.hasOwnProperty.call(value, "value")) return value.value === null || value.value === ""; + return Object.keys(value).every(function (k) { return isEmpty(value[k]); }); + } + return false; +} + +function shapeOf(value) { + if (value === null || typeof value !== "object") return "scalar"; + if (Array.isArray(value)) return "array"; + // Order matters: a multitext has no "value"/"values" key of its own, only writing systems. + if (Object.prototype.hasOwnProperty.call(value, "values")) return "multi-listref"; + if (Object.prototype.hasOwnProperty.call(value, "value")) return "single-listref"; + if (Object.prototype.hasOwnProperty.call(value, "paragraphs")) return "paragraphs"; + const keys = Object.keys(value); + if (keys.indexOf("qaa-Qaad") !== -1) return "date"; + if (keys.indexOf("qaa-Zmth") !== -1) return "number"; + const allMultitext = keys.length > 0 && keys.every(function (k) { + const inner = value[k]; + return inner && typeof inner === "object" && Object.prototype.hasOwnProperty.call(inner, "value"); + }); + return allMultitext ? "multitext" : "unrecognised"; +} + +function record(level, holder, seenThisEntry) { + const fields = (holder || {}).customFields || {}; + Object.keys(fields).forEach(function (name) { + const value = fields[name]; + if (isEmpty(value)) return; + if (!found[name]) { + found[name] = { level: level, entries: 0, values: 0, shapes: [], writingSystems: [], wsEntries: {} }; + } + const record_ = found[name]; + record_.values += 1; + const shape = shapeOf(value); + if (record_.shapes.indexOf(shape) === -1) record_.shapes.push(shape); + if (!seenThisEntry[name]) seenThisEntry[name] = {}; + if (shape === "multitext" || shape === "date" || shape === "number") { + Object.keys(value).forEach(function (ws) { + // An empty alternative is not use of that writing system, matching isEmpty() above. + if (isEmpty(value[ws])) return; + if (record_.writingSystems.indexOf(ws) === -1) record_.writingSystems.push(ws); + seenThisEntry[name][ws] = true; + }); + } + }); +} + +if (target.getCollectionNames().indexOf("lexicon") !== -1) { + target.lexicon.find({}, { customFields: 1, senses: 1 }).forEach(function (doc) { + const seenThisEntry = {}; + record("entry", doc, seenThisEntry); + (doc.senses || []).forEach(function (sense) { + if (!sense || typeof sense !== "object") return; + record("senses", sense, seenThisEntry); + (sense.examples || []).forEach(function (example) { + if (example && typeof example === "object") record("examples", example, seenThisEntry); + }); + }); + // entries counts entries, not values: a field used twice in one entry still counts once, and + // the same goes for each of its writing systems. + Object.keys(seenThisEntry).forEach(function (name) { + found[name].entries += 1; + Object.keys(seenThisEntry[name]).forEach(function (ws) { + found[name].wsEntries[ws] = (found[name].wsEntries[ws] || 0) + 1; + }); + }); + }); +} + +print(EJSON.stringify(found)); +""" + return mongosh_json(config, js, what=f"Scanning custom field usage in {db_name}") + + +def strip_custom_field_prefix(name: str, level: str) -> str: + """The bare field name, with LF's "customField__" prefix removed. + + LfMerge builds these names in ConvertUtilities.NormalizedFieldName as + customField__, so the level has to + come off as well as the "customField_" part. Whatever is left is the LCM field name with its + spaces still encoded; report_custom_fields() puts those back. + """ + for candidate in (f"{CUSTOM_FIELD_PREFIX}{level}_", CUSTOM_FIELD_PREFIX): + if name.startswith(candidate): + return name[len(candidate):] + return name + + +def describe_custom_fields(declared: dict[str, dict], used: dict[str, dict]) -> list[dict]: + """Merge the config and the data into one row per custom field, sorted for reporting. + + A field declared in the config takes its type from there. A field only found in the data has its + type guessed from the value shape, and is marked as a guess -- a guess is a reasonable basis for + a report and a poor basis for creating a field in FieldWorks, where the type is hard to undo. + """ + rows = [] + for name in sorted(set(declared) | set(used)): + config_entry = declared.get(name, {}) + usage = used.get(name, {}) + level = config_entry.get("level") or usage.get("level") or "?" + shapes = list(usage.get("shapes") or []) + + lf_type = config_entry.get("type") + guessed = False + if lf_type: + lcm_type = LCM_TYPE_FOR_LF_TYPE.get(lf_type) + elif len(shapes) == 1: + lf_type, lcm_type = LF_TYPE_FOR_VALUE_SHAPE.get(shapes[0], (None, None)) + guessed = True + else: + lf_type, lcm_type = None, None + guessed = bool(shapes) + + rows.append({ + "name": name, + "short_name": strip_custom_field_prefix(name, level), + "level": level, + "lf_type": lf_type, + "lcm_type": lcm_type, + "lcm_class": LCM_CLASS_FOR_LEVEL.get(level), + "guessed": guessed, + "declared": name in declared, + "label": config_entry.get("label"), + "list_code": config_entry.get("listCode"), + "input_systems": config_entry.get("inputSystems") or usage.get("writingSystems") or [], + "ws_entries": usage.get("wsEntries") or {}, + "entries": usage.get("entries", 0), + "values": usage.get("values", 0), + "shapes": shapes, + }) + order = {"entry": 0, "senses": 1, "examples": 2} + rows.sort(key=lambda row: (order.get(row["level"], 9), -row["entries"], row["name"])) + return rows + + +def report_custom_fields(rows: list[dict]) -> None: + """Print the custom fields, and what a future LexBox API would need in order to create them.""" + if not rows: + log("Custom fields: none") + return + + log(f"Custom fields ({len(rows)}):") + for row in rows: + detail = f"list={row['list_code']}" if row["list_code"] else ",".join(row["input_systems"]) + flags = [] + if row["guessed"]: + flags.append("GUESSED from data, not in config") + elif not row["declared"]: + flags.append("not in config") + if row["entries"] == 0: + flags.append("declared but unused") + if row["list_code"] and row["list_code"] not in STANDARD_LF_OPTION_LIST_CODES: + flags.append(f"option list '{row['list_code']}' is not one FieldWorks ships") + if row["lf_type"] is None: + flags.append(f"type undetermined (shapes seen: {', '.join(row['shapes']) or 'none'})") + log(f" {row['level']:8s} {row['short_name']:34s} {str(row['lf_type']):24s} " + f"{detail:18s} {row['entries']:6d} entries" + + (" [" + "; ".join(flags) + "]" if flags else "")) + + guessed = [row for row in rows if row["guessed"]] + lists_needed = sorted({row["list_code"] for row in rows + if row["list_code"] and row["list_code"] not in STANDARD_LF_OPTION_LIST_CODES}) + log("") + log(" To create these in LexBox, an API would need per field: LCM class, field name, type,") + log(" writing systems, and for list fields the possibility list. As derived here:") + for row in rows: + if row["lcm_type"] is None: + continue + field_name = row["short_name"].replace("_", " ") + log(f" {row['lcm_class'] or '?':19s} {field_name:34s} {row['lcm_type']:19s}" + + (f" list={row['list_code']}" if row["list_code"] else "") + + (" (type guessed)" if row["guessed"] else "")) + if guessed: + log("") + log(f" WARNING: {len(guessed)} field(s) above are typed by GUESS from the value shape, because") + log(" the project config does not declare them. Confirm each before anything creates") + log(" it in FieldWorks: a wrong custom field type is painful to undo once synced.") + if lists_needed: + log("") + log(f" WARNING: {len(lists_needed)} option list(s) are not ones FieldWorks ships and would have to be") + log(f" created first: {', '.join(lists_needed)}") + + +def plan_custom_field_renames(rows: list[dict]) -> tuple[list[dict], list[str]]: + """Work out which custom fields have to be renamed, and which case clashes are unhandled. + + Returns (renames, problems). Each rename carries the level, the Mongo key to rename and the key + to rename it to, plus the FieldWorks names either side for reporting. + + Two custom fields whose names differ only by case are a hazard, because FLEx is inconsistent + about whether it treats them as one field or two. Only the clashes named in + CASE_CLASH_CUSTOM_FIELD_RENAMES are handled; anything else is reported for a human rather than + guessed at, since which of the two should keep its name -- and whether they even hold the same + kind of data -- is not derivable from the names. + + Names are compared within a level. A field on LexEntry and one on LexSense do not collide even + when identically named, because FieldWorks keys a custom field by its class as well as its name. + """ + groups: dict[tuple[str, str], list[dict]] = {} + for row in rows: + groups.setdefault((row["level"], row["short_name"].lower()), []).append(row) + taken = set(groups) + + renames: list[dict] = [] + problems: list[str] = [] + for (level, _lowered), group in sorted(groups.items()): + spellings = {row["short_name"] for row in group} + if len(spellings) < 2: + continue + + renamed: list[dict] = [] + for row in sorted(group, key=lambda item: item["short_name"]): + new_name = CASE_CLASH_CUSTOM_FIELD_RENAMES.get(row["short_name"].replace("_", " ")) + if new_name is None: + continue + new_short = new_name.replace(" ", "_") + if (level, new_short.lower()) in taken: + problems.append( + f"{level}: cannot rename {row['short_name'].replace('_', ' ')!r} to " + f"{new_name!r} because the project already has a field of that name" + ) + continue + # Take the prefix off the row's own key rather than rebuilding it, so whatever spelling + # of customField__ this project uses is preserved exactly. + prefix = row["name"][: len(row["name"]) - len(row["short_name"])] + renames.append({ + "level": level, + "old_key": row["name"], + "new_key": prefix + new_short, + "old_name": row["short_name"].replace("_", " "), + "new_name": new_name, + }) + taken.add((level, new_short.lower())) + renamed.append(row) + + left = sorted(spellings - {row["short_name"] for row in renamed}) + if len(left) > 1: + problems.append( + f"{level}: these custom field names differ only by case and none of them is one " + f"this script knows how to rename: " + + ", ".join(repr(name.replace("_", " ")) for name in left) + ) + return renames, problems + + +def apply_custom_field_renames(rows: list[dict], renames: list[dict]) -> None: + """Rewrite the described rows in place so everything downstream sees the new names. + + The FieldWorks definitions -- and so FLExProject.CustomProperties -- are built from these rows. + A definition under the old name would not match the data the migration is about to write, and + LfMerge would skip the field. + """ + by_key = {(rename["level"], rename["old_key"]): rename for rename in renames} + for row in rows: + rename = by_key.get((row["level"], row["name"])) + if rename is not None: + row["name"] = rename["new_key"] + row["short_name"] = rename["new_name"].replace(" ", "_") + + +def report_custom_field_renames(renames: list[dict], problems: list[str]) -> None: + """Say what will be renamed, and what cannot be.""" + for rename in renames: + log(f" {rename['level']:8s} {rename['old_name']!r} -> {rename['new_name']!r}" + f" ({rename['old_key']} -> {rename['new_key']})") + for problem in problems: + log(f" ERROR: {problem}") + + +def rename_custom_fields_in_lexicon( + config: Config, db_name: str, renames: list[dict] +) -> dict: + """Rename custom field keys throughout db_name.lexicon, in the COPY only. + + Renames customFields and customFieldGuids alike, at whichever level the field lives at. The + whole document is rewritten rather than updated with $rename, which cannot reach a key inside + an array element; nothing else about it is touched, so dateModified and the dirtySR flags keep + saying when the entry was last really worked on. + + A container that somehow already holds the new key is reported and left alone rather than + overwritten, which would lose whichever value was there. + """ + if not renames: + return {"entriesChanged": 0, "keysRenamed": 0, "conflicts": []} + + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +const renames = {json.dumps(renames)}; +""" + """ +// The containers a rename applies to, by the level the field was declared at. +function containersFor(doc, level) { + if (level === "entry") return [doc]; + const senses = (doc.senses || []).filter(function (s) { return s && typeof s === "object"; }); + if (level === "senses") return senses; + const examples = []; + senses.forEach(function (sense) { + (sense.examples || []).forEach(function (example) { + if (example && typeof example === "object") examples.push(example); + }); + }); + return examples; +} + +let entriesChanged = 0; +let keysRenamed = 0; +const conflicts = []; + +target.lexicon.find({}).forEach(function (doc) { + let dirty = false; + renames.forEach(function (rename) { + containersFor(doc, rename.level).forEach(function (container) { + ["customFields", "customFieldGuids"].forEach(function (holder) { + const fields = container[holder]; + if (!fields || typeof fields !== "object") return; + if (!Object.prototype.hasOwnProperty.call(fields, rename.old_key)) return; + if (Object.prototype.hasOwnProperty.call(fields, rename.new_key)) { + conflicts.push({ id: doc._id.toString(), holder: holder, key: rename.new_key }); + return; + } + fields[rename.new_key] = fields[rename.old_key]; + delete fields[rename.old_key]; + keysRenamed += 1; + dirty = true; + }); + }); + }); + if (dirty) { + target.lexicon.replaceOne({ _id: doc._id }, doc); + entriesChanged += 1; + } +}); + +print(EJSON.stringify({ + entriesChanged: entriesChanged, + keysRenamed: keysRenamed, + conflicts: conflicts +})); +""" + result = mongosh_json(config, js, what=f"Renaming custom fields in {db_name}") + if result["conflicts"]: + raise Fatal( + f"{len(result['conflicts'])} container(s) already held the renamed custom field key, " + "so the rename was abandoned there rather than overwrite a value: " + + ", ".join(f"{c['id']}.{c['holder']}.{c['key']}" for c in result["conflicts"][:5]) + ) + log(f" {result['keysRenamed']} custom field key(s) renamed in " + f"{result['entriesChanged']} entry/entries") + return result + + +def derive_ws_selector( + field_ws: list[str], vernacular: list[str], analysis: list[str], ws_entries: dict[str, int] +) -> tuple[int, bool, str]: + """Pick FieldWorks' magic writing system selector for a text custom field. + + A field holding data in exactly one writing system gets the selector naming just that one, when + it leads a list: -2 for the first vernacular writing system, -1 for the first analysis one. Most + text custom fields in these projects are like this, and the single-writing-system selectors are + what makes them look right in FLEx rather than showing an empty row per writing system in a + list. A writing system leading both lists takes -2, since either selector resolves to it. + + Otherwise -4 covers every vernacular writing system of the project and -3 every analysis one, so + a field whose writing systems sit entirely inside one of the two lists gets that list's selector. + + A field that spans both gets a combined selector instead: -6 for every vernacular writing + system followed by every analysis one, -5 for the other order. Both cover the same set, so the + choice is only about which the user sees first, and that is settled by where the data actually + is -- the writing systems holding the most entries lead. FieldWorks still has no selector + meaning "these particular writing systems", so a combined selector brings in the project's + other writing systems too; that is harmless next to dropping the ones a single-sided selector + would leave out. + + The selector is not recoverable from LF: LfMerge expanded it into a plain list of writing + systems on the way in (ConvertLcmToMongoCustomField), so all that survives is where those + writing systems ended up. Returns (selector, confident, why not). + """ + vern, anal = set(vernacular), set(analysis) + tags = set(field_ws) + + # Checked before anything else, and against the DATA rather than the declaration: a field with + # one writing system's worth of data can be named exactly, and that beats any list selector. + with_data = [tag for tag, count in ws_entries.items() if count > 0] + if len(with_data) == 1: + only = with_data[0] + selector = None + if vernacular and only == vernacular[0]: + selector = WS_SELECTOR_FIRST_VERNACULAR + elif analysis and only == analysis[0]: + selector = WS_SELECTOR_FIRST_ANALYSIS + if selector is not None: + # Narrowing is possible: a field declaring several writing systems but holding data in + # one keeps only that one in FLEx. No data is lost, but say so, since it is a decision. + wider = sorted(tags - {only}) + if wider: + return (selector, False, + f"all its data is in '{only}', so {selector} names just that one; the " + f"config also lists {', '.join(wider)}, which the field will no longer offer") + return selector, True, "" + + if not tags: + return WS_SELECTOR_ALL_VERNACULAR, False, "the field declares no writing systems at all" + within_vernacular, within_analysis = tags <= vern, tags <= anal + if within_vernacular and not within_analysis: + return WS_SELECTOR_ALL_VERNACULAR, True, "" + if within_analysis and not within_vernacular: + return WS_SELECTOR_ALL_ANALYSIS, True, "" + if within_vernacular and within_analysis: + # Covered either way, so the selector is not in doubt; which side the field belongs to is, + # and nothing in the config can settle it. + return (WS_SELECTOR_ALL_VERNACULAR, False, + "every writing system it uses is both vernacular and analysis in this project") + + # Neither single-sided selector covers all of it. A combined one reaches both lists, so the + # only remaining question is which side leads: the one holding more of this field's data. A + # writing system on both sides counts for both and so cannot break the tie. + # Even a combined selector only reaches writing systems the project knows about, though. + uncovered = tags - vern - anal + if not (tags & vern) and not (tags & anal): + # Every writing system it uses is outside both lists, so no selector reaches any of them. + # The selector is not the thing to fix here; the project's writing systems are. + return (WS_SELECTOR_VERNACULAR_THEN_ANALYSIS, False, + f"none of its writing systems ({', '.join(sorted(tags))}) are in the project's " + f"vernacular or analysis lists, so no selector reaches them") + + vernacular_entries = sum(ws_entries.get(tag, 0) for tag in tags & vern) + analysis_entries = sum(ws_entries.get(tag, 0) for tag in tags & anal) + if analysis_entries > vernacular_entries: + selector, leading = WS_SELECTOR_ANALYSIS_THEN_VERNACULAR, "analysis" + else: + selector, leading = WS_SELECTOR_VERNACULAR_THEN_ANALYSIS, "vernacular" + + why = f"neither -3 nor -4 covers all of its writing systems, so {selector} takes in both lists" + if analysis_entries == vernacular_entries: + why += (f" and leads with the {leading} ones, the two sides holding the same amount of its " + f"data ({vernacular_entries} entries each)") + else: + why += (f" and leads with the {leading} ones, which hold more of its data " + f"({max(vernacular_entries, analysis_entries)} entries vs " + f"{min(vernacular_entries, analysis_entries)})") + if uncovered: + plural = len(uncovered) > 1 + why += (f"; {', '.join(sorted(uncovered))} " + f"{'are' if plural else 'is'} in neither list and " + f"{'stay' if plural else 'stays'} uncovered") + return selector, False, why + + +def plan_custom_field_definitions( + rows: list[dict], vernacular: list[str], analysis: list[str] +) -> tuple[list[dict], list[tuple[dict, str]]]: + """Turn described custom fields into FieldWorks definitions, and say what was left out. + + Returns (definitions, dropped). A definition carries the attributes FieldWorks writes and the + comment lines that belong above it; `dropped` pairs each omitted field with the reason, so the + caller can report rather than silently lose it. + """ + definitions: list[dict] = [] + dropped: list[tuple[dict, str]] = [] + + for row in rows: + lcm_type, lcm_class = row["lcm_type"], row["lcm_class"] + if lcm_type is None: + dropped.append((row, f"type undetermined (shapes seen: {', '.join(row['shapes']) or 'none'})")) + continue + if lcm_class is None: + dropped.append((row, f"unrecognised level {row['level']!r}")) + continue + + # LF never offered the single-string custom field type, so every text field it holds is a + # MultiUnicode that happens to have one writing system -- never a FieldWorks String. + is_text = lcm_type == "MultiUnicode" + if not is_text and row["entries"] == 0: + # A list or paragraph field with no data buys nothing and costs a possibility list that + # a new project may not even have. There is no data to lose by leaving it out. + dropped.append((row, "no data, and not a text field")) + continue + + name = row["short_name"].replace("_", " ") + attributes: dict[str, str] = { + "class": lcm_class, + "key": f"{lcm_class}{name}", + "name": name, + "type": lcm_type, + } + dest_class = LCM_DEST_CLASS_FOR_TYPE.get(lcm_type) + if dest_class is not None: + attributes["destclass"] = str(dest_class) + # LF's label is the only human-readable description that survives the round trip; FieldWorks + # keeps its own helpString, which LfMerge never carried across. Only worth writing when it + # says something the field name does not. + label = (row["label"] or "").strip() + if label and label != name: + attributes["helpString"] = label + + notes: list[str] = [] + needs_review = False + + if row["list_code"]: + list_root = LCM_LIST_ROOT_GUIDS.get(row["list_code"]) + if list_root is None: + list_root = LIST_ROOT_PLACEHOLDER + needs_review = True + notes.append( + f"REVIEW: option list {row['list_code']!r} has no known GUID in a newly created " + f"FieldWorks project. Create the list, then replace listRoot below with its GUID." + ) + attributes["listRoot"] = list_root + + if is_text: + selector, confident, why = derive_ws_selector( + row["input_systems"], vernacular, analysis, row["ws_entries"]) + attributes["wsSelector"] = str(selector) + if not confident: + needs_review = True + notes.append(f"REVIEW: wsSelector guessed as {selector}, because {why}.") + + if row["guessed"]: + needs_review = True + notes.append( + f"REVIEW: type GUESSED as {lcm_type} from the data (shapes seen: " + f"{', '.join(row['shapes']) or 'none'}); the LF config does not declare this field." + ) + + if needs_review: + notes.append(_ws_usage_note(row)) + + definitions.append({"row": row, "attributes": attributes, "notes": notes, + "needs_review": needs_review}) + + return definitions, dropped + + +def _ws_usage_note(row: dict) -> str: + """How many entries hold data in each writing system -- the evidence for checking a guess.""" + counts = row["ws_entries"] + if not counts: + return f"{row['entries']} entry/entries carry data, none of it in a named writing system" + detail = ", ".join( + f"{ws}={counts[ws]}" for ws in sorted(counts, key=lambda tag: (-counts[tag], tag)) + ) + return f"entries with data per writing system: {detail}" + + +def xml_comment_text(text: str) -> str: + """Make text safe inside an XML comment, which may not contain "--" nor end with "-".""" + text = " ".join(text.split()) + while "--" in text: + text = text.replace("--", "- -") + return f"{text} " if text.endswith("-") else text + + +def build_custom_properties_xml(definitions: list[dict]) -> str: + """Render FLExProject.CustomProperties exactly as FieldWorks writes it. + + Elements sorted by class then name, attributes alphabetical and one per line, tab indented, + CRLF throughout and no trailing newline -- all copied from real Chorus repositories so that the + first commit a human makes looks like one FieldWorks would have made. + """ + lines = [CUSTOM_PROPERTIES_HEADER, ""] + for definition in sorted( + definitions, key=lambda item: (item["attributes"]["class"], item["attributes"]["name"]) + ): + for note in definition["notes"]: + lines.append(f"\t") + lines.append("\t") + return CUSTOM_PROPERTIES_EOL.join(lines) + + +def write_custom_properties(xml: str, path: str) -> str: + """Write the XML to path, or path.1 / path.2 / ... Never overwrites an existing file. + + O_EXCL rather than an exists() check, so that two runs racing each other still get a file each. + newline="" keeps the CRLF endings already in the string from being translated again. + """ + candidate, index = path, 0 + while True: + try: + handle = os.open(candidate, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError: + index += 1 + candidate = f"{path}.{index}" + continue + with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: + stream.write(xml) + return candidate + + +def report_custom_field_definitions( + definitions: list[dict], dropped: list[tuple[dict, str]] +) -> None: + """List the fields left out of the XML, and the ones whose definition is a guess.""" + if dropped: + log("") + log(f" {len(dropped)} field(s) deliberately NOT included:") + for row, reason in dropped: + log(f" {row['level']:8s} {row['short_name']:34s} {reason}") + review = [definition for definition in definitions if definition["needs_review"]] + if review: + log("") + log(f" {len(review)} definition(s) carry a REVIEW comment and need checking before you commit:") + for definition in review: + log(f" {definition['attributes']['class']:19s} {definition['attributes']['name']}") + + +def log_custom_properties_xml(xml: str) -> None: + """Print the XML between rules, so it can be read or copied straight out of the run.""" + log("") + log(f"--- {CUSTOM_PROPERTIES_FILENAME} " + "-" * 44) + # Printed with the platform's own line endings; a written file keeps the CRLF FieldWorks uses. + log(xml.replace(CUSTOM_PROPERTIES_EOL, "\n")) + log("-" * 78) + + +def emit_custom_properties( + definitions: list[dict], + dropped: list[tuple[dict, str]], + new_code: str, + lexbox_code: str, + lexbox_host: str, +) -> str: + """Write and print FLExProject.CustomProperties, and say what the human has to do with it.""" + xml = build_custom_properties_xml(definitions) + path = write_custom_properties(xml, f"{new_code}.{CUSTOM_PROPERTIES_FILENAME}") + + log("") + log("=" * 78) + log(f"STOPPING: {len(definitions)} custom field definition(s) must exist in FieldWorks first.") + log("=" * 78) + log("") + log("LfMerge does not create custom fields -- it skips them, and the data in them, with") + log("\"Custom field X from LF skipped, because we're not yet creating new custom fields in") + log("LCM\". The first Send/Receive is the only pass that would carry this project's data, so") + log("the definitions have to be in the repository before it runs. Nothing has been pushed.") + log("") + log(f"Written to: {os.path.abspath(path)}") + + report_custom_field_definitions(definitions, dropped) + + scheme, host = split_lexbox_host(lexbox_host) + log("") + log("To create them, commit the file to the project's Mercurial repository:") + log(f" hg clone {scheme}://{host}/hg/{lexbox_code} {lexbox_code}") + log(f" cp {path} {lexbox_code}/{CUSTOM_PROPERTIES_FILENAME}") + log(f" cd {lexbox_code} && hg commit -m 'Add custom fields' && hg push") + + log_custom_properties_xml(xml) + return path + + +def count_anchor_writing_system_usage(config: Config, db_name: str) -> dict[str, dict]: + """How many entries hold data in each writing system, split by vernacular or analysis field. + + Returns {tag: {"vernacular": n, "analysis": n}}, counting entries rather than values so a + writing system used three times in one entry still counts once, the same way the custom field + scan counts. + + The fields are the ones whose role is not in doubt: + + vernacular lexeme, citationForm, senses.examples.sentence + analysis senses.definition, senses.gloss, senses.examples.translation + + etymology is deliberately absent. In the 2026-07-06 corpus 741 projects configure it with + analysis writing systems and 378 with vernacular ones, so it is evidence of nothing. + """ + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +""" + """ +const counts = {}; + +function isFilled(alternative) { + return alternative !== null && typeof alternative === "object" + && alternative.value !== null && alternative.value !== undefined && alternative.value !== ""; +} + +// Records into a per-entry set, so an entry using a writing system repeatedly counts once. +function record(multitext, side, seen) { + if (!multitext || typeof multitext !== "object") return; + Object.keys(multitext).forEach(function (ws) { + if (isFilled(multitext[ws])) seen[side][ws] = true; + }); +} + +if (target.getCollectionNames().indexOf("lexicon") !== -1) { + target.lexicon.find({}).forEach(function (doc) { + const seen = { vernacular: {}, analysis: {} }; + record(doc.lexeme, "vernacular", seen); + record(doc.citationForm, "vernacular", seen); + (doc.senses || []).forEach(function (sense) { + if (!sense || typeof sense !== "object") return; + record(sense.definition, "analysis", seen); + record(sense.gloss, "analysis", seen); + (sense.examples || []).forEach(function (example) { + if (!example || typeof example !== "object") return; + record(example.sentence, "vernacular", seen); + record(example.translation, "analysis", seen); + }); + }); + ["vernacular", "analysis"].forEach(function (side) { + Object.keys(seen[side]).forEach(function (ws) { + if (!counts[ws]) counts[ws] = { vernacular: 0, analysis: 0 }; + counts[ws][side] += 1; + }); + }); + }); +} + +print(EJSON.stringify(counts)); +""" + return mongosh_json(config, js, what=f"Counting writing system usage in {db_name}") + + +def classify_writing_systems( + found: dict[str, list[str]], + override_vernacular: list[str] | None = None, + override_analysis: list[str] | None = None, + anchor_counts: dict[str, dict] | None = None, +) -> tuple[list[str], list[str], list[str]]: + """Split the project's writing systems into vernacular and analysis. + + Returns (vernacular, analysis, unclassified). + + Every writing system the project knows about has to reach the FieldWorks + project, or data in it could not sync. Anything that is neither vernacular + nor analysis by the field conventions is reported as `unclassified` and + appended to the analysis list, which is the safer of the two: an extra + analysis writing system is inert, whereas a missing one loses data. + + The overrides replace the derived base lists, but coverage still applies -- + uncovered writing systems are appended to analysis either way, since the + whole point of the check is that the union is complete. + + A writing system in both lists is then removed from one of them, so that each writing system + has a single role in the new FieldWorks project. LF has no such constraint, so the overlap is + common: one project lists "rw" and "en" in its lexeme field and both again in its + definition. A writing system in both lists also tells nothing about the role of any field that + uses it, which is what leaves custom field wsSelectors undecidable. + + Which list it leaves is decided by `anchor_counts` -- where its data actually is, per + count_anchor_writing_system_usage -- and not by which field the config declares it on, since a + declaration can rest on a single stray value. Without counts, or on a tie, vernacular keeps it. + The vernacular list is never emptied, since LexBox requires one, and an explicitly overridden + list is never trimmed: --ws-vernacular and --ws-analysis say what the lists are to be. + VERNACULAR_ANALYSIS_OVERLAP_EXEMPTION is never removed from analysis. + + If the analysis list leads with an unlisted-language (qaa) writing system it + is reordered to lead with ANTHRO_DEFAULT_ANALYSIS_WS instead, and says so in + the log. The first entry becomes the new FieldWorks project's default + analysis writing system, and liblcm cannot write the anthropology list in a + qaa one. `en` is added if the project does not have it; an extra analysis + writing system is inert, and without it the project cannot be created. Any + other leading writing system is left where the project put it. + """ + if override_vernacular: + vernacular = dedup(override_vernacular) + else: + vernacular = dedup(found["lexeme"] + found["citationForm"]) + + if override_analysis: + analysis = dedup(override_analysis) + else: + analysis = dedup(found["definition"] + found["gloss"]) + + every = dedup(found["config_all"] + found["declared"]) + classified = set(vernacular) | set(analysis) + unclassified = [tag for tag in every if tag not in classified] + if unclassified: + analysis = dedup(analysis + unclassified) + + # Before the qaa check below, so that runs against the list the project will actually get. + counts = anchor_counts or {} + for tag in [tag for tag in analysis if tag in set(vernacular)]: + usage = counts.get(tag) or {} + in_vernacular = usage.get("vernacular", 0) + in_analysis = usage.get("analysis", 0) + evidence = f"{in_vernacular} vernacular vs {in_analysis} analysis entries" + + # Whichever side holds less of the writing system's data is the side it leaves. A tie -- + # which includes a writing system with no data at all -- leaves the vernacular list alone. + leaves = "vernacular" if in_analysis > in_vernacular else "analysis" + keep_reason = None + forced_reason = None + + if override_vernacular and override_analysis: + keep_reason = "both lists were given explicitly" + elif leaves == "vernacular" and override_vernacular: + leaves = "analysis" + forced_reason = "--ws-vernacular says what the vernacular list is to be" + elif leaves == "analysis" and override_analysis: + leaves = "vernacular" + forced_reason = "--ws-analysis says what the analysis list is to be" + + # LexBox requires a vernacular writing system, so the last one never leaves that list. + if keep_reason is None and leaves == "vernacular" and len(vernacular) == 1: + leaves = "analysis" + forced_reason = "it is the project's only vernacular writing system, which LexBox requires" + + # The anthropology list can only be written in this one, so it never leaves analysis. + if keep_reason is None and leaves == "analysis" and tag == VERNACULAR_ANALYSIS_OVERLAP_EXEMPTION: + keep_reason = "the anthropology list needs it and can be written in no other" + + if keep_reason is not None: + log(f" NOTE: '{tag}' is in both writing system lists ({evidence}); left in both,") + log(f" because {keep_reason}.") + continue + + if leaves == "vernacular": + vernacular = [item for item in vernacular if item != tag] + else: + analysis = [item for item in analysis if item != tag] + + # Compare against the side it actually left, not the side the data preferred: a guard above + # may have overruled the data, and saying "where less of its data is" would then be false. + left_behind = in_vernacular if leaves == "vernacular" else in_analysis + kept = in_analysis if leaves == "vernacular" else in_vernacular + log(f" NOTE: '{tag}' was in both writing system lists ({evidence}); removed from {leaves},") + if left_behind > kept: + because = f", because {forced_reason}" if forced_reason else "" + log(f" even though that is where more of its data is{because}.") + else: + log(" which is where less of its data is.") + + if not analysis: + log(" WARNING: no analysis writing systems are left. LexBox will fall back to") + log(f" '{ANTHRO_DEFAULT_ANALYSIS_WS}'; pass --ws-analysis to choose instead.") + + # Last, because the first analysis writing system is the one liblcm writes the anthropology + # list in, and it cannot write it in a qaa one. Applied even to an explicit --ws-analysis: the + # constraint is liblcm's, not a preference, and getting it wrong fails the whole creation. + if analysis and analysis[0].lower().startswith(UNLISTED_LANGUAGE_PREFIX): + leading = analysis[0] + analysis = [ANTHRO_DEFAULT_ANALYSIS_WS] + [ + tag for tag in analysis if tag != ANTHRO_DEFAULT_ANALYSIS_WS + ] + log(f" NOTE: analysis writing systems led with '{leading}'; moved " + f"'{ANTHRO_DEFAULT_ANALYSIS_WS}' to the front so that the new project has a default") + log(" analysis writing system the anthropology list can be written in.") + + return vernacular, analysis, unclassified + + +def count_collections(config: Config, db_name: str) -> int: + js = f""" +print(JSON.stringify({{ count: db.getSiblingDB({json.dumps(db_name)}).getCollectionNames().length }})); +""" + return int(mongosh_json(config, js, what=f"Inspecting database {db_name}")["count"]) + + +def copy_project_database(config: Config, source_db: str, target_db: str, *, drop: bool) -> None: + """Copy every collection and index of source_db into target_db. + + Uses mongodump piped through mongorestore --nsFrom/--nsTo so that BSON + types, indexes and collection options are preserved exactly -- no + document-by-document marshalling as backup.mjs has to do. + """ + require_tool("mongodump") + require_tool("mongorestore") + + with tempfile.TemporaryDirectory(prefix="lf-migrate-") as tmpdir: + archive = os.path.join(tmpdir, f"{source_db}.archive") + + log(f" dumping {source_db}...") + run( + ["mongodump", f"--uri={config.uri(source_db)}", f"--archive={archive}", "--quiet"], + what=f"mongodump of {source_db}", + ) + + if not os.path.exists(archive) or os.path.getsize(archive) == 0: + log(f" {source_db} is empty; nothing to restore") + return + + log(f" restoring into {target_db}...") + cmd = [ + "mongorestore", + f"--uri={config.uri()}", + f"--archive={archive}", + f"--nsFrom={source_db}.*", + f"--nsTo={target_db}.*", + "--quiet", + ] + if drop: + cmd.append("--drop") + run(cmd, what=f"mongorestore into {target_db}") + + +# A numeric BSON value does not always reach mongosh as a JS number: int32 and +# double do, but int64 and decimal arrive as wrapper objects. Shared by both +# coercion scripts below, which is why it is a separate string and not inlined. +NUMERIC_HELPERS_JS = """ +function isNumeric(value) { + if (typeof value === "number") return true; + if (value === null || typeof value !== "object") return false; + const kind = value._bsontype; + return kind === "Int32" || kind === "Long" || kind === "Double" || kind === "Decimal128"; +} + +function asString(value) { + return typeof value === "number" ? String(value) : value.toString(); +} +""" + + +def coerce_option_list_keys(config: Config, db_name: str) -> dict: + """Rewrite every numeric option list item key in db_name as a string. + + Runs against the COPY, straight after the database duplication, so the + original project keeps its integers. `items[].key` is the only field in an + optionlists document that is ever numeric in the corpus; `defaultItemKey` + names an item key too, so it is coerced alongside them, defensively -- no + project in the 2026-07-06 dump has a numeric one. + + A coerced key that duplicates a key already in the same list would make that + list ambiguous. No such case exists in the corpus, so this reports any it + finds rather than refusing to migrate, and still performs the coercion. + """ + js = f""" +const targetName = {json.dumps(db_name)}; +""" + NUMERIC_HELPERS_JS + """ +const target = db.getSiblingDB(targetName); + +let lists = 0; +let listsChanged = 0; +let keysCoerced = 0; +let defaultKeysCoerced = 0; +const collisions = []; +const samples = []; + +if (target.getCollectionNames().indexOf("optionlists") !== -1) { + target.optionlists.find({}).forEach(function (doc) { + lists += 1; + + const items = Array.isArray(doc.items) ? doc.items : []; + // Keys already spoken for, so a coerced key landing on one of them can be + // spotted instead of silently creating a duplicate. + const taken = new Set(); + items.forEach(function (item) { + if (item && typeof item.key === "string") taken.add(item.key); + }); + + let coercedHere = 0; + const newItems = items.map(function (item) { + if (!item || !isNumeric(item.key)) return item; + const key = asString(item.key); + if (taken.has(key)) { + collisions.push({ code: doc.code, key: key, value: item.value }); + } + taken.add(key); + coercedHere += 1; + if (samples.length < 5) { + samples.push({ code: doc.code, key: key, value: item.value }); + } + // Object.assign keeps the field order, and `key` already exists on the + // item, so it is replaced in place rather than moved to the end. + return Object.assign({}, item, { key: key }); + }); + + const update = {}; + if (coercedHere > 0) { + update.items = newItems; + keysCoerced += coercedHere; + } + if (isNumeric(doc.defaultItemKey)) { + update.defaultItemKey = asString(doc.defaultItemKey); + defaultKeysCoerced += 1; + } + if (Object.keys(update).length > 0) { + target.optionlists.updateOne({ _id: doc._id }, { $set: update }); + listsChanged += 1; + } + }); +} + +print(EJSON.stringify({ + lists: lists, + listsChanged: listsChanged, + keysCoerced: keysCoerced, + defaultKeysCoerced: defaultKeysCoerced, + collisions: collisions, + samples: samples, +})); +""" + result = mongosh_json(config, js, what=f"Coercing option list keys in {db_name}") + + log(f" {result['lists']} option list(s) examined") + if result["keysCoerced"]: + log(f" {result['keysCoerced']} numeric key(s) coerced to strings " + f"across {result['listsChanged']} list(s)") + for sample in result["samples"]: + log(f" {sample['code']}: {sample['key']!r} ({sample['value']})") + else: + log(" no numeric keys found; nothing to coerce") + if result["defaultKeysCoerced"]: + log(f" {result['defaultKeysCoerced']} numeric defaultItemKey(s) coerced") + if result["collisions"]: + log("") + log("WARNING: coercing produced a key that the list already had as a string.") + log(" The list now has two items with the same key, and a reference to") + log(" it is ambiguous. Inspect these before pushing to LexBox:") + for collision in result["collisions"]: + log(f" {collision['code']}: key {collision['key']!r}") + log("") + return result + + +def coerce_special_lexicon_references(config: Config, db_name: str, source_code: str) -> dict: + """Coerce the two known numeric lexicon references, and report any others. + + Only the sense named in SPECIAL_LEXICON_INT_REFERENCES for this project code + is rewritten, and only when its `values` list is exactly the expected one. + Every other numeric reference -- in another project, another field, or a + values list that does not match -- is counted and reported, never touched. + + Nothing here matters unless coerce_option_list_keys has already run: it is + the list-side coercion that leaves these two references dangling. + """ + special = SPECIAL_LEXICON_INT_REFERENCES.get(source_code) + field, expected = special if special else ("", []) + + js = f""" +const targetName = {json.dumps(db_name)}; +const specialField = {json.dumps(field)}; +const specialValues = {json.dumps(expected)}; +""" + NUMERIC_HELPERS_JS + """ +const target = db.getSiblingDB(targetName); + +// Exact match against the expected list: same length, every entry numeric, and +// every entry the same number. That is what makes this a special case for two +// known senses rather than a blanket lexicon rewrite. +function isSpecialList(values) { + if (!specialField || specialValues.length === 0) return false; + if (!Array.isArray(values) || values.length !== specialValues.length) return false; + for (let index = 0; index < values.length; index += 1) { + if (!isNumeric(values[index])) return false; + if (asString(values[index]) !== String(specialValues[index])) return false; + } + return true; +} + +let entriesChanged = 0; +let valuesCoerced = 0; +const coerced = []; +const untouched = []; + +if (target.getCollectionNames().indexOf("lexicon") !== -1) { + // Only `senses` is read or written, so leave the rest of each entry on the + // server -- a lexicon can run to tens of thousands of documents. + target.lexicon.find({}, { senses: 1 }).forEach(function (doc) { + const senses = Array.isArray(doc.senses) ? doc.senses : []; + let changed = false; + + senses.forEach(function (sense, senseIndex) { + if (!sense || typeof sense !== "object") return; + Object.keys(sense).forEach(function (fieldName) { + const holder = sense[fieldName]; + if (!holder || typeof holder !== "object" || !Array.isArray(holder.values)) return; + if (!holder.values.some(isNumeric)) return; + + const shown = holder.values.map(function (value) { + return isNumeric(value) ? asString(value) : value; + }); + + if (fieldName !== specialField || !isSpecialList(holder.values)) { + untouched.push({ + entry: doc._id.toString(), + sense: senseIndex, + field: fieldName, + values: shown, + }); + return; + } + + holder.values = holder.values.map(function (value) { + return isNumeric(value) ? asString(value) : value; + }); + valuesCoerced += holder.values.length; + coerced.push({ + entry: doc._id.toString(), + sense: senseIndex, + field: fieldName, + values: shown, + }); + changed = true; + }); + }); + + if (changed) { + target.lexicon.updateOne({ _id: doc._id }, { $set: { senses: senses } }); + entriesChanged += 1; + } + }); +} + +print(EJSON.stringify({ + entriesChanged: entriesChanged, + valuesCoerced: valuesCoerced, + coerced: coerced, + untouched: untouched, +})); +""" + result = mongosh_json(config, js, what=f"Coercing lexicon references in {db_name}") + + if special: + log(f" {source_code!r} is a known special case: " + f"{field}.values == {expected}") + if result["coerced"]: + for record in result["coerced"]: + log(f" coerced entry {record['entry']} sense #{record['sense']} " + f"{record['field']}.values {record['values']} -> strings") + log(f" {result['valuesCoerced']} value(s) in {result['entriesChanged']} entry/entries") + elif special: + log(" expected special-case sense was NOT found; nothing coerced") + log(" (the project data may have changed since the dump this was derived from)") + else: + log(" not a special-case project; no lexicon value was changed") + + if result["untouched"]: + log("") + log(f"WARNING: {len(result['untouched'])} numeric option list reference(s) in the") + log(" lexicon were left as they are, because they are not one of the two") + log(" known special cases. They will not resolve against the coerced") + log(" option lists. Add them to SPECIAL_LEXICON_INT_REFERENCES if they") + log(" should be converted too:") + for record in result["untouched"][:10]: + log(f" entry {record['entry']} sense #{record['sense']} " + f"{record['field']}.values = {record['values']}") + if len(result["untouched"]) > 10: + log(f" ... and {len(result['untouched']) - 10} more") + log("") + return result + + + +def find_empty_entry_guids(config: Config, db_name: str) -> list[dict]: + """Return the lexicon entries in db_name whose guid is an empty string. + + Matches `guid: ""` exactly, because that is the only shape the problem takes + in the 2026-07-06 corpus: five projects with one such entry each, no nulls, + no missing guid fields and no malformed values. An entry whose guid is + merely unusual -- a UUID that is not version 4, say -- is left alone, since + LfMerge parses it happily. + + The lexeme form comes back with each hit purely so the log can name the + entry instead of only its ObjectId. Read-only, so it is safe to point at the + source database to preview what a run would do. + """ + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +const found = []; +if (target.getCollectionNames().indexOf("lexicon") !== -1) {{ + target.lexicon.find({{ guid: "" }}, {{ lexeme: 1 }}).forEach(function (doc) {{ + // lexeme is a multitext, {{ "": {{ "value": "..." }} }}. Any one form is + // enough to identify the entry in a log line. + let label = ""; + const lexeme = doc.lexeme || {{}}; + Object.keys(lexeme).forEach(function (ws) {{ + if (!label && lexeme[ws] && lexeme[ws].value) {{ + label = ws + ": " + lexeme[ws].value; + }} + }}); + found.push({{ id: doc._id.toString(), label: label }}); + }}); +}} +print(EJSON.stringify({{ found: found }})); +""" + return list(mongosh_json(config, js, what=f"Finding empty entry guids in {db_name}")["found"]) + + +def fix_empty_entry_guids(config: Config, db_name: str) -> dict: + """Give every lexicon entry with an empty-string guid a fresh UUID4. + + Runs against the COPY, so the original project keeps its empty guid. + + An empty guid does not make LfMerge fail, which is why it goes unnoticed. + LfMerge reads the field with Guid.TryParse (MongoConnection.CanParseGuid), + so such an entry is quietly dropped from the ObjectId->Guid map that + comments are attached by, and ConvertMongoToLcmLexicon then falls back to + `lfEntry.Guid ?? Guid.Empty`. The entry would be created in FieldWorks under + the all-zeros GUID, and any later entry in the same state would find that + object and overwrite it instead of creating its own. Fixing this before the + first Send/Receive is what keeps the entries distinct. + + The new guids are generated here rather than in mongosh: uuid.uuid4() + produces exactly what Guid::create() does in PHP -- lowercase, hyphenated, + version 4 -- which is the only form Guid::isValid accepts. mongosh's UUID() + would produce BSON binary, the wrong type for this field. + + Only `guid` is written. dateModified and dirtySR are left alone: restoring an + identity is not a content edit, and touching them would misreport when the + entry was last worked on. + """ + entries = find_empty_entry_guids(config, db_name) + if not entries: + log(" no entries with an empty guid") + return {"fixed": 0, "assignments": []} + + # The unique {guid: 1} index (LexEntryModel.php:515) would reject a duplicate + # part way through the batch, so make sure this batch is distinct with itself. + seen: set[str] = set() + assignments = [] + for entry in entries: + guid = str(uuid.uuid4()) + while guid in seen: + guid = str(uuid.uuid4()) + seen.add(guid) + assignments.append({"id": entry["id"], "guid": guid, "label": entry.get("label", "")}) + + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +const assignments = {json.dumps(assignments)}; +const updated = []; +const collisions = []; + +assignments.forEach(function (assignment) {{ + // Cheap insurance: a collision would be unrecoverable half way through a + // batch, and this costs one indexed lookup per entry. + if (target.lexicon.findOne({{ guid: assignment.guid }}) !== null) {{ + collisions.push(assignment.guid); + return; + }} + // The guid: "" in the filter means a document that somehow already got a + // guid is left alone rather than overwritten. + const result = target.lexicon.updateOne( + {{ _id: new ObjectId(assignment.id), guid: "" }}, + {{ $set: {{ guid: assignment.guid }} }} + ); + if (result.modifiedCount === 1) {{ + updated.push(assignment.id); + }} +}}); + +print(EJSON.stringify({{ updated: updated, collisions: collisions }})); +""" + result = mongosh_json(config, js, what=f"Assigning entry guids in {db_name}") + + if result["collisions"]: + raise Fatal( + "generated a guid that the lexicon already contains, which should not be possible: " + + ", ".join(result["collisions"]) + ) + if len(result["updated"]) != len(assignments): + raise Fatal( + f"expected to give guids to {len(assignments)} entry/entries but updated " + f"{len(result['updated'])}; the copy may have changed underneath us" + ) + + for assignment in assignments: + label = f" ({assignment['label']})" if assignment["label"] else "" + log(f" entry {assignment['id']}{label} -> {assignment['guid']}") + log(f" {len(assignments)} empty guid(s) replaced") + return {"fixed": len(assignments), "assignments": assignments} + + +def create_project_record( + config: Config, + source_code: str, + new_code: str, + new_name: str, + identifier: str, + renames: list[dict] | None = None, +) -> str: + """Insert the copied project document and register it with its members. + + Returns the new project's _id as a hex string. Called AFTER the project + database has been copied, so there is never a window in which the project + exists but its lexicon database does not (same ordering as backup.mjs). + + `renames` applies custom field renames to the copied config. It has to happen here rather than + in a step of its own because the document is copied wholesale from the source, config included, + so this is the only moment at which the new project's config exists but nothing has read it. + A custom field name appears in the config both as a key -- under entry.fields, senses.fields and + each roleViews..fields -- and as a value, in the fieldOrder lists, so both are rewritten. + """ + js = f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const renames = {json.dumps(renames or [])}; +const doc = main.projects.findOne({{ projectCode: {json.dumps(source_code)} }}); +if (doc === null) {{ + throw new Error("source project disappeared: " + {json.dumps(source_code)}); +}} + +delete doc._id; + +// Rename custom fields wherever the config mentions them, as a key or as a fieldOrder entry. +// Object.keys() is a snapshot, so re-keying while walking it is safe. +function renameThroughout(node, oldKey, newKey) {{ + if (Array.isArray(node)) {{ + for (let i = 0; i < node.length; i++) {{ + if (node[i] === oldKey) {{ node[i] = newKey; }} + else {{ renameThroughout(node[i], oldKey, newKey); }} + }} + }} else if (node !== null && typeof node === "object") {{ + Object.keys(node).forEach(function (key) {{ + if (key === oldKey) {{ + node[newKey] = node[key]; + delete node[key]; + renameThroughout(node[newKey], oldKey, newKey); + }} else {{ + renameThroughout(node[key], oldKey, newKey); + }} + }}); + }} +}} +// A field whose label still reads the old name is only half renamed, and the label is what LF +// shows and what this script would emit as the FieldWorks helpString. Only a label that matches +// the old name is rewritten, so a label somebody deliberately set to something else survives. +function relabel(node, key, oldLabel, newLabel) {{ + if (Array.isArray(node)) {{ + node.forEach(function (item) {{ relabel(item, key, oldLabel, newLabel); }}); + }} else if (node !== null && typeof node === "object") {{ + Object.keys(node).forEach(function (k) {{ + const child = node[k]; + if (k === key && child !== null && typeof child === "object" && child.label === oldLabel) {{ + child.label = newLabel; + }} + relabel(child, key, oldLabel, newLabel); + }}); + }} +}} +let configKeysRenamed = 0; +renames.forEach(function (rename) {{ + const before = JSON.stringify(doc.config || {{}}); + renameThroughout(doc.config, rename.old_key, rename.new_key); + relabel(doc.config, rename.new_key, rename.old_name, rename.new_name); + if (JSON.stringify(doc.config || {{}}) !== before) {{ configKeysRenamed += 1; }} +}}); +doc.projectCode = {json.dumps(new_code)}; +doc.projectName = {json.dumps(new_name)}; +// Point the copy at its own (new) Send/Receive project. The identifier is +// derived from the projectCode but normalised for LexBox, so the two differ +// whenever the projectCode has capitals or underscores. +doc.sendReceiveProjectIdentifier = {json.dumps(identifier)}; + +const inserted = main.projects.insertOne(doc); +const newId = inserted.insertedId; + +// Give every member of the original project access to the copy as well. +const memberIds = Object.keys(doc.users || {{}}).map((id) => new ObjectId(id)); +let membersUpdated = 0; +if (memberIds.length) {{ + membersUpdated = main.users.updateMany( + {{ _id: {{ $in: memberIds }} }}, + {{ $addToSet: {{ projects: newId }} }} + ).modifiedCount; +}} + +print(EJSON.stringify({{ + id: newId.toString(), + membersUpdated: membersUpdated, + configKeysRenamed: configKeysRenamed, +}})); +""" + result = mongosh_json(config, js, what="Creating project record") + log(f" new project _id: {result['id']}") + log(f" granted access to {result['membersUpdated']} member(s)") + if renames: + renamed = result.get("configKeysRenamed", 0) + log(f" renamed {renamed} custom field(s) in the copied config") + if renamed != len(renames): + raise Fatal( + f"expected to rename {len(renames)} custom field(s) in the config but changed " + f"{renamed}; the project config is not shaped as this script expects" + ) + return result["id"] + + +def delete_project_record(config: Config, project_id: str) -> None: + mongosh_json( + config, + f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const doomed = new ObjectId({json.dumps(project_id)}); +main.users.updateMany({{ projects: doomed }}, {{ $pull: {{ projects: doomed }} }}); +main.projects.deleteOne({{ _id: doomed }}); +print(JSON.stringify({{ ok: 1 }})); +""", + what="Removing project record", + ) + + +# --------------------------------------------------------------------------- # +# asset steps +# --------------------------------------------------------------------------- # + + +def stage_assets(config: Config, app_name: str, source_db: str, target_db: str) -> str | None: + """Copy audio/pictures to the new project's own assets folder, as real dirs. + + A non-Send/Receive project keeps these as ordinary directories + (LexProjectModel::createAssetsFolders takes its `else` branch), so this is a + plain recursive copy. Symlinks are dereferenced anyway, defensively: they + may exist and even be broken if the project was ever converted + (cf. `tar chf` in backup.mjs). + + These directories are turned into symlinks into the webwork tree later, by + link_assets_into_webwork(), once the first Send/Receive has created it. + """ + source_dir = config.assets_path(app_name, source_db) + target_dir = config.assets_path(app_name, target_db) + + if not os.path.isdir(source_dir): + log(f" no assets directory at {source_dir}; skipping") + return None + + os.makedirs(target_dir, exist_ok=True) + total = 0 + for subfolder, _ in ASSET_LINKED_FILES: + source_sub = os.path.join(source_dir, subfolder) + target_sub = os.path.join(target_dir, subfolder) + # os.path.isdir follows symlinks and is False for a broken one. + if not os.path.isdir(source_sub): + log(f" {subfolder}: missing or broken symlink; creating empty directory") + os.makedirs(target_sub, exist_ok=True) + continue + if os.path.exists(target_sub) or os.path.islink(target_sub): + if os.path.islink(target_sub): + os.unlink(target_sub) + else: + shutil.rmtree(target_sub) + shutil.copytree(source_sub, target_sub, symlinks=False) + count = sum(len(files) for _, _, files in os.walk(target_sub)) + log(f" {subfolder}: {count} file(s) copied") + total += count + + chown_to_web_user(target_dir) + if total == 0: + log(" (source project had no asset files)") + return target_dir + + +def copy_missing_files(source_dir: str, target_dir: str) -> tuple[int, int]: + """Recursively copy files from source_dir into target_dir, skipping existing. + + "Any assets NOT found in the LinkedFiles folder get copied in" -- anything + the Send/Receive already brought down wins, so a re-run is a no-op. + """ + copied = skipped = 0 + for root, _dirnames, filenames in os.walk(source_dir): + relative = os.path.relpath(root, source_dir) + destination_root = target_dir if relative == "." else os.path.join(target_dir, relative) + os.makedirs(destination_root, exist_ok=True) + for name in filenames: + destination = os.path.join(destination_root, name) + if os.path.exists(destination): + skipped += 1 + else: + shutil.copy2(os.path.join(root, name), destination) + copied += 1 + return copied, skipped + + +def link_assets_into_webwork(config: Config, app_name: str, target_db: str, new_code: str) -> None: + """Move staged assets into the webwork LinkedFiles tree and symlink to them. + + This is the Python equivalent of + LexProjectModel::moveExistingFilesAndCreateSymlink, and leaves the copy with + the on-disk layout a real Send/Receive project has: + + assets//sf_/audio -> webwork//LinkedFiles/AudioVisual + assets//sf_/pictures -> webwork//LinkedFiles/Pictures + + Must run AFTER the first Send/Receive, which is what creates webwork/. + """ + webwork = config.webwork_path(new_code) + if not os.path.isdir(webwork): + raise Fatal( + f"Send/Receive work folder {webwork} does not exist. The first Send/Receive " + "does not appear to have cloned the project, so there is nowhere to put the " + "assets. Check `docker compose logs lfmerge`." + ) + + assets_dir = config.assets_path(app_name, target_db) + for subfolder, linked_subfolder in ASSET_LINKED_FILES: + asset_path = os.path.join(assets_dir, subfolder) + linked_path = os.path.join(config.linked_files_path(new_code), linked_subfolder) + + os.makedirs(linked_path, exist_ok=True) + + if os.path.islink(asset_path): + if os.path.realpath(asset_path) == os.path.realpath(linked_path): + log(f" {subfolder}: already symlinked into webwork") + continue + log(f" {subfolder}: replacing symlink to {os.path.realpath(asset_path)}") + os.unlink(asset_path) + elif os.path.isdir(asset_path): + copied, skipped = copy_missing_files(asset_path, linked_path) + log(f" {subfolder}: {copied} file(s) moved into webwork, {skipped} already there") + shutil.rmtree(asset_path) + elif os.path.exists(asset_path): + os.unlink(asset_path) + else: + log(f" {subfolder}: nothing staged") + + os.makedirs(os.path.dirname(asset_path), exist_ok=True) + os.symlink(linked_path, asset_path) + + # LFMerge commits these as www-data, so it must own them. + chown_to_web_user(config.linked_files_path(new_code)) + chown_to_web_user(assets_dir) + + +# --------------------------------------------------------------------------- # +# send/receive steps +# --------------------------------------------------------------------------- # + + +def split_lexbox_host(host: str) -> tuple[str, str]: + """Split a --lexbox-host value into (scheme, netloc). + + A bare host like "lexbox.org" gets https, since LexBox communication is over + HTTPS. An explicit scheme is honoured, which is how a development LexBox is + reached: it serves plain HTTP, with no certificate at all. + """ + raw = (host or "").strip().rstrip("/") + if not raw: + raise Fatal("--lexbox-host must not be empty") + if "//" in raw: + parts = urlsplit(raw) + scheme, netloc = (parts.scheme or "https"), parts.netloc + else: + scheme, netloc = "https", raw + if not netloc: + raise Fatal(f"could not parse a hostname out of --lexbox-host {host!r}") + if scheme not in ("http", "https"): + raise Fatal(f"unsupported scheme {scheme!r} in --lexbox-host {host!r}") + return scheme, netloc + + +class LexboxClient: + """Minimal LexBox API client, standard library only. + + Login is POST /api/login (LoginController.cs:21 sets the route to /api/login, + and the action has a bare [HttpPost]). The response BODY is the LexAuthUser + record; the JWT itself comes back only as the `.LexBoxAuth` cookie, because + the controller calls HttpContext.SignInAsync and AuthKernel.cs:229 replaces + the cookie ticket format with a JWT writer. + """ + + def __init__( + self, + host: str, + username: str, + password: str, + *, + auth_mode: str = "cookie", + timeout: int = DEFAULT_LEXBOX_TIMEOUT_SECONDS, + ) -> None: + self.scheme, self.netloc = split_lexbox_host(host) + self.base_url = f"{self.scheme}://{self.netloc}" + self.hostname = urlsplit(f"//{self.netloc}").hostname or self.netloc + self.username = username + self.password = password + self.auth_mode = auth_mode + self.timeout = timeout + self.cookie_jar = http.cookiejar.CookieJar() + self.jwt: str | None = None + + # -- transport --------------------------------------------------------- # + + @property + def is_localhost(self) -> bool: + """True for localhost and any name under the reserved .localhost TLD. + + The subdomains matter because a development LexBox is reached at + admin.localhost, not localhost. RFC 6761 reserves .localhost for local + use, so treating the whole family as local cannot loosen anything for a + real host such as lexbox.org. + """ + host = (self.hostname or "").lower() + return host == "localhost" or host.endswith(".localhost") + + def ssl_context(self) -> ssl.SSLContext: + """Verify certificates everywhere except the localhost family. + + A local dev LexBox may present a self-signed certificate, which is + accepted if and only if the host is localhost or a .localhost subdomain. + For any other host the full chain and hostname are verified, so a + self-signed or mismatched certificate is refused. + """ + context = ssl.create_default_context() + if self.is_localhost: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + def _opener(self) -> urllib.request.OpenerDirector: + return urllib.request.build_opener( + urllib.request.HTTPSHandler(context=self.ssl_context()), + urllib.request.HTTPCookieProcessor(self.cookie_jar), + ) + + def _send( + self, + method: str, + path: str, + *, + query: list[tuple[str, str]] | None = None, + json_body: Any = None, + authenticate: bool = False, + ) -> tuple[int, Any, bytes]: + params = list(query or []) + headers: dict[str, str] = {} + + if authenticate: + if not self.jwt: + raise Fatal("internal error: authenticated request attempted before login") + if self.auth_mode == "bearer": + headers["Authorization"] = f"Bearer {self.jwt}" + elif self.auth_mode == "jwt-query": + params.append(("jwt", self.jwt)) + # "cookie" mode needs nothing: the CookieJar replays .LexBoxAuth. + + url = self.base_url + path + if params: + url += "?" + urlencode(params) + + data = None + if json_body is not None: + data = json.dumps(json_body).encode("utf-8") + headers["Content-Type"] = "application/json" + elif method == "POST": + data = b"" # POST with an empty body; everything is in the query string + + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with self._opener().open(request, timeout=self.timeout) as response: + return response.status, response.headers, response.read() + except urllib.error.HTTPError as exc: + return exc.code, exc.headers, exc.read() + except urllib.error.URLError as exc: + hint = "" + if isinstance(exc.reason, ssl.SSLError): + hint = ( + " (certificate rejected; self-signed certificates are only accepted " + "for localhost and .localhost subdomains)" + ) + elif self.scheme == "https": + hint = ( + " (a development LexBox serves plain HTTP -- try " + f"--lexbox-host {DEFAULT_LEXBOX_HOST})" + ) + raise Fatal(f"{method} {url} failed: {exc.reason}{hint}") from exc + + @staticmethod + def _describe(status: int, body: bytes) -> str: + """Pull `detail` out of an ASP.NET problem+json response.""" + text = body.decode("utf-8", errors="replace").strip() + try: + document = json.loads(text) + except json.JSONDecodeError: + return f"HTTP {status}: {text[:300]}" + if isinstance(document, dict): + detail = document.get("detail") or document.get("title") + if detail: + return f"HTTP {status}: {detail}" + return f"HTTP {status}: {text[:300]}" + + # -- operations -------------------------------------------------------- # + + def login(self) -> str: + """POST /api/login and return the JWT taken from the .LexBoxAuth cookie.""" + status, _headers, body = self._send( + "POST", + "/api/login", + # defaultScope=true is required: with false the JWT carries no scopes + # and every later call fails the LexboxApi scope requirement. + query=[("defaultScope", "true")], + json_body={ + "password": self.password, + "emailOrUsername": self.username, + "preHashedPassword": False, + }, + ) + if status == 401: + raise Fatal(f"LexBox rejected the credentials for user {self.username!r} (HTTP 401)") + if status != 200: + raise Fatal(f"LexBox login failed: {self._describe(status, body)}") + + for cookie in self.cookie_jar: + if cookie.name == LEXBOX_AUTH_COOKIE: + self.jwt = cookie.value + break + if not self.jwt: + raise Fatal( + f"LexBox login returned HTTP 200 but no {LEXBOX_AUTH_COOKIE} cookie, " + "so no JWT could be obtained." + ) + return self.jwt + + def init_fwdata_project( + self, + code: str, + name: str, + ws_vernacular: list[str], + ws_analysis: list[str], + ) -> str: + """POST /api/project/initFwDataProject, wait for it, and return the new project GUID. + + Multi-valued writing systems are sent as repeated query parameters, which + is how ASP.NET binds `[FromQuery] string[]` (ProjectController.cs). + The endpoint is admin-only and NOT idempotent, so this never retries. + + The endpoint answers 202 Accepted as soon as the project row and its empty + repository exist, and builds the .fwdata in a background job. This does not + return until wait_for_fwdata_project() has seen that job finish, so callers + can go on treating the project as ready, as they did when the endpoint did + all of its work before answering. + """ + query: list[tuple[str, str]] = [("code", code)] + query += [("wsVernacular", tag) for tag in ws_vernacular] + query += [("wsAnalysis", tag) for tag in ws_analysis] + if name: + query.append(("name", name)) + + status, _headers, body = self._send( + "POST", "/api/project/initFwDataProject", query=query, authenticate=True + ) + + if status == 202: + # AcceptedAtAction(..., projectId) serializes the Guid as a bare JSON string. + guid = json.loads(body.decode("utf-8")) + if not isinstance(guid, str) or not guid: + raise Fatal(f"initFwDataProject returned an unexpected body: {body[:200]!r}") + self.wait_for_fwdata_project(guid, code) + return guid + + if status in (401, 403): + raise Fatal( + f"initFwDataProject was refused ({self._describe(status, body)}). " + f"It requires an admin account, and the JWT must be sent in a way LexBox accepts " + f"(current --lexbox-auth={self.auth_mode})." + ) + raise Fatal(f"initFwDataProject failed: {self._describe(status, body)}") + + def wait_for_fwdata_project(self, project_id: str, code: str) -> None: + """Poll GET /api/project/fwDataProjectCreationStatus/{projectId} until creation has finished. + + LexBox reports Creating while the background job runs and Created once the project is ready. + A failed job deletes the project again, and LexBox reports CreationFailed with the reason for + an hour afterwards -- or answers 404 once it has forgotten, or after a restart. The status + names are compared without regard to case, in case LexBox's JSON naming policy changes them. + """ + deadline = time.monotonic() + self.timeout + last_reported = None + while True: + status, _headers, body = self._send( + "GET", f"/api/project/fwDataProjectCreationStatus/{project_id}", authenticate=True + ) + if status == 404: + raise Fatal( + f"LexBox has no record of creating project {code!r} ({project_id}). Its creation " + "failed and LexBox no longer remembers why, or LexBox restarted." + ) + if status != 200: + raise Fatal(f"checking on the creation of project {code!r} failed: {self._describe(status, body)}") + report = json.loads(body.decode("utf-8")) + if not isinstance(report, dict): + raise Fatal(f"fwDataProjectCreationStatus returned an unexpected body: {body[:200]!r}") + reported = str(report.get("status") or "") + if reported != last_reported: + log(f" creation status: {reported or '(none)'}") + last_reported = reported + + if reported.lower() == "created": + return + if reported.lower() == "creationfailed": + raise Fatal(f"LexBox could not create project {code!r}: {report.get('error') or 'no reason given'}") + if time.monotonic() > deadline: + raise Fatal( + f"LexBox was still creating project {code!r} after {self.timeout}s (--lexbox-timeout). " + "It may yet finish, or fail and be deleted. If it finishes, run again with " + "--no-init; if it is deleted, just run again." + ) + time.sleep(POLL_INTERVAL_SECONDS) + + def fetch_repo_file(self, code: str, rev: str, path: str) -> bytes | None: + """GET /hg/{code}/raw-file/{rev}/{path}: one file from LexBox's copy of the repository. + + This goes through LexBox's hgweb proxy (ProxyKernel.MapSyncProxy), which accepts the login + cookie and lets an admin read any project. Returns None when the file is not there. + """ + status, _headers, body = self._send( + "GET", f"/hg/{code}/raw-file/{rev}/{path}", authenticate=True + ) + if status == 404: + return None + if status != 200: + raise Fatal(f"reading {path} from LexBox repository {code!r} failed: {self._describe(status, body)}") + return body + + +def make_lexbox_client(args: argparse.Namespace) -> LexboxClient: + return LexboxClient( + args.lexbox_host, + args.lexbox_user, + args.lexbox_password + if args.lexbox_password is not None + else os.environ.get("LEXBOX_PASSWORD", DEFAULT_LEXBOX_PASSWORD), + auth_mode=args.lexbox_auth, + timeout=args.lexbox_timeout, + ) + + +def require_custom_properties_pushed( + lexbox: LexboxClient, lexbox_code: str, definitions: list[dict], lexbox_host: str +) -> None: + """For --resume: refuse to go on until the custom field definitions are in LexBox. + + The pause exists because LfMerge skips custom fields FieldWorks does not know about, and the + first Send/Receive is the only one that would have carried their data. So before that runs, + FLExProject.CustomProperties at the tip of LexBox's own copy of the repository -- the copy + LfMerge clones -- has to define every field this project needs. A commit that was made but + never pushed is not there, and neither is one pushed to the wrong repository. + + Fields are matched by the `key` attribute, written exactly as build_custom_properties_xml() + writes it, so a file a human has reformatted or annotated still passes as long as every field is + defined. The listRoot placeholder must be gone: FieldWorks cannot use a definition that still + has it. + """ + log(f"--resume: checking that the custom field definitions have been pushed to LexBox ({lexbox_code})") + lexbox.login() + scheme, host = split_lexbox_host(lexbox_host) + body = lexbox.fetch_repo_file(lexbox_code, "tip", CUSTOM_PROPERTIES_FILENAME) + if body is None: + # Every FLEx repository has this file from revision 0, so a 404 means the repository itself + # was not found rather than a missing commit. + raise Fatal( + f"LexBox has no {CUSTOM_PROPERTIES_FILENAME} at {scheme}://{host}/hg/{lexbox_code} (HTTP 404).\n" + " Check that the LexBox project exists and that --lexbox-host is the one the first run\n" + " used. Nothing has been changed." + ) + text = body.decode("utf-8", errors="replace") + + missing = [] + for definition in definitions: + key = definition["attributes"]["key"] + if 'key="' + xml_escape(key, {'"': """}) + '"' not in text: + missing.append(key) + + if missing: + shown = ", ".join(missing[:5]) + (f" and {len(missing) - 5} more" if len(missing) > 5 else "") + raise Fatal( + f"the custom field definitions have not reached LexBox. {CUSTOM_PROPERTIES_FILENAME} at " + f"the tip of {scheme}://{host}/hg/{lexbox_code} lacks {len(missing)} of the " + f"{len(definitions)} field(s) this project needs ({shown}).\n" + f" If you have committed the file, you probably have not pushed it yet. In your clone:\n" + f" hg push\n" + f" Otherwise commit the {CUSTOM_PROPERTIES_FILENAME} file the first run wrote, as it\n" + f" explained, and push it. Then run with --resume again. Nothing has been changed." + ) + if LIST_ROOT_PLACEHOLDER in text: + raise Fatal( + f"{CUSTOM_PROPERTIES_FILENAME} at the tip of {scheme}://{host}/hg/{lexbox_code} still " + f"contains {LIST_ROOT_PLACEHOLDER}.\n" + " Replace it with the GUID of the list the REVIEW comment above it names, commit, and\n" + " push. Then run with --resume again. Nothing has been changed." + ) + log(f" all {len(definitions)} custom field definition(s) are there") + + +def init_lexbox_project( + lexbox: LexboxClient, + lexbox_code: str, + new_name: str, + ws_vernacular: list[str], + ws_analysis: list[str], +) -> str: + """Create the empty LexBox project that the copy will Send/Receive with. + + Logs in and calls initFwDataProject with the writing systems taken from the + Language Forge project config. Returns the GUID it reports. + + `lexbox_code` is the project's sendReceiveProjectIdentifier, which is the + normalised form of the Language Forge projectCode, not the projectCode + itself. + + The project is created empty apart from the template .fwdata: the first + Send/Receive is what pushes the lexicon from Mongo, and the second adds the + assets. + """ + if not ws_vernacular: + raise Fatal( + "LexBox requires at least one vernacular writing system, but the Language Forge " + "project config has none (config.entry.fields.lexeme.inputSystems is empty). " + "Pass --ws-vernacular explicitly." + ) + + if lexbox.scheme != "https": + tls = "not in use (plain HTTP)" + elif lexbox.is_localhost: + tls = "self-signed certificates accepted (localhost family)" + else: + tls = "certificate will be verified" + + log(f" LexBox at {lexbox.base_url}") + log(f" TLS: {tls}") + log(f" logging in as {lexbox.username!r}...") + lexbox.login() + log(f" got JWT from the {LEXBOX_AUTH_COOKIE} cookie ({len(lexbox.jwt or '')} chars)") + + log(f" creating project code={lexbox_code!r} name={new_name!r}") + log(f" wsVernacular: {', '.join(ws_vernacular)}") + log(f" wsAnalysis: {', '.join(ws_analysis) or '(none -- LexBox will default to en)'}") + guid = lexbox.init_fwdata_project(lexbox_code, new_name, ws_vernacular, ws_analysis) + log(f" LexBox project created, GUID {guid}") + + return guid + + +def read_sr_state(config: Config, project_code: str) -> tuple[dict | None, float | None]: + """Read the LFMerge state file. Returns (parsed, mtime), either may be None.""" + path = config.state_file(project_code) + try: + mtime = os.path.getmtime(path) + with open(path, encoding="utf-8") as handle: + raw = handle.read() + except (FileNotFoundError, NotADirectoryError): + return None, None + try: + return json.loads(raw), mtime + except json.JSONDecodeError: + # LFMerge may be part-way through writing it; treat as "not readable yet". + return None, mtime + + +def clear_send_receive_state(config: Config, project_code: str, suffix: str) -> list[str]: + """Remove any webwork clone and LFMerge state file left over from an earlier attempt. + + A LexBox project gets a fresh root changeset every time it is created: initFwDataProject + builds a template .fwdata and commits it, and that commit's hash depends on when it was + made, so creating the same project code twice produces two repositories with no ancestor + in common. A clone taken before the project was recreated is therefore unrelated to it + permanently, and the first Send/Receive fails inside Chorus with + + abort: repository is unrelated + + Nothing else clears it. LFMerge's EnsureCloneAction skips its initial clone whenever the + state file and the .fwdata both exist, so a stale clone is used rather than replaced, and + this script has just created the LexBox project the clone would have to match. + + Only ever called for the copy's code, which this script generates by appending the suffix, + so anything found under it is a leftover from a previous run of this same migration and + never a project a user has data in. The suffix is checked again here because this deletes + a repository. + + Returns the paths removed, for the caller to report. + """ + if not suffix or not project_code.endswith(suffix): + raise Fatal( + f"refusing to clear Send/Receive state for '{project_code}': it does not end with " + f"'{suffix}', so it is not a project code this script generated" + ) + + removed = [] + webwork = config.webwork_path(project_code) + if os.path.isdir(webwork): + shutil.rmtree(webwork) + removed.append(webwork) + + state_file = config.state_file(project_code) + if os.path.exists(state_file): + os.unlink(state_file) + removed.append(state_file) + + return removed + + +def trigger_send_receive(config: Config, project_code: str) -> str: + """Queue the project for Send/Receive by dropping a file in the syncqueue. + + LFMerge's lfmergeqm-looping.sh waits on `inotifywait -e close_write` over + the syncqueue directory, so this fires within milliseconds. The FILENAME is + the project code -- that is what LfMerge's Queue.cs reads; the contents are + ignored, but we write the same line PHP writes + (SendReceiveCommands::queueProjectForSync) for consistency. + """ + queue_dir = config.syncqueue_path + if not os.path.isdir(queue_dir): + os.makedirs(queue_dir, mode=0o2775, exist_ok=True) + + queue_file = config.queue_file(project_code) + with open(queue_file, "w", encoding="utf-8") as handle: + handle.write(f"projectCode: {project_code}") + try: + os.chmod(queue_file, 0o664) + except PermissionError: + pass + chown_to_web_user(queue_file, recursive=False) + return queue_file + + +def wait_for_send_receive(config: Config, project_code: str, *, timeout: int, label: str) -> None: + """Block until LFMerge has finished the queued Send/Receive. + + Two phases, because a stale state file from a previous run would otherwise + look like instant success: + + 1. Pickup -- wait until lfmergeqm has dequeued our file (LfMerge's + Queue.cs deletes it when it starts) AND either the state file has been + rewritten since we queued, or it reports an in-progress state. + 2. Completion -- wait for SRState to reach IDLE/SYNCED, or fail on + HOLD/ERROR. + """ + queue_file = config.queue_file(project_code) + _, state_mtime_at_trigger = read_sr_state(config, project_code) + started = time.monotonic() + picked_up = False + last_reported: str | None = None + + stale, _ = read_sr_state(config, project_code) + stale_state = (stale or {}).get("SRState") + if stale_state in SR_FAILED: + log(f" [{label}] note: previous run left SRState={stale_state}; waiting for a fresh result") + + while True: + elapsed = time.monotonic() - started + queued = os.path.exists(queue_file) + state, mtime = read_sr_state(config, project_code) + srstate = (state or {}).get("SRState") + + if srstate and srstate != last_reported: + log(f" [{label}] SRState={srstate}") + last_reported = srstate + + if not picked_up and not queued: + state_is_fresh = mtime is not None and ( + state_mtime_at_trigger is None or mtime > state_mtime_at_trigger + ) + if srstate in SR_IN_PROGRESS or state_is_fresh: + picked_up = True + + # Both the failure and the success check are gated on pickup, because a + # leftover state file from a previous run would otherwise be read as an + # instant verdict on this one -- an old ERROR would abort a legitimate + # retry, and an old IDLE would declare success before LFMerge started. + if picked_up and srstate in SR_FAILED: + raise Fatal( + f"Send/Receive ({label}) reported SRState={srstate}. " + f"See {config.state_file(project_code)} and `docker compose logs lfmerge`." + ) + + if picked_up and srstate in SR_DONE: + log(f" [{label}] finished after {elapsed:.0f}s (SRState={srstate})") + return + + if not picked_up and elapsed > PICKUP_TIMEOUT_SECONDS: + raise Fatal( + f"LFMerge did not pick up the Send/Receive request within " + f"{PICKUP_TIMEOUT_SECONDS}s (queue file {queue_file} " + f"{'still present' if queued else 'was consumed but no state was written'}). " + "Is the lfmerge container running?" + ) + + if elapsed > timeout: + raise Fatal( + f"Send/Receive ({label}) did not finish within {timeout}s " + f"(last SRState={srstate}). It may still be running; check " + "`docker compose logs lfmerge` before retrying." + ) + + time.sleep(POLL_INTERVAL_SECONDS) + + +def send_receive(config: Config, project_code: str, *, timeout: int, label: str) -> None: + queue_file = trigger_send_receive(config, project_code) + log(f" queued {queue_file}") + wait_for_send_receive(config, project_code, timeout=timeout, label=label) + + +# --------------------------------------------------------------------------- # +# main +# --------------------------------------------------------------------------- # + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Migrate a non-Send/Receive Language Forge project into Send/Receive.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("project_code", metavar="projectCode", help="projectCode of the project to copy") + parser.add_argument( + "--suffix", + default=MIGRATE_SUFFIX, + help=f"suffix for the new projectCode (default: {MIGRATE_SUFFIX})", + ) + parser.add_argument( + "--project-name", + default=None, + help="projectName for the copy (default: original name with the suffix appended)", + ) + parser.add_argument( + "--force", + action="store_true", + help="overwrite an existing copy instead of refusing to run", + ) + parser.add_argument( + "--no-assets", + action="store_true", + help="do not copy audio/pictures, and skip the second Send/Receive", + ) + parser.add_argument( + "--no-send-receive", + action="store_true", + help="stop after init_lexbox_project; do not run either Send/Receive", + ) + parser.add_argument( + "--sr-timeout", + type=int, + default=DEFAULT_SR_TIMEOUT_SECONDS, + metavar="SECONDS", + help=f"how long to wait for each Send/Receive (default: {DEFAULT_SR_TIMEOUT_SECONDS})", + ) + + lexbox = parser.add_argument_group("LexBox") + lexbox.add_argument( + "--lexbox-host", + default=DEFAULT_LEXBOX_HOST, + metavar="HOST", + help=( + f"LexBox host (default: {DEFAULT_LEXBOX_HOST}, which is where a development " + "LexBox lives -- it runs outside this Compose environment and is reached via " + "the admin.localhost extra_hosts entry. Production: lexbox.org). HTTPS is used " + "unless a scheme is given; self-signed certificates are accepted only for " + "localhost and .localhost subdomains." + ), + ) + lexbox.add_argument( + "--lexbox-user", + default=DEFAULT_LEXBOX_USER, + metavar="USER", + help=f"LexBox username (default: {DEFAULT_LEXBOX_USER}); must be a LexBox admin", + ) + lexbox.add_argument( + "--lexbox-password", + default=None, + metavar="PASSWORD", + help=( + f"LexBox password (default: the LEXBOX_PASSWORD environment variable, " + f"or {DEFAULT_LEXBOX_PASSWORD!r})" + ), + ) + lexbox.add_argument( + "--lexbox-auth", + choices=("cookie", "jwt-query", "bearer"), + default="cookie", + help=( + "how to present the login JWT on API calls. 'cookie' replays the .LexBoxAuth " + "cookie (default, always works); 'jwt-query' sends ?jwt=; 'bearer' sends " + "an Authorization: Bearer header, which LexBox routes to OpenIddict and rejects " + "whenever OpenId is enabled -- as it is in Development" + ), + ) + lexbox.add_argument( + "--lexbox-timeout", + type=int, + default=DEFAULT_LEXBOX_TIMEOUT_SECONDS, + metavar="SECONDS", + help=( + f"timeout for LexBox calls, and for a new project's creation to finish " + f"(default: {DEFAULT_LEXBOX_TIMEOUT_SECONDS})" + ), + ) + lexbox.add_argument( + "--ws-vernacular", + action="append", + default=None, + metavar="TAG", + help="override the vernacular writing systems (repeat for several, in order)", + ) + lexbox.add_argument( + "--ws-analysis", + action="append", + default=None, + metavar="TAG", + help="override the analysis writing systems (repeat for several, in order)", + ) + parser.add_argument( + "--no-init", + action="store_true", + help=( + "skip the LexBox init and assume the project and its repository already exist, in a " + "correct state. For recovering when the init failed on our side (a 504 Gateway Timeout, " + "or giving up waiting for the creation to finish) but LexBox created the project anyway. " + "Combines with --force, which only ever touches " + "the Mongo side; under --resume the init is skipped regardless" + ), + ) + parser.add_argument( + "--resume", + action="store_true", + help=( + "finish a migration that stopped because the project has custom fields: skip the copy, " + "the project record and the LexBox init, and run only the Send/Receive steps. Use this " + "once the FLExProject.CustomProperties commit has been pushed to LexBox; it checks, and " + "stops without changing anything if LexBox does not have every custom field yet." + ), + ) + parser.add_argument( + "--custom-fields-only", + action="store_true", + help=( + "report the project's custom fields and stop, without copying anything. Use this to " + "survey what a LexBox custom-field API would have to create for a project." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and report the plan without changing anything", + ) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + config = Config() + + source_code = args.project_code.strip() + if not source_code: + log("ERROR: projectCode must not be empty") + return 2 + if source_code.endswith(args.suffix): + log(f"ERROR: '{source_code}' already ends with '{args.suffix}'; refusing to copy a copy") + return 2 + if args.resume and args.force: + log("ERROR: --resume finishes an existing migration and --force restarts one; pick one") + return 2 + if args.resume and args.no_init: + # Not an error: --resume already skips the init, so --no-init asks for nothing extra. + log("NOTE: --no-init is redundant with --resume, which skips the LexBox init anyway") + + new_code = source_code + args.suffix + # The Send/Receive identifier -- and therefore the LexBox project code -- is + # a normalised form of the projectCode, so the two are not interchangeable. + # Everything LFMerge keys on locally (webwork, state, syncqueue, the project + # database) still uses the projectCode. + new_identifier = send_receive_identifier(new_code) + source_db = database_name(source_code) + target_db = database_name(new_code) + + log(f"Mongo: {config.safe_uri()}") + log(f"Main DB: {config.database}") + + # ---- validate -------------------------------------------------------- # + source = fetch_project(config, source_code) + if source is None: + log(f"ERROR: no project with projectCode '{source_code}' in {config.database}.projects") + return 1 + + app_name = source["appName"] or "lexicon" + new_name = args.project_name if args.project_name is not None else f"{source['projectName']}{args.suffix}" + + log("") + log(f"Source: {source['projectCode']} ({source['projectName']!r}, _id {source['id']})") + log(f" app={app_name} site={source['siteName']} members={source['memberCount']}") + log(f" database={source_db}") + if source["isArchived"]: + log(" NOTE: source project is archived") + + # This script exists to migrate projects INTO the Send/Receive system, so a + # project that already has an identifier is out of scope: copying it would + # produce a second project pointing at a repository that already has a + # different LF project syncing with it. + existing_identifier = (source["sendReceiveProjectIdentifier"] or "").strip() + if existing_identifier: + log("") + log(f"ERROR: '{source_code}' is already a Send/Receive project.") + log(f" sendReceiveProjectIdentifier = {existing_identifier!r}") + log(" This script only migrates projects that are not yet using Send/Receive.") + return 1 + log(" sendReceive: none (eligible for migration)") + + # LexBox will reject the code outright, so check before copying anything. + # Note LF project codes may contain uppercase, which LexBox does not allow. + if len(new_identifier) < LEXBOX_CODE_MIN_LENGTH or not LEXBOX_CODE_RE.match(new_identifier): + log("") + log(f"ERROR: projectCode '{new_code}' normalises to '{new_identifier}',") + log(" which is not a valid LexBox project code.") + log(f" LexBox requires at least {LEXBOX_CODE_MIN_LENGTH} characters matching") + log(f" {LEXBOX_CODE_RE.pattern} (lowercase letters, digits and hyphens only,") + log(" not starting with a hyphen).") + return 1 + + writing_systems = fetch_writing_systems(config, source_code) + # Read from the SOURCE, because this runs before anything is copied. Used to settle which list a + # writing system declared in both should leave. + anchor_counts = count_anchor_writing_system_usage(config, source_db) + ws_vernacular, ws_analysis, ws_unclassified = classify_writing_systems( + writing_systems, args.ws_vernacular, args.ws_analysis, anchor_counts + ) + + log("") + log(f"Copy to: {new_code} ({new_name!r})") + log(f" database={target_db}") + log(f" sendReceiveProjectIdentifier={new_identifier}", ) + if new_identifier != new_code: + log(f" (normalised from projectCode {new_code!r}; LexBox requires lowercase)") + log(f" LexBox project code={new_identifier}") + log(f" webwork={config.webwork_path(new_code)}") + log("") + log("Writing systems (from the Language Forge project config):") + log(f" lexeme={writing_systems['lexeme']} citationForm={writing_systems['citationForm']}") + log(f" definition={writing_systems['definition']} gloss={writing_systems['gloss']}") + log(f" all config fields={writing_systems['config_all']}") + log(f" project.inputSystems={writing_systems['declared']}") + if anchor_counts: + summary = ", ".join( + f"{tag}={counts.get('vernacular', 0)}v/{counts.get('analysis', 0)}a" + for tag, counts in sorted(anchor_counts.items()) + ) + log(f" entries with data (vernacular/analysis fields): {summary}") + + if ws_unclassified: + log("") + log("WARNING: these writing systems are in the project but are neither vernacular") + log(f" nor analysis by the field conventions: {', '.join(ws_unclassified)}") + log(" Treating them as ANALYSIS writing systems so that they still reach the") + log(" FieldWorks project. Check the result, and pass --ws-vernacular /") + log(" --ws-analysis if any of them belong somewhere else.") + log("") + + source_note = " (from --ws-vernacular)" if args.ws_vernacular else "" + log(f" wsVernacular: {ws_vernacular}{source_note}") + notes = [] + if args.ws_analysis: + notes.append("from --ws-analysis") + if ws_unclassified: + notes.append(f"includes {len(ws_unclassified)} unclassified") + source_note = f" ({', '.join(notes)})" if notes else "" + log(f" wsAnalysis: {ws_analysis}{source_note}") + if not ws_vernacular: + log("") + log("ERROR: LexBox requires at least one vernacular writing system, but this project's") + log(" config.entry.fields.lexeme.inputSystems is empty. Pass --ws-vernacular.") + return 1 + log("") + + # Custom fields are read from the SOURCE project: this runs before anything is copied, so the + # target database does not exist yet. Report only -- LfMerge cannot create custom fields in + # FieldWorks ("Custom field X from LF skipped, because we're not yet creating new custom fields + # in LCM"), so anything listed here will not survive the migration until something does. + custom_fields = describe_custom_fields( + fetch_custom_fields(config, source_code), + scan_custom_field_usage(config, source_db), + ) + report_custom_fields(custom_fields) + log("") + + # Two custom fields whose names differ only by case have to be separated before FieldWorks sees + # them. Checked here, before anything has been copied, so an unhandled clash costs nothing. + custom_field_renames, rename_problems = plan_custom_field_renames(custom_fields) + if custom_field_renames or rename_problems: + log("Custom field names differing only by case:") + report_custom_field_renames(custom_field_renames, rename_problems) + log("") + if rename_problems: + log("ERROR: refusing to migrate a project whose custom field names differ only by case") + log(" without knowing which one should keep its name. FLEx is inconsistent about") + log(" whether it treats such a pair as one field or two, so migrating them as they") + log(" are risks conflating two fields that hold different data.") + log(f" Add the name to rename to CASE_CLASH_CUSTOM_FIELD_RENAMES in " + f"{os.path.basename(sys.argv[0])}, having first checked which of the two should") + log(" keep its name and whether they hold the same kind of data.") + return 1 + # Everything downstream -- the FieldWorks definitions, and so the XML -- has to describe the + # fields under the names the copy will actually use. + apply_custom_field_renames(custom_fields, custom_field_renames) + + # LfMerge skips any LF custom field that does not already exist in FieldWorks, and the first + # Send/Receive is the only pass that would have carried the data. So a project with custom + # fields stops before Send/Receive and prints the definitions for a human to commit; --resume + # picks it up again afterwards. + custom_field_defs, custom_fields_dropped = plan_custom_field_definitions( + custom_fields, ws_vernacular, ws_analysis + ) + stop_for_custom_fields = bool(custom_field_defs) and not args.resume + + if args.custom_fields_only: + # Show the definitions this project would need, but write nothing: this flag exists to + # survey a project without touching it, and a file on disk is a change to something. + if custom_field_defs: + report_custom_field_definitions(custom_field_defs, custom_fields_dropped) + log_custom_properties_xml(build_custom_properties_xml(custom_field_defs)) + log("") + log("--custom-fields-only: stopping here, nothing was changed and no file was written.") + return 0 + + existing = fetch_project(config, new_code) + target_collections = count_collections(config, target_db) + if args.resume: + # --resume continues a run that stopped for custom fields, so the copy it is resuming has + # to still be there. Recreating it here would give the project a new LexBox repository and + # throw away the custom field commit that --resume exists to wait for. + if existing is None: + log(f"ERROR: --resume: there is no project '{new_code}' to resume.") + log(" Run without --resume first; that is what creates the copy.") + return 1 + if target_collections == 0: + log(f"ERROR: --resume: database '{target_db}' is empty, so the copy is not there to resume.") + return 1 + log(f"--resume: continuing the existing copy '{new_code}' (_id {existing['id']})") + # Checked before anything runs, so a forgotten `hg push` costs nothing. Raises Fatal, which + # ends the run with its message; nothing has been changed yet to roll back. + if custom_field_defs and not args.no_send_receive: + require_custom_properties_pushed( + make_lexbox_client(args), new_identifier, custom_field_defs, args.lexbox_host + ) + log("") + else: + if existing is not None and not args.force: + log(f"ERROR: project '{new_code}' already exists (_id {existing['id']}). Use --force to overwrite.") + return 1 + if target_collections > 0 and not args.force: + log(f"ERROR: database '{target_db}' already has {target_collections} collection(s). Use --force to overwrite.") + return 1 + + special = SPECIAL_LEXICON_INT_REFERENCES.get(source_code) + if special: + special_field, special_values = special + log(f"Special case: {source_code!r} has a lexicon reference to an option list item") + log(f" by number ({special_field}.values == {special_values}). That sense will be") + log(" coerced to strings along with the option list keys.") + log("") + + if args.dry_run: + log("Planned sequence:") + log(" 0. (already done above) report the source project's custom fields") + if args.resume: + log(" --resume: steps 1-6 are SKIPPED; the existing copy is picked up at step 7") + log(f" 1. copy database {source_db} -> {target_db}") + log(f" 2. coerce numeric option list keys in {target_db} to strings" + + (", and the special-case lexicon sense" if special else "")) + empty_guid_count = len(find_empty_entry_guids(config, source_db)) + log(f" 3. replace empty entry guids with fresh UUID4s" + f" ({empty_guid_count} found in {source_db})") + if custom_field_renames: + log(f" 3b. rename {len(custom_field_renames)} custom field(s) in {target_db}.lexicon" + " and in the copied config:") + for rename in custom_field_renames: + log(f" {rename['level']:8s} {rename['old_key']} -> {rename['new_key']}") + log(" 4. stage assets" + (" (SKIPPED: --no-assets)" if args.no_assets else "")) + log(" 5. create project record") + if args.no_init: + log(f" 6. SKIPPED: --no-init; assuming LexBox project {new_identifier!r} already exists") + else: + log(f" 6. init_lexbox_project: log in to {split_lexbox_host(args.lexbox_host)[0]}://" + f"{split_lexbox_host(args.lexbox_host)[1]} as {args.lexbox_user!r},") + log(" POST /api/project/initFwDataProject") + if args.no_send_receive: + log(" 7-9. SKIPPED: --no-send-receive") + elif stop_for_custom_fields: + log(f" 7-9. SKIPPED: {len(custom_field_defs)} custom field(s) found. The run will stop after") + log(f" step 6 and write {CUSTOM_PROPERTIES_FILENAME} for a human to commit to the") + log(" project's Mercurial repository, then be resumed with --resume.") + else: + log(f" 7. clear any stale webwork clone and state file for {new_code},") + log(" then Send/Receive #1 (push Mongo data), wait for completion") + if args.no_assets: + log(" 8-9. SKIPPED: --no-assets") + else: + log(" 8. move assets into webwork LinkedFiles, symlink back") + log(" 9. Send/Receive #2 (push assets), wait for completion") + log("") + log("--dry-run: stopping here, nothing was changed.") + return 0 + + # ---- copy ------------------------------------------------------------ # + # Undo actions, applied in reverse order if a later step fails. Cleared once + # the first Send/Receive starts, since by then data has left for LexBox and + # LFMerge owns the webwork tree. + undo: list[tuple[str, Callable[[], None]]] = [] + new_id = None + lexbox_guid = None + coercion: dict | None = None + lexicon_coercion: dict | None = None + guid_fix: dict | None = None + rename_result: dict | None = None + + try: + if args.resume: + # Everything below was done by the run that stopped for custom fields. Redoing any + # of it would replace the LexBox project, and with it the repository holding the + # custom field commit that --resume exists to wait for. + new_id = existing["id"] + log("--resume: skipping the copy, the project record and the LexBox init") + else: + if args.force and existing is not None: + log(f"--force: removing existing project '{new_code}' (_id {existing['id']})") + delete_project_record(config, existing["id"]) + + log(f"Copying database {source_db} -> {target_db}") + copy_project_database(config, source_db, target_db, drop=args.force) + undo.append( + ( + f"drop database {target_db}", + lambda: mongosh_json( + config, + f"db.getSiblingDB({json.dumps(target_db)}).dropDatabase();\nprint(JSON.stringify({{ok:1}}));\n", + what=f"Dropping {target_db}", + ), + ) + ) + + # Before anything else looks at the copy: option list keys are meant to + # be strings, and the lexicon already references them that way. Runs + # inside the same try, so a failure here rolls the copy back. + log(f"Coercing numeric option list keys in {target_db}") + coercion = coerce_option_list_keys(config, target_db) + lexicon_coercion = coerce_special_lexicon_references(config, target_db, source_code) + + # Also before the first Send/Receive: an entry with an empty guid would + # be pushed to FieldWorks under the all-zeros GUID. + log(f"Replacing empty entry guids in {target_db}") + guid_fix = fix_empty_entry_guids(config, target_db) + + if custom_field_renames: + log(f"Renaming custom fields in {target_db}") + rename_result = rename_custom_fields_in_lexicon( + config, target_db, custom_field_renames + ) + + if args.no_assets: + log("Skipping assets (--no-assets)") + else: + log("Staging assets") + staged = stage_assets(config, app_name, source_db, target_db) + if staged: + undo.append( + ( + f"remove assets {staged}", + lambda path=staged: shutil.rmtree(path, ignore_errors=True), + ) + ) + + log("Creating project record") + new_id = create_project_record( + config, source_code, new_code, new_name, new_identifier, custom_field_renames + ) + undo.append( + (f"remove project record {new_id}", lambda pid=new_id: delete_project_record(config, pid)) + ) + + if args.no_init: + # Nothing here is rolled back on failure -- init_lexbox_project has no undo entry, + # which is exactly how a 504 leaves a LexBox project behind while everything on the + # Language Forge side is undone. Re-creating it would fail on the existing code, so + # this trusts the project that is already there. + log(f"Skipping LexBox init (--no-init): assuming project {new_identifier!r} and its") + log(f" repository already exist at {args.lexbox_host}, with the right writing systems.") + log(f" Expected wsVernacular: {', '.join(ws_vernacular)}") + log(f" Expected wsAnalysis: {', '.join(ws_analysis) or '(none)'}") + else: + log("Initializing LexBox project") + lexbox_client = make_lexbox_client(args) + lexbox_guid = init_lexbox_project( + lexbox_client, + new_identifier, + new_name, + ws_vernacular, + ws_analysis, + ) + + if stop_for_custom_fields: + # Deliberately before any Send/Receive: LfMerge would skip every one of these fields + # and the data in them, and the first Send/Receive is the only pass that would have + # carried it. The definitions have to reach the repository first. + custom_properties_path = emit_custom_properties( + custom_field_defs, custom_fields_dropped, new_code, new_identifier, args.lexbox_host + ) + log("") + log("Send/Receive NOT run, and no data has been pushed to LexBox yet.") + log(f"Once {custom_properties_path} has been committed and pushed, finish with:") + log(f" {os.path.basename(sys.argv[0])} {source_code} --suffix {args.suffix} --resume") + elif args.no_send_receive: + log("Skipping both Send/Receive runs (--no-send-receive)") + log(" NOTE: assets are still real directories; they will not be linked into") + log(" webwork until a Send/Receive has created it.") + else: + # A newly created LexBox project has a root changeset no earlier clone can share, so + # any clone a previous attempt left behind has to go, or LFMerge would reuse it and + # Chorus would abort the pull as unrelated. Done under --no-init as well: the clone is + # then of the same repository and would not be rejected, but a clone left by an attempt + # that failed partway is not worth trusting either. + stale = clear_send_receive_state(config, new_code, args.suffix) + for path in stale: + log(f"Removed stale Send/Receive state from an earlier attempt: {path}") + + # Past this point a failure is not safely reversible: LFMerge has + # started pushing to LexBox and owns the webwork clone. + undo.clear() + + log("Send/Receive #1: pushing Mongo data to LexBox") + send_receive(config, new_code, timeout=args.sr_timeout, label="1/2") + + if args.no_assets: + log("Skipping asset linking and Send/Receive #2 (--no-assets)") + else: + log("Moving assets into the Send/Receive work tree") + link_assets_into_webwork(config, app_name, target_db, new_code) + + log("Send/Receive #2: pushing assets to LexBox") + send_receive(config, new_code, timeout=args.sr_timeout, label="2/2") + + except Fatal as exc: + log("") + log(f"ERROR: {exc}") + if undo: + log("Rolling back:") + for description, action in reversed(undo): + log(f" {description}") + try: + action() + except Exception as cleanup_exc: # noqa: BLE001 - best effort + log(f" rollback step failed: {cleanup_exc}") + elif new_id is not None: + log("NOT rolling back: Send/Receive has already run, so the copy and its") + log(f"LexBox project may hold real data. Inspect '{new_code}' before retrying.") + return 1 + + # ---- report ---------------------------------------------------------- # + log("") + if stop_for_custom_fields: + log(f"Paused for custom fields; not finished. '{source_code}' was left untouched.") + else: + log(f"Done. '{source_code}' was left untouched.") + log(f" new project: {new_code} (_id {new_id})") + log(f" new database: {target_db}") + if coercion: + log(f" option list keys coerced to strings: {coercion['keysCoerced']}" + f" (in {coercion['listsChanged']} list(s))") + if lexicon_coercion and lexicon_coercion["valuesCoerced"]: + log(f" lexicon values coerced to strings: {lexicon_coercion['valuesCoerced']}" + f" (in {lexicon_coercion['entriesChanged']} entry/entries)") + if guid_fix and guid_fix["fixed"]: + log(f" empty entry guids replaced: {guid_fix['fixed']}") + if rename_result and rename_result["keysRenamed"]: + log(f" custom field keys renamed: {rename_result['keysRenamed']}" + f" (in {rename_result['entriesChanged']} entry/entries)") + for rename in custom_field_renames: + log(f" {rename['old_name']!r} -> {rename['new_name']!r}") + if new_identifier != new_code: + log(f" S/R identifier: {new_identifier}") + if lexbox_guid: + log(f" LexBox GUID: {lexbox_guid}") + elif args.no_init and not args.resume: + log(f" LexBox: init skipped (--no-init); used the existing {new_identifier!r}") + if not args.no_send_receive and not stop_for_custom_fields: + log(f" state file: {config.state_file(new_code)}") + if not args.no_assets: + log(f" assets now symlink into {config.linked_files_path(new_code)}") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except Fatal as exc: + log(f"ERROR: {exc}") + sys.exit(1) + except KeyboardInterrupt: + log("Interrupted.") + sys.exit(130) diff --git a/scripts/migrate-projects.sh b/scripts/migrate-projects.sh new file mode 100755 index 0000000000..3c23f3a850 --- /dev/null +++ b/scripts/migrate-projects.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Read the non-Send/Receive project list and print each project's code and Mongo ObjectId. +# +# nonSRProjects.txt is tab-separated, one project per line, largest first: +# +set -euo pipefail + +# Resolve the list next to this script, so it can be run from any directory. +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +project_list="${1:-$script_dir/projectsWorthMigrating.txt}" + +KUBE_CONTEXT="lf-prod" + +MONGODB_PASS=$(kubectl --context=$KUBE_CONTEXT get secret/mongo-auth -o json | jq -r .data.MONGODB_PASS | base64 -d) +MONGODB_USER="admin" + +echo "Starting migration of $(wc -l $project_list) projects, largest ones first..." +echo "" + +# IFS is set per-read so only tabs split the fields: a project name or code could in +# principle contain a space, and the default IFS would break it apart. +# The `|| [[ -n "$project_code" ]]` keeps the last line if the file has no trailing newline. +# +# entry_count and comment_count are read into their own variables but not used yet; they are what +# the list is sorted by and what says whether a project has comments to convert, so a real +# migration loop here will want them. +# shellcheck disable=SC2034 + +# Old: +#while IFS=$'\t' read -u 5 -r project_code entry_count comment_count project_id || [[ -n "$project_code" ]]; do + +# New: + +while IFS=$'\t' read -u 5 -r project_code entry_count comment_count project_id edit_category human_readable_explanation || [[ -n "$project_code" ]]; do + # Skip blank lines and comments, so the list can be annotated later. + [[ -z "$project_code" || "$project_code" == \#* ]] && continue + + echo "About to fetch $project_code ($project_id) from production..." + echo "" + # backup.mjs uses MONGO_USER and MONGO_PASS, which is why they were set earlier + time MONGO_USER=$MONGODB_USER MONGO_PASS=$MONGODB_PASS node backup.mjs "$project_id" prod + echo "Done with exit code $?" + echo "$project_code should have $entry_count entries and $comment_count comments; does that look right?" + echo "" + time docker compose exec -T app python3 /var/www/scripts/copy_project_for_migration.py "$project_code" --lexbox-host http://admin.localhost + echo "Done with exit code $?" +done 5< "$project_list" diff --git a/scripts/migrate_custom_field_projects.sh b/scripts/migrate_custom_field_projects.sh new file mode 100755 index 0000000000..9cd39a90ec --- /dev/null +++ b/scripts/migrate_custom_field_projects.sh @@ -0,0 +1,334 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2004 # ${P[...]} inside (( )) on purpose: under set -u a missing key is +# an error, where a bare P[...] would quietly read as 0. +# Migrate a list of projects that HAVE custom fields into Send/Receive, in two stages. +# +# scripts/migrate_custom_field_projects.sh projectsWorthMigrating-custom-fields.txt +# scripts/migrate_custom_field_projects.sh - < list.txt +# scripts/migrate_custom_field_projects.sh list.txt -- --lexbox-host https://lexbox.org +# +# LfMerge cannot create custom fields, so copy_project_for_migration.py stops each of these +# projects short of Send/Receive and writes the FLExProject.CustomProperties file a human has to +# commit to the project's LexBox repository first. This script runs that over the whole list: +# +# FIRST RUN -- stage 1 +# Each project is copied and its LexBox project created (the migration run without options, +# which stops for custom fields). Its CustomProperties file is copied out of the container into +# ./custom-fields/, and the copy is checked against the list: the listed entry and comment +# counts (or the source's current counts, with a warning, if it has changed since), as many +# entries with custom field data as the source, no Send/Receive yet, and one definition in the +# file for every custom field the migration found. Then "stage1" goes into +# ./customFieldsMigrationProgress.txt. +# +# With stage 1 done for every project, the script prints, grouped together at the end, the hg +# commands that commit each project's file into a clone under ./custom-fields-repos/, and stops +# (exit status 3). +# +# SECOND RUN -- stage 2 +# For every project, the clone at ./custom-fields-repos/ must exist, come from that +# project's LexBox repository, and have FLExProject.CustomProperties committed (on the same branch +# as revision 0), with no uncommitted changes, with every custom field the generated file defines, +# and with no FILL-IN-LIST-ROOT-GUID placeholder left in it. A committed file is taken to be pushed: +# hg outgoing would need the LexBox credentials. If any project fails, nothing goes on to stage 2; +# the script lists what is wrong, prints the hg commands again for those projects, and exits 3. +# +# Otherwise each project is finished with `--resume` (both Send/Receives) and verified as +# migrate_non_custom_field_projects.sh verifies a finished project -- including that the copy +# still has as many entries with custom field data as the source, which is what would show that +# LfMerge skipped the custom fields. Then "stage2" goes into the progress log. +# +# RESUMING +# As in migrate_non_custom_field_projects.sh: the first problem stops the run, and the next run +# skips what the progress log records and picks the interrupted project up from the state it is +# really in. A stage 1 that was cut off is started again; a project whose copy has somehow been +# through Send/Receive before stage 2 is stopped for a human, because its custom field data may +# not have reached LexBox. +# +# Before either stage, every project still to do is checked, as in the other script, except that +# each one must have custom fields. See migration_batch_common.sh for the environment variables. +# +# Needs hg on the PATH of the machine this runs on, for the stage 2 check. + +set -euo pipefail + +SCRIPT_NAME=$(basename -- "$0") +PROGRESS_FILE=customFieldsMigrationProgress.txt +MODE=custom +# shellcheck source=migration_batch_common.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/migration_batch_common.sh" + +CF_DIR="$WORK_DIR/custom-fields" +CLONES_DIR="$WORK_DIR/custom-fields-repos" +CUSTOM_PROPERTIES=FLExProject.CustomProperties +PLACEHOLDER=FILL-IN-LIST-ROOT-GUID +EXIT_WAITING_FOR_COMMITS=3 + +declare -A COMMIT_STATE=() COMMIT_NOTES=() + +usage_details() { + cat < 0)); then + fail "$code" "The copy ${P[new_code]} has already been through a Send/Receive, before stage 1 was recorded." \ + "LfMerge skips custom fields that are not yet defined in FieldWorks, so their data may not have" \ + "reached LexBox. Check the LexBox project '${P[lexbox_code]}' by hand before doing anything else." + fi + set_restart_flags "$code" + + log=$(new_log "$code" stage1) + run_migration "$code" "$log" "${RESTART_FLAGS[@]}" || fail "$code" "Stage 1 failed; see $log" + + if grep -q '^Paused for custom fields' "$log"; then + path=$(sed -n 's/^Written to: //p' "$log" | tail -n 1) + [[ -n $path ]] || fail "$code" "The migration paused but did not say where it wrote $CUSTOM_PROPERTIES; see $log" + mkdir -p -- "$CF_DIR" + dest=$(cf_file "$code") + app_exec cat -- "$path" "$dest.part" || + fail "$code" "Could not copy $path out of the app container." + mv -- "$dest.part" "$dest" + say " custom field definitions saved to $dest" + verify_stage1 "$code" + progress_add "$code" stage1 + elif grep -q '^Done\. ' "$log"; then + # Its custom fields have gone since the check before stage 1, so the migration went all + # the way. Nothing is waiting on a commit; treat it as a finished migration. + warn "$code: needed no custom field definitions after all, so stage 1 migrated it completely" + VERIFIED_STAGE=stage2 + verify_finished "$code" + progress_add "$code" stage1 + progress_add "$code" stage2 + ((MIGRATED += 1)) + else + fail "$code" "Stage 1 exited without pausing for custom fields or finishing; see $log" + fi +} + +verify_stage1() { + local code=$1 file written + PROBLEMS=() + probe_or_fail "$code" custom-fields + check_copy "$code" + ! copy_synced || PROBLEMS+=("the copy has already been through a Send/Receive") + file=$(cf_file "$code") + written=$(grep -c '^[[:space:]]*/dev/null || true) + default=${default%/} + if [[ $default != */hg/"$lexbox" ]]; then + COMMIT_STATE[$code]=problem + COMMIT_NOTES[$code]="the clone at $repo comes from '${default:-nowhere}', not from LexBox project $lexbox" + return + fi + + local changes rev + changes=$(hgq status -- "$CUSTOM_PROPERTIES" | tr '\n' ' ') + changes=${changes% } + rev=$(hgq log -r "last(file('path:$CUSTOM_PROPERTIES') and not 0)" -T '{node|short}' 2>/dev/null || true) + if [[ -z $rev ]]; then + COMMIT_STATE[$code]=waiting + COMMIT_NOTES[$code]="cloned, but $CUSTOM_PROPERTIES has not been committed${changes:+ (hg status: $changes)}" + return + fi + [[ -z $changes ]] || problems+=("$CUSTOM_PROPERTIES has uncommitted changes (hg status: $changes)") + + local committed key missing=() + committed=$(hgq cat -r "$rev" -- "$CUSTOM_PROPERTIES") + while IFS= read -r key; do + grep -qF -- "$key" <<<"$committed" || missing+=("$key") + done < <(grep -o 'key="[^"]*"' "$file") + ((${#missing[@]} == 0)) || + problems+=("the committed file (changeset $rev) lacks ${#missing[@]} field(s) the generated one defines: ${missing[*]}") + ! grep -qF "$PLACEHOLDER" <<<"$committed" || + problems+=("the committed file still contains $PLACEHOLDER; a listRoot has to be filled in") + + local branch branch0 + branch=$(hgq log -r "$rev" -T '{branch}') + branch0=$(hgq log -r 0 -T '{branch}') + [[ $branch == "$branch0" ]] || + problems+=("changeset $rev is on branch '$branch', but the project's history is on '$branch0'") + + if ((${#problems[@]})); then + COMMIT_STATE[$code]=problem + COMMIT_NOTES[$code]=$(join_notes "${problems[@]}") + return + fi + + notes+=("committed in changeset $rev") + if cmp -s <(hgq cat -r "$rev" -- "$CUSTOM_PROPERTIES") "$file"; then + notes+=("identical to the generated file") + else + notes+=("edited from the generated file, but defines all its fields") + fi + [[ $(hgq log -r "$rev" -T '{phase}') != draft ]] || + notes+=("still in draft phase, so make sure it has been pushed") + COMMIT_STATE[$code]=ok + COMMIT_NOTES[$code]=$(join_notes "${notes[@]}") +} + +join_notes() { + local joined + printf -v joined '%s; ' "$@" + printf '%s' "${joined%; }" +} + +print_commit_block() { + local code=$1 lexbox=${LEXBOX_CODE[$1]} file repo reviews + file=$(cf_file "$code") + repo="$CLONES_DIR/$lexbox" + printf '\n----- %s (LexBox project %s) -----\n' "$code" "$lexbox" + printf '# %s\n' "${COMMIT_NOTES[$code]}" + [[ -s $file ]] || return 0 + + [[ -d $repo/.hg ]] || printf 'hg clone %q %q\n' "${HG_URL[$code]}" "$repo" + printf "hg --cwd %q up 7500002.7000072\n" "$repo" + printf 'cp %q %q\n' "$file" "$repo/$CUSTOM_PROPERTIES" + if grep -qF "$PLACEHOLDER" "$file"; then + printf '# EDIT FIRST: replace %s in %s with the GUID of the list it names\n' "$PLACEHOLDER" "$repo/$CUSTOM_PROPERTIES" + printf '# (the REVIEW comment above it says which); that list has to exist in the project.\n' + fi + reviews=$(grep -c 'REVIEW:' "$file" || true) + ((reviews == 0)) || printf '# Check the %s REVIEW comment(s) in the file before committing.\n' "$reviews" + printf "hg --cwd %q commit -A -m 'Add custom fields' %s\n" "$repo" "$CUSTOM_PROPERTIES" + printf 'hg --cwd %q push\n' "$repo" +} + +# Check every project waiting for stage 2; if any is not ready, print the hg commands for those, +# grouped at the end, and stop. +require_commits() { + local code not_ready=() + heading "Custom field commits (clones in $CLONES_DIR)" + for code in "$@"; do + check_commit "$code" + printf ' %-8s %s: %s\n' "${COMMIT_STATE[$code]}" "$code" "${COMMIT_NOTES[$code]}" + [[ ${COMMIT_STATE[$code]} == ok ]] || not_ready+=("$code") + done + ((${#not_ready[@]})) || return 0 + + heading "Commit the custom fields for ${#not_ready[@]} project(s)" + say "Run these from any directory. The second Send/Receive cannot start until every project in the" + say "list is ready, because the migration's first Send/Receive is the only one that carries the" + say "custom field data. A commit is taken to have been pushed; this script cannot check that." + for code in "${not_ready[@]}"; do + print_commit_block "$code" + done + printf '\nThen run this script again with the same list (and the same options) to start stage 2.\n' + exit "$EXIT_WAITING_FOR_COMMITS" +} + +# --------------------------------------------------------------------------- # + +main() { + init "$@" + command -v hg >/dev/null || die "hg is not on the PATH; it is needed to check the custom field commits" + + local code pending=() stage1=() stage2=() + for code in "${CODES[@]}"; do + progress_has "$code" stage2 || pending+=("$code") + done + say "$((${#CODES[@]} - ${#pending[@]})) of ${#CODES[@]} project(s) already migrated (in $PROGRESS_FILE)." + if ((${#pending[@]} == 0)); then + say "Nothing to do." + return 0 + fi + + preflight "${pending[@]}" + + for code in "${pending[@]}"; do + progress_has "$code" stage1 || stage1+=("$code") + done + if ((${#stage1[@]})); then + heading "Stage 1: ${#stage1[@]} project(s)" + local index=0 + for code in "${stage1[@]}"; do + ((index += 1)) + heading "[stage 1, $index/${#stage1[@]}] $code" + migrate_stage1 "$code" + done + fi + + # Stage 1 may have finished a project outright; see migrate_stage1. + for code in "${pending[@]}"; do + progress_has "$code" stage2 || stage2+=("$code") + done + if ((${#stage2[@]})); then + require_commits "${stage2[@]}" + + heading "Stage 2: ${#stage2[@]} project(s)" + local index=0 + for code in "${stage2[@]}"; do + ((index += 1)) + heading "[stage 2, $index/${#stage2[@]}] $code" + migrate_to_finish "$code" stage2 + done + fi + + print_summary +} + +main "$@" diff --git a/scripts/migrate_non_custom_field_projects.sh b/scripts/migrate_non_custom_field_projects.sh new file mode 100755 index 0000000000..bee5af4652 --- /dev/null +++ b/scripts/migrate_non_custom_field_projects.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Migrate a list of projects that have NO custom fields into Send/Receive, one at a time. +# +# scripts/migrate_non_custom_field_projects.sh projectsWorthMigrating-no-custom-fields.txt +# scripts/migrate_non_custom_field_projects.sh - < list.txt +# scripts/migrate_non_custom_field_projects.sh list.txt -- --lexbox-host https://lexbox.org +# +# For each project in the list, copy_project_for_migration.py makes the "-migrate" copy and takes it +# all the way through both Send/Receives. Each one is then checked against the list: the source +# must have the listed Mongo _id, and the copy must have the listed numbers of entries and comments +# (or, if the source has changed since the list was made, the source's current numbers, with a +# warning), as many entries with custom field data as the source, a completed Send/Receive and its +# assets linked into the Send/Receive work tree. Only then is the project code added to +# ./nonCustomFieldsMigrationProgress.txt. +# +# Before migrating anything, every project is checked, and nothing is migrated if any check fails: +# the source exists with the listed _id and is not already a Send/Receive project, it has no custom +# fields (those belong to migrate_custom_field_projects.sh), no leftover LexBox project is in the +# way, and the migration's own --dry-run succeeds. +# +# RESUMING +# The first problem stops the run. Running it again skips every project in the progress log and +# picks up the one that stopped from whatever state it is really in, not from what the log says: +# - nothing created, or the first Send/Receive never completed: start that project again, +# keeping its LexBox project if this script created it (--force --no-init); +# - the first Send/Receive completed: just move the assets and run the second one; +# - everything ran but was never verified: verify it. +# A project whose migration or Send/Receive is still running is left alone, with a message. +# +# See migration_batch_common.sh for the environment variables and the files this writes. + +set -euo pipefail + +SCRIPT_NAME=$(basename -- "$0") +PROGRESS_FILE=nonCustomFieldsMigrationProgress.txt +MODE=non-custom +# shellcheck source=migration_batch_common.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/migration_batch_common.sh" + +usage_details() { + cat < 0)); then + printf 'has %s custom field definition(s); migrate it with migrate_custom_field_projects.sh' "$count" + fi +} + +main() { + init "$@" + + local code pending=() + for code in "${CODES[@]}"; do + progress_has "$code" || pending+=("$code") + done + say "$((${#CODES[@]} - ${#pending[@]})) of ${#CODES[@]} project(s) already migrated (in $PROGRESS_FILE)." + if ((${#pending[@]} == 0)); then + say "Nothing to do." + return 0 + fi + + preflight "${pending[@]}" + + local index=0 + for code in "${pending[@]}"; do + ((index += 1)) + heading "[$index/${#pending[@]}] $code" + migrate_to_finish "$code" whole + done + + print_summary +} + +main "$@" diff --git a/scripts/migration_batch_common.sh b/scripts/migration_batch_common.sh new file mode 100644 index 0000000000..dfc775184c --- /dev/null +++ b/scripts/migration_batch_common.sh @@ -0,0 +1,551 @@ +# shellcheck shell=bash +# shellcheck disable=SC2004 # ${P[...]} inside (( )) on purpose: under set -u a missing key is +# an error, where a bare P[...] would quietly read as 0. +# Shared code for migrate_custom_field_projects.sh and migrate_non_custom_field_projects.sh. +# Sourced by them, not run. The calling script sets, before sourcing this file: +# +# SCRIPT_NAME its own name, for messages +# PROGRESS_FILE the progress log's file name (created in the current directory) +# MODE "custom" or "non-custom" +# +# WHERE THINGS LIVE +# Everything this writes goes in the directory it is run from: +# $PROGRESS_FILE one line per finished project (or, for custom fields, per finished stage) +# migration-logs/ the full output of every migration run, one file per run +# The migration itself runs inside the app container, through +# ${MIGRATION_APP_EXEC:-docker compose exec -T app} +# started from the root of this repository so that docker compose finds docker-compose.yml. +# +# ENVIRONMENT +# MIGRATION_APP_EXEC command prefix that runs a command in the app container +# MIGRATION_SCRIPT path of copy_project_for_migration.py INSIDE the container +# (default /var/www/scripts/copy_project_for_migration.py) +# LEXBOX_PASSWORD passed through to the container when set, as the migration script reads it + +REPO_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +WORK_DIR=$PWD +LOG_DIR="$WORK_DIR/migration-logs" +MIGRATION_SCRIPT=${MIGRATION_SCRIPT:-/var/www/scripts/copy_project_for_migration.py} +HELPER_SCRIPT="$(dirname -- "$MIGRATION_SCRIPT")/migration_batch_helper.py" +read -r -a APP_EXEC <<<"${MIGRATION_APP_EXEC:-docker compose exec -T app}" + +# Options a batch may pass through to the migration script. Everything else it controls itself: +# --resume, --force, --no-init, --dry-run and --custom-fields-only are chosen per project, and the +# per-project options (--project-name, --ws-vernacular, --ws-analysis) make no sense for a batch. +# --no-assets and --no-send-receive would leave every project half done by design. +PASSABLE_OPTIONS=(--lexbox-host --lexbox-user --lexbox-password --lexbox-auth --lexbox-timeout --sr-timeout --suffix) + +LIST_FILE="" +FORWARD=() +CODES=() +declare -A LIST_ENTRIES=() LIST_COMMENTS=() LIST_ID=() +declare -A NEW_CODE=() LEXBOX_CODE=() HG_URL=() +declare -A P=() # the last probe's results +PROBLEMS=() # what the last verification found wrong +WARNINGS=() # collected for the summary at the end +MIGRATED=0 + +# --------------------------------------------------------------------------- # +# messages +# --------------------------------------------------------------------------- # + +say() { printf '%s\n' "$*"; } +heading() { printf '\n=== %s ===\n' "$*"; } +die() { + printf '%s: ERROR: %s\n' "$SCRIPT_NAME" "$*" >&2 + exit 1 +} +warn() { + WARNINGS+=("$*") + printf 'WARNING: %s\n' "$*" >&2 +} + +# Stop the whole batch over one project. Everything finished so far is in the progress log, so +# running the script again carries on from this project. +fail() { + local code=$1 + shift + printf '\n%s: STOPPED at %s\n' "$SCRIPT_NAME" "$code" >&2 + local line + for line in "$@"; do + printf ' %s\n' "$line" >&2 + done + printf '\nProjects finished before this one are recorded in %s;\n' "$PROGRESS_FILE" >&2 + printf 'run the script again with the same list to carry on from %s.\n' "$code" >&2 + exit 1 +} + +on_interrupt() { + printf '\n%s: interrupted.\n' "$SCRIPT_NAME" >&2 + printf 'A migration started inside the app container keeps running when this script is stopped.\n' >&2 + printf 'Running the script again is safe: it waits for nothing, but refuses to touch a project\n' >&2 + printf 'whose migration or Send/Receive is still in progress, and says so.\n' >&2 + exit 130 +} + +usage() { + cat </dev/null || die "'${APP_EXEC[0]}' not found (set MIGRATION_APP_EXEC?)" + mkdir -p -- "$LOG_DIR" + + # Two runs at once would migrate the same projects twice over. The lock lasts as long as this + # process, because file descriptor 9 stays open until it exits. + exec 9>>"$WORK_DIR/$PROGRESS_FILE" + flock -n 9 || die "another run is already using $WORK_DIR/$PROGRESS_FILE" + + trap on_interrupt INT TERM +} + +# --------------------------------------------------------------------------- # +# progress log +# --------------------------------------------------------------------------- # + +# Non-custom projects have one stage, recorded as the bare project code. Custom field projects have +# two, recorded as "codestage1" and "codestage2". +progress_line() { + if [[ -n ${2:-} ]]; then + printf '%s\t%s' "$1" "$2" + else + printf '%s' "$1" + fi +} +progress_has() { + grep -qxF -- "$(progress_line "$1" "${2:-}")" "$WORK_DIR/$PROGRESS_FILE" +} +progress_add() { + printf '%s\n' "$(progress_line "$1" "${2:-}")" >>"$WORK_DIR/$PROGRESS_FILE" +} + +# --------------------------------------------------------------------------- # +# running things in the app container +# --------------------------------------------------------------------------- # + +app_exec() { + (cd -- "$REPO_ROOT" && "${APP_EXEC[@]}" "$@") +} +app_python() { + local env_args=() + [[ -n ${LEXBOX_PASSWORD+set} ]] && env_args=(env "LEXBOX_PASSWORD=$LEXBOX_PASSWORD") + app_exec "${env_args[@]}" python3 "$@" +} + +new_log() { + printf '%s/%s.%s.%s.log' "$LOG_DIR" "$1" "$2" "$(date +%Y%m%d-%H%M%S)" +} + +# probe CODE [EXTRAS] -- fill P with the helper's description of a project and its copy. +# shellcheck disable=SC2034 # NEW_CODE, LEXBOX_CODE and HG_URL are read by the calling scripts +probe() { + local code=$1 extras=${2:-} output line + local args=(probe "$code") + [[ -n $extras ]] && args+=(--with "$extras") + output=$(app_python "$HELPER_SCRIPT" "${args[@]}" -- "${FORWARD[@]}" &1 | tee -- "$log" + status=${PIPESTATUS[0]} + set -e + return "$status" +} + +run_migration() { + local code=$1 log=$2 + shift 2 + say " running: copy_project_for_migration.py $code $*" + run_logged "$log" "$MIGRATION_SCRIPT" "$code" "$@" "${FORWARD[@]}" +} + +run_finish() { + local code=$1 log=$2 + say " running: the remaining asset steps (move assets into webwork, Send/Receive #2)" + run_logged "$log" "$HELPER_SCRIPT" finish "$code" -- "${FORWARD[@]}" +} + +# --------------------------------------------------------------------------- # +# reading a probe +# --------------------------------------------------------------------------- # + +copy_synced() { + # The copy starts with the source's lastSyncedDate, and LfMerge sets it at the end of every + # successful Send/Receive. So a later date than the source's means one has completed. + [[ ${P[copy_exists]} == 1 ]] && ((${P[copy_last_synced]} > ${P[source_last_synced]})) +} + +leftovers_exist() { + [[ ${P[copy_exists]} == 1 ]] || ((${P[copy_collections]} > 0)) +} + +# Non-empty output (the reason) if the project must not be touched right now. +busy_reason() { + if [[ ${P[migration_running]} == 1 ]]; then + printf 'a migration of this project is still running in the app container' + elif [[ ${P[queue_present]} == 1 ]]; then + printf 'a Send/Receive of %s is queued but has not started (is the lfmerge container running?)' "${P[new_code]}" + elif [[ ${P[sr_class]} == in_progress ]]; then + printf 'a Send/Receive of %s is in progress (SRState=%s)' "${P[new_code]}" "${P[sr_state]}" + elif [[ ${P[sr_class]} == unknown ]]; then + printf "the Send/Receive state of %s is '%s', which is neither finished nor failed" "${P[new_code]}" "${P[sr_state]}" + fi +} + +# Whether an existing LexBox project with the copy's code was created by an earlier run of this +# migration. The only record of that is the logs: the migration logs this line just before it +# calls initFwDataProject, which is how a 504 can leave a project behind with nothing on the +# Language Forge side. +lexbox_created_by_us() { + grep -qsF "Initializing LexBox project" "$LOG_DIR/$1".*.log +} + +# How to (re)start a migration whose first Send/Receive has not happened yet. Sets RESTART_FLAGS, +# or fails if a LexBox project is in the way. (Not printed for capture: fail has to be able to +# stop this script, not a subshell.) +set_restart_flags() { + local code=$1 + RESTART_FLAGS=() + if [[ ${P[lexbox_exists]} == 1 ]]; then + lexbox_created_by_us "$code" || fail "$code" \ + "LexBox already has a project '${P[lexbox_code]}', and nothing in $LOG_DIR shows this script creating it." \ + "It may be left over from a test. Delete it in LexBox and run this script again, or, if you are" \ + "sure it is an empty project from an earlier attempt at this migration, run by hand:" \ + " copy_project_for_migration.py $code --force --no-init" + # Rebuild the Language Forge side and keep the LexBox project. --force also clears away + # whatever an interrupted attempt left in Mongo. + RESTART_FLAGS=(--force --no-init) + elif leftovers_exist; then + RESTART_FLAGS=(--force) + fi +} + +# Whether a LexBox project with the copy's code is one this batch may use. Stage 1 of a custom +# field migration creates it, so once that is recorded it is ours by definition. +lexbox_project_in_the_way() { + local code=$1 + [[ ${P[lexbox_exists]} == 1 ]] || return 1 + [[ $MODE == custom ]] && progress_has "$code" stage1 && return 1 + copy_synced && return 1 + ! lexbox_created_by_us "$code" +} + +# --------------------------------------------------------------------------- # +# verification +# --------------------------------------------------------------------------- # + +# check_count CODE WHAT LISTED -- compare one count in the copy with the list, and with the source. +check_count() { + local code=$1 what=$2 listed=$3 copied=${P[copy_$2]} now=${P[source_$2]} + [[ $copied == "$listed" ]] && return 0 + if [[ $copied == "$now" ]]; then + warn "$code: the list says $listed $what but the source now has $now; the copy matches the source" + return 0 + fi + PROBLEMS+=("$what: the copy has $copied, the list says $listed, and the source has $now") +} + +# Checks that apply to the copy at any stage, against a fresh probe. +check_copy() { + local code=$1 + [[ ${P[copy_exists]} == 1 ]] || { + PROBLEMS+=("the copy ${P[new_code]} does not exist") + return + } + [[ ${P[copy_sr]} == "${P[lexbox_code]}" ]] || + PROBLEMS+=("the copy's sendReceiveProjectIdentifier is '${P[copy_sr]}', expected '${P[lexbox_code]}'") + check_count "$code" entries "${LIST_ENTRIES[$code]}" + check_count "$code" comments "${LIST_COMMENTS[$code]}" + # Compared with the source rather than the list, which has no such column. After a Send/Receive + # this is the check that catches custom field data LfMerge skipped. + [[ ${P[copy_cf_entries]} == "${P[source_cf_entries]}" ]] || + PROBLEMS+=("entries with custom field data: the copy has ${P[copy_cf_entries]}, the source has ${P[source_cf_entries]}") +} + +# Checks for a migration that should be completely finished. +verify_finished() { + local code=$1 + PROBLEMS=() + probe_or_fail "$code" + check_copy "$code" + copy_synced || PROBLEMS+=("the copy has not completed a Send/Receive (its lastSyncedDate was never updated)") + [[ ${P[sr_class]} == "done" ]] || PROBLEMS+=("the copy's Send/Receive state is '${P[sr_state]}', not finished") + [[ ${P[assets_all_linked]} == 1 ]] || + PROBLEMS+=("the copy's assets are not linked into the Send/Receive work tree (audio: ${P[assets_audio]}, pictures: ${P[assets_pictures]})") + report_verification "$code" +} + +report_verification() { + local code=$1 + if ((${#PROBLEMS[@]})); then + fail "$code" "The migration ran, but the result does not check out:" "${PROBLEMS[@]/#/ - }" \ + "Investigate before going on. If the result is in fact correct, add this line to $PROGRESS_FILE" \ + "so that the next run skips it: $(progress_line "$code" "${VERIFIED_STAGE:-}")" + fi + say " verified: ${P[copy_entries]} entries, ${P[copy_comments]} comments, ${P[copy_cf_entries]} with custom field data" +} + +# --------------------------------------------------------------------------- # +# the stage that finishes a migration +# --------------------------------------------------------------------------- # + +# migrate_to_finish CODE KIND -- take a project from wherever it is to fully migrated and verified. +# KIND is "whole" for a project without custom fields, or "stage2" for a custom field project whose +# stage 1 is done and whose custom fields have been committed. Records the stage when done. +migrate_to_finish() { + local code=$1 kind=$2 stage="" log reason + [[ $kind == stage2 ]] && stage=stage2 + VERIFIED_STAGE=$stage + + probe_or_fail "$code" lexbox + check_source "$code" + reason=$(busy_reason) + [[ -z $reason ]] || fail "$code" "Not touching it: $reason." "Wait for it to finish, then run this script again." + + local action=migrate flags=() + if ! copy_synced; then + if ((${P[assets_links]} > 0)); then + fail "$code" "The copy's assets are already linked into the Send/Receive work tree, but no Send/Receive" \ + "has completed. That should not happen; inspect ${P[new_code]} by hand." + fi + if [[ $kind == stage2 ]]; then + [[ ${P[copy_exists]} == 1 ]] || fail "$code" "Stage 1 is recorded as done, but the copy ${P[new_code]} does not exist." + [[ ${P[lexbox_exists]} == 1 ]] || fail "$code" "Stage 1 is recorded as done, but LexBox has no project '${P[lexbox_code]}'." + flags=(--resume) + else + set_restart_flags "$code" + flags=("${RESTART_FLAGS[@]}") + fi + elif [[ ${P[sr_class]} == failed ]]; then + fail "$code" "The copy's last Send/Receive failed (SRState=${P[sr_state]}); see \`docker compose logs lfmerge\`." \ + "Once it is fixed, run this script again; it will finish the remaining asset steps." + elif [[ ${P[assets_all_linked]} == 1 && ${P[sr_class]} == "done" ]] && + ((${P[sr_state_mtime]} > ${P[assets_link_mtime]})); then + # Everything ran, and a Send/Receive finished after the assets were linked: a previous run + # was cut off after the migration but before it was verified. + action=verify + else + # The first Send/Receive finished; the assets still need moving and pushing. Not --resume, + # which deletes the work tree the assets may already have been moved into. + action=finish + fi + + case $action in + migrate) + log=$(new_log "$code" "${stage:-migrate}") + if ! run_migration "$code" "$log" "${flags[@]}"; then + fail "$code" "The migration failed; see $log" + fi + if [[ $kind == whole ]] && grep -q '^Paused for custom fields' "$log"; then + fail "$code" "The migration stopped for custom fields, so this project belongs in the custom fields list." \ + "Its copy and LexBox project now exist; migrate_custom_field_projects.sh can carry it on." + fi + grep -q '^Done\. ' "$log" || fail "$code" "The migration exited without saying it was done; see $log" + ;; + finish) + log=$(new_log "$code" finish) + run_finish "$code" "$log" || fail "$code" "Finishing the asset steps failed; see $log" + ;; + verify) + say " the migration had already finished; verifying it" + ;; + esac + + verify_finished "$code" + progress_add "$code" "$stage" + ((MIGRATED += 1)) +} + +# The source has to be the project the list means, and must still be eligible. +check_source() { + local code=$1 + [[ ${P[source_exists]} == 1 ]] || fail "$code" "There is no project '$code' in Mongo." + [[ ${P[source_id]} == "${LIST_ID[$code]}" ]] || + fail "$code" "Project '$code' has _id ${P[source_id]}, but the list says ${LIST_ID[$code]}. Is the list for this database?" + [[ -z ${P[source_sr]} ]] || fail "$code" "Project '$code' is already a Send/Receive project ('${P[source_sr]}')." +} + +# --------------------------------------------------------------------------- # +# preflight +# --------------------------------------------------------------------------- # + +# preflight CODE... -- check every project before any is migrated, and stop if any would fail. +# A project is checked against what it should be for this script; see preflight_classification +# in the calling script. Projects nobody has started on also get a --dry-run of the migration, +# which catches what the migration itself would refuse. +preflight() { + heading "Checking ${#} project(s) before migrating any" + local code problems=() before log + for code in "$@"; do + before=${#problems[@]} + # A failure here is nearly always the same for every project (container down, LexBox login + # refused), so stop at the first rather than repeat it for each. + probe "$code" lexbox,custom-fields || + die "could not inspect $code (the error is above). Is the app container running, and can it reach LexBox?" + [[ ${P[source_exists]} == 1 ]] || { + problems+=("$code: no such project in Mongo") + continue + } + [[ ${P[source_id]} == "${LIST_ID[$code]}" ]] || + problems+=("$code: _id is ${P[source_id]}, but the list says ${LIST_ID[$code]}") + [[ -z ${P[source_sr]} ]] || problems+=("$code: already a Send/Receive project ('${P[source_sr]}')") + ((${P[custom_field_case_problems]} == 0)) || + problems+=("$code: has custom field names differing only by case that the migration does not know how to rename") + ((${P[custom_fields_dropped_with_data]} == 0)) || + problems+=("$code: has ${P[custom_fields_dropped_with_data]} custom field(s) holding data that the migration would leave out (type undetermined)") + + local classification + classification=$(preflight_classification "$code") + [[ -z $classification ]] || problems+=("$code: $classification") + + if lexbox_project_in_the_way "$code"; then + problems+=("$code: LexBox already has a project '${P[lexbox_code]}', and nothing in $LOG_DIR shows this script creating it (a leftover test?)") + fi + + if ! leftovers_exist && ((${#problems[@]} == before)); then + log=$(new_log "$code" dry-run) + if ! app_python "$MIGRATION_SCRIPT" "$code" --dry-run "${FORWARD[@]}" "$log" 2>&1; then + problems+=("$code: the migration's --dry-run failed; see $log") + fi + fi + + if ((${#problems[@]} == before)); then + say " ok $code (${P[source_entries]} entries, ${P[source_comments]} comments, ${P[custom_field_definitions]} custom field definition(s))" + else + say " FAIL $code" + fi + done + + if ((${#problems[@]})); then + printf '\n%s: nothing was migrated, because of these problems:\n' "$SCRIPT_NAME" >&2 + printf ' - %s\n' "${problems[@]}" >&2 + exit 1 + fi +} + +print_summary() { + heading "Summary" + say "Migrated and verified in this run: $MIGRATED" + if ((${#WARNINGS[@]})); then + say "Warnings:" + printf ' - %s\n' "${WARNINGS[@]}" + fi +} diff --git a/scripts/migration_batch_helper.py b/scripts/migration_batch_helper.py new file mode 100644 index 0000000000..b5a17dbbb1 --- /dev/null +++ b/scripts/migration_batch_helper.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Inspect, and finish, migrations for the batch scripts in this directory. + + migration_batch_helper.py probe CODE [--with lexbox,custom-fields] [-- MIGRATION-OPTIONS] + migration_batch_helper.py finish CODE [-- MIGRATION-OPTIONS] + +Used by migrate_custom_field_projects.sh and migrate_non_custom_field_projects.sh, which run on the +host. This runs where copy_project_for_migration.py runs -- inside the app container -- and imports +it, so the Mongo connection, the file paths, the LexBox client and the custom field logic are all +the migration script's own rather than copies of them. MIGRATION-OPTIONS are the options the batch +script passes through to the migration script (--lexbox-host, --suffix, ...), parsed by its own +parse_args so that both see the same values. + +probe changes nothing. It prints key=value lines describing the source project and its copy (as +counted the same way as the projectsWorthMigrating lists: entries and comments not marked as +deleted), the copy's Send/Receive state and assets, and whether a migration of this project is +running right now. --with adds the slower checks: "lexbox" logs in to LexBox and asks whether the +copy's project code is taken, and "custom-fields" works out how many custom field definitions the +migration would write, by calling the same functions the migration does. + +finish does the last two steps of a migration whose first Send/Receive has completed: move the +staged assets into the Send/Receive work tree, and run the second Send/Receive. Both are the +migration script's own functions, and both are safe to repeat: link_assets_into_webwork skips what +is already linked. It exists because `copy_project_for_migration.py --resume` clears the work tree +before it starts, which after the assets have been moved there would delete them. +""" + +from __future__ import annotations + +import json +import os +import sys +from urllib.parse import quote + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import copy_project_for_migration as m # noqa: E402 + +EXTRAS = frozenset({"lexbox", "custom-fields"}) + +# Counts both projects the way the projectsWorthMigrating lists were counted, so the numbers can be +# compared directly. isEmpty is the one scan_custom_field_usage() uses, so "has custom field data" +# means the same thing here as it does to the migration. +PROBE_JS = r""" +function isEmpty(value) { + if (value === null || value === undefined) return true; + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "string") return value === ""; + if (typeof value === "object") { + if (Object.prototype.hasOwnProperty.call(value, "values")) return (value.values || []).length === 0; + if (Object.prototype.hasOwnProperty.call(value, "value")) return value.value === null || value.value === ""; + return Object.keys(value).every(function (k) { return isEmpty(value[k]); }); + } + return false; +} + +function holderHasCustomData(holder) { + const fields = (holder || {}).customFields || {}; + return Object.keys(fields).some(function (name) { return !isEmpty(fields[name]); }); +} + +function entryHasCustomData(entry) { + if (holderHasCustomData(entry)) return true; + return (entry.senses || []).some(function (sense) { + if (!sense || typeof sense !== "object") return false; + if (holderHasCustomData(sense)) return true; + return (sense.examples || []).some(function (example) { + return example && typeof example === "object" && holderHasCustomData(example); + }); + }); +} + +const mainDb = db.getSiblingDB(mainDbName); +const result = {}; +Object.keys(sides).forEach(function (side) { + const code = sides[side][0]; + const dbName = sides[side][1]; + const r = { exists: 0, id: "", sr: "", app: "", last_synced: 0, + collections: 0, entries: 0, comments: 0, cf_entries: 0 }; + + const doc = mainDb.projects.findOne({ projectCode: code }); + if (doc !== null) { + r.exists = 1; + r.id = doc._id.toString(); + r.sr = doc.sendReceiveProjectIdentifier || ""; + r.app = doc.appName || ""; + const synced = doc.lastSyncedDate; + if (synced instanceof Date && !isNaN(synced.getTime())) r.last_synced = synced.getTime(); + } + + const target = db.getSiblingDB(dbName); + const names = target.getCollectionNames(); + r.collections = names.length; + if (names.indexOf("lexicon") !== -1) { + target.lexicon.find({ isDeleted: { $ne: true } }, { customFields: 1, senses: 1 }).forEach(function (entry) { + r.entries += 1; + if (entryHasCustomData(entry)) r.cf_entries += 1; + }); + } + if (names.indexOf("lexiconComments") !== -1) { + r.comments = target.lexiconComments.countDocuments({ isDeleted: { $ne: true } }); + } + result[side] = r; +}); +print(EJSON.stringify(result)); +""" + + +def parse_cli(argv: list[str]) -> tuple[str, str, set[str], object]: + if len(argv) < 2 or argv[0] not in ("probe", "finish"): + raise SystemExit(__doc__) + command, code, rest = argv[0], argv[1], argv[2:] + extras: set[str] = set() + if rest[:1] == ["--with"]: + if len(rest) < 2: + raise SystemExit("--with needs a comma-separated list: " + ", ".join(sorted(EXTRAS))) + extras = {item for item in rest[1].split(",") if item} + unknown = extras - EXTRAS + if unknown: + raise SystemExit(f"--with: unknown check(s) {', '.join(sorted(unknown))}") + rest = rest[2:] + if rest[:1] == ["--"]: + rest = rest[1:] + return command, code, extras, m.parse_args([code, *rest]) + + +def mongo_state(config: m.Config, code: str, new_code: str) -> dict: + sides = { + "source": [code, m.database_name(code)], + "copy": [new_code, m.database_name(new_code)], + } + js = ( + f"const mainDbName = {json.dumps(config.database)};\n" + f"const sides = {json.dumps(sides)};\n" + + PROBE_JS + ) + return m.mongosh_json(config, js, what=f"Inspecting {code} and {new_code}") + + +def send_receive_state(config: m.Config, new_code: str) -> dict: + """The copy's LfMerge state file, classified the way wait_for_send_receive() classifies it.""" + out = { + "sr_state": "", + "sr_class": "none", + "sr_state_mtime": 0, + "queue_present": int(os.path.exists(config.queue_file(new_code))), + } + state, mtime = m.read_sr_state(config, new_code) + if mtime is None: + return out + out["sr_state_mtime"] = int(mtime) + sr_state = (state or {}).get("SRState") or "" + out["sr_state"] = sr_state + if sr_state in m.SR_IN_PROGRESS: + out["sr_class"] = "in_progress" + elif sr_state in m.SR_DONE: + out["sr_class"] = "done" + elif sr_state in m.SR_FAILED: + out["sr_class"] = "failed" + else: + # Includes a state file LfMerge is part-way through writing, which reads as no state. + out["sr_class"] = "unknown" + return out + + +def asset_state(config: m.Config, app: str, new_code: str) -> dict: + """Whether link_assets_into_webwork() has run: each asset folder a symlink into the work tree.""" + assets_dir = config.assets_path(app, m.database_name(new_code)) + out: dict[str, object] = {} + links = correct = newest = 0 + for subfolder, linked_subfolder in m.ASSET_LINKED_FILES: + path = os.path.join(assets_dir, subfolder) + if os.path.islink(path): + kind = "link" + links += 1 + newest = max(newest, int(os.lstat(path).st_mtime)) + expected = os.path.join(config.linked_files_path(new_code), linked_subfolder) + if os.path.realpath(path) == os.path.realpath(expected): + correct += 1 + elif os.path.isdir(path): + kind = "dir" + elif os.path.exists(path): + kind = "other" + else: + kind = "missing" + out[f"assets_{subfolder}"] = kind + out["assets_links"] = links + out["assets_all_linked"] = int(correct == len(m.ASSET_LINKED_FILES)) + out["assets_link_mtime"] = newest + return out + + +def migration_running(code: str) -> bool: + """True if this container is running a migration, or a finish, of this project right now. + + The batch scripts start these through `docker compose exec -T`, and interrupting the client does + not stop the process inside the container, so a rerun has to look for one before it starts. + """ + me = os.getpid() + helper = os.path.basename(__file__) + for entry in os.listdir("/proc"): + if not entry.isdigit() or int(entry) == me: + continue + try: + with open(f"/proc/{entry}/cmdline", "rb") as handle: + argv = [arg.decode("utf-8", "replace") for arg in handle.read().split(b"\0") if arg] + except OSError: + continue + if code not in argv: + continue + names = {os.path.basename(arg) for arg in argv} + if "copy_project_for_migration.py" in names: + return True + if helper in names and "finish" in argv: + return True + return False + + +def custom_field_counts(config: m.Config, code: str) -> dict: + """What the migration would decide about custom fields, from the same functions it uses. + + The writing system lists only steer the wsSelector inside a definition, never whether there is + one, so empty lists give the same count as the migration's own. + """ + rows = m.describe_custom_fields( + m.fetch_custom_fields(config, code), + m.scan_custom_field_usage(config, m.database_name(code)), + ) + renames, problems = m.plan_custom_field_renames(rows) + m.apply_custom_field_renames(rows, renames) + definitions, dropped = m.plan_custom_field_definitions(rows, [], []) + return { + "custom_field_definitions": len(definitions), + "custom_field_case_problems": len(problems), + # Left out of the definitions but holding data: LfMerge would skip that data, and the + # migration does not stop for it. + "custom_fields_dropped_with_data": sum(1 for row, _reason in dropped if row["entries"] > 0), + } + + +def lexbox_has_project(args, lexbox_code: str) -> bool: + """Ask LexBox whether the project code is taken (GET /api/project/projectCodeAvailable/{code}).""" + password = ( + args.lexbox_password + if args.lexbox_password is not None + else os.environ.get("LEXBOX_PASSWORD", m.DEFAULT_LEXBOX_PASSWORD) + ) + client = m.LexboxClient( + args.lexbox_host, + args.lexbox_user, + password, + auth_mode=args.lexbox_auth, + timeout=args.lexbox_timeout, + ) + client.login() + # The client has no public method for this endpoint; _send is the same transport its own + # calls use, and handles authentication and errors the same way. + status, _headers, body = client._send( + "GET", "/api/project/projectCodeAvailable/" + quote(lexbox_code, safe=""), authenticate=True + ) + if status != 200: + raise m.Fatal(f"asking LexBox whether {lexbox_code!r} exists failed: {client._describe(status, body)}") + available = json.loads(body.decode("utf-8")) + if not isinstance(available, bool): + raise m.Fatal(f"projectCodeAvailable returned an unexpected body: {body[:200]!r}") + return not available + + +def probe(config: m.Config, args, code: str, extras: set[str]) -> dict: + new_code = code + args.suffix + lexbox_code = m.send_receive_identifier(new_code) + scheme, host = m.split_lexbox_host(args.lexbox_host) + out: dict[str, object] = { + "new_code": new_code, + "lexbox_code": lexbox_code, + # The same URL emit_custom_properties() tells a human to clone. + "hg_url": f"{scheme}://{host}/hg/{lexbox_code}", + } + found = mongo_state(config, code, new_code) + for side in ("source", "copy"): + for key, value in found[side].items(): + out[f"{side}_{key}"] = value + app = found["copy"]["app"] or found["source"]["app"] or "lexicon" + out.update(send_receive_state(config, new_code)) + out.update(asset_state(config, app, new_code)) + out["migration_running"] = int(migration_running(code)) + if "custom-fields" in extras: + out.update(custom_field_counts(config, code)) + if "lexbox" in extras: + out["lexbox_exists"] = int(lexbox_has_project(args, lexbox_code)) + return out + + +def finish(config: m.Config, args, code: str) -> None: + new_code = code + args.suffix + copy = m.fetch_project(config, new_code) + if copy is None: + raise m.Fatal(f"there is no project {new_code!r} to finish") + app = copy["appName"] or "lexicon" + m.log("Moving assets into the Send/Receive work tree") + m.link_assets_into_webwork(config, app, m.database_name(new_code), new_code) + m.log("Send/Receive #2: pushing assets to LexBox") + m.send_receive(config, new_code, timeout=args.sr_timeout, label="2/2") + m.log("") + m.log(f"Done. '{new_code}' has finished its asset steps.") + + +def main(argv: list[str]) -> int: + command, code, extras, args = parse_cli(argv) + config = m.Config() + try: + if command == "probe": + for key, value in probe(config, args, code, extras).items(): + # One line per key: collapse any whitespace a value might carry. + print(f"{key}={' '.join(str(value).split())}") + else: + finish(config, args, code) + except m.Fatal as exc: + m.log(f"ERROR: {exc}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))