From 6e8322234ff283c9a01bfc5f621b2efe5a6ed39d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 13 Aug 2026 22:09:56 +0700 Subject: [PATCH 01/51] Add failing tests for non-canonical writing system tags LF stores a writing system tag as it was typed, but LCM canonicalizes a tag when it creates the writing system and thereafter knows it only by that canonical id. TryGet and GetWsFromStr both match the id literally (they are case-insensitive, but not structure-aware), so a tag whose canonical form differs structurally is never found: qaa-x-qaa-v -> qaa-x-v (redundant private-use "qaa" absorbed) th-Thai -> th (suppress-script dropped) LfWsToLcmWs then takes its else branch and calls Create() -- which DOES canonicalize -- so Set() collides with the writing system already there: Unable to set writing system 'qaa-x-kal' because this id already exists. The second test covers the quieter half: LfMultiText resolves each multitext key with GetWsFromStr and skips anything returning 0, so a value keyed by a non-canonical tag is dropped, and the clearing pass then blanks whatever LCM had for that field. testlangproj already contains qaa-x-kal and qaa-Zxxx-x-kal-audio, so these use a non-canonical spelling of an existing writing system rather than adding to the shared fixture project. The qaa-Zxxx-x-kal-AUDIO case already passes (case-only difference) and is a guard against regressing it. Co-Authored-By: Claude Opus 5 (1M context) --- .../ConvertMongoToLcmWritingSystemsTests.cs | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs new file mode 100644 index 00000000..b614d4b9 --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LfMerge.Core.MongoConnector; +using LfMergeBridge.LfMergeModel; +using MongoDB.Bson; +using NUnit.Framework; +using SIL.LCModel; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// Language Forge stores a writing system tag exactly as it was typed, but LCM canonicalizes a + /// tag when it creates a writing system and thereafter knows that writing system only by the + /// canonical id. Every LCM lookup -- WritingSystemManager.TryGet, GetWsFromStr -- matches that id + /// literally, so a tag that is not already canonical is never found. + /// + /// testlangproj already contains "qaa-x-kal" and "qaa-Zxxx-x-kal-audio", so a non-canonical LF + /// spelling of either exercises the mismatch without adding anything to the shared fixture project. + /// + public class ConvertMongoToLcmWritingSystemsTests : LcmTestBase + { + // The apltw2016 shape: the private-use section repeats the "qaa" language subtag, which + // canonicalizing absorbs (apltw2016 itself has "qaa-x-qaa-v", which becomes "qaa-x-v"). + private const string RedundantPrivateUseTag = "qaa-x-qaa-kal"; + private const string RedundantPrivateUseId = "qaa-x-kal"; + + // The shape found in the testlangproj-derived projects, where canonicalizing only changes case. + // This one ALREADY works, because both LCM lookups (TryGet and GetWsFromStr) are + // case-insensitive; it is here as a guard so that a fix for the structural case does not + // regress it. Only differences in structure -- a subtag absorbed or dropped -- go unfound. + private const string MixedCaseTag = "qaa-Zxxx-x-kal-AUDIO"; + private const string MixedCaseId = "qaa-Zxxx-x-kal-audio"; + + /// + /// Adds one input system to the memoized mock project record, and returns an action that puts + /// the record back as it was. The double memoizes per project code, so without the restore the + /// addition would leak into every later test in this fixture. + /// + private Action AddLfInputSystem(string tag) + { + MongoProjectRecord record = _recordFactory.Create(_lfProj); + Assert.That(record, Is.Not.Null, "the mock project record factory should supply a record"); + Assert.That(record.InputSystems.ContainsKey(tag), Is.False, + "the fixture's input systems should not already contain {0}", tag); + + record.InputSystems[tag] = new LfInputSystemRecord { + Abbreviation = tag, + Tag = tag, + LanguageName = "Unlisted Language", + IsRightToLeft = false + }; + return () => record.InputSystems.Remove(tag); + } + + private IEnumerable LcmWritingSystemIds() + { + return _cache.ServiceLocator.WritingSystemManager.WritingSystems.Select(ws => ws.Id); + } + + [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)] + [TestCase(MixedCaseTag, MixedCaseId)] + public void LfWsToLcmWs_NonCanonicalTag_ReusesTheCanonicalWritingSystem(string lfTag, string canonicalId) + { + // Setup: LF holds the writing system under a non-canonical tag, while the LCM project + // already holds it under the canonical id that LCM derives from that same tag. + Assert.That(LcmWritingSystemIds(), Contains.Item(canonicalId), + "testlangproj should already contain {0}", canonicalId); + int countBefore = LcmWritingSystemIds().Count(id => id == canonicalId); + Action restore = AddLfInputSystem(lfTag); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify: the existing writing system was reused, not duplicated, and the raw LF tag + // did not become an id of its own. + Assert.That(LcmWritingSystemIds().Count(id => id == canonicalId), Is.EqualTo(countBefore), + "{0} should still appear exactly once", canonicalId); + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(lfTag), + "the non-canonical tag {0} should not be stored as an id of its own", lfTag); + } + finally + { + restore(); + } + } + + [Test] + public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWritingSystem() + { + // Setup: a definition keyed by the non-canonical tag. LfMultiText resolves each key with + // GetWsFromStr and skips anything that returns 0, so an unresolved key silently drops the + // value -- and the clearing pass then blanks whatever LCM had for that field. + const string newDefinition = "Definition stored under a non-canonical writing system tag"; + var data = new SampleData(); + data.bsonTestData["senses"][0]["definition"] = new BsonDocument { + { RedundantPrivateUseTag, new BsonDocument { { "value", newDefinition } } } + }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + int wsId = _cache.WritingSystemFactory.GetWsFromStr(RedundantPrivateUseId); + Assert.That(wsId, Is.Not.EqualTo(0), "{0} should resolve in testlangproj", RedundantPrivateUseId); + + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + Guid expectedGuid = Guid.Parse(data.bsonTestData["guid"].AsString); + var entry = _cache.ServiceLocator.GetObject(expectedGuid) as ILexEntry; + Assert.That(entry, Is.Not.Null); + Assert.That(entry.SensesOS[0].Definition.get_String(wsId).Text, Is.EqualTo(newDefinition), + "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId); + } + } +} From dee46fd6561452191f44b8e9534f8c2bee521f46 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 14 Aug 2026 11:31:27 +0700 Subject: [PATCH 02/51] Resolve lexicon multitext keys by canonical language tag LF stores a writing system tag exactly as it was typed; LCM canonicalizes it when the writing system is created and thereafter knows it only by that canonical id. GetWsFromStr matches that id literally -- case-insensitively, but not structure-aware -- so a multitext keyed by a tag whose canonical form differs structurally resolves to 0: qaa-x-qaa-v -> qaa-x-v (redundant private-use "qaa" absorbed) th-Thai -> th (suppress-script dropped) WriteToLcmMultiString skips a key that resolves to 0, and its clearing pass then blanks whatever LCM held for that writing system, so the value is not merely dropped on the way in, it is deleted on the way out. Canonicalize the key before the lookup, here and in WsIdAndFirstNonEmptyString. LanguageTags.Canonical guards IetfLanguageTag.Canonicalize with IsValid, which it needs: Canonicalize throws on a tag it cannot parse, and a corpus scan found 48 such strings (custom field names that look like multitext keys). Verified against SIL.WritingSystems: Canonicalize agrees with the id liblcm itself stores for all 1251 valid tags in that corpus, and is idempotent on all of them. Co-Authored-By: Claude Opus 5 (1M context) --- .../LanguageForge/Model/LfMultiText.cs | 8 +++- src/LfMerge.Core/LanguageTags.cs | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 src/LfMerge.Core/LanguageTags.cs diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 96dd29dd..74000edf 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -117,7 +117,7 @@ public KeyValuePair WsIdAndFirstNonEmptyString(LcmCache cache) KeyValuePair kv = FirstNonEmptyKeyValue(); if (kv.Key == null) return new KeyValuePair(); ILgWritingSystemFactory wsManager = cache.ServiceLocator.WritingSystemManager; - int wsId = wsManager.GetWsFromStr(kv.Key); + int wsId = wsManager.GetWsFromStr(LanguageTags.Canonical(kv.Key)); return new KeyValuePair(wsId, kv.Value); } @@ -136,7 +136,11 @@ public void WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFacto HashSet destWsIdsToClear = new HashSet(dest.AvailableWritingSystemIds); foreach (KeyValuePair kv in this) { - int wsId = wsManager.GetWsFromStr(kv.Key); + // By the canonical tag: LF may hold a non-canonical spelling of the id LCM knows the + // writing system by, and GetWsFromStr matches that id literally. Without this the value + // is skipped as unidentified AND the clearing pass below blanks whatever LCM had for + // that writing system, so the data is not merely dropped but deleted. + int wsId = wsManager.GetWsFromStr(LanguageTags.Canonical(kv.Key)); if (wsId == 0) continue; // Skip any unidentified writing systems string value = kv.Value.Value; ITsString tss = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(value, wsId, wsManager); diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs new file mode 100644 index 00000000..187fe7ec --- /dev/null +++ b/src/LfMerge.Core/LanguageTags.cs @@ -0,0 +1,38 @@ +using SIL.WritingSystems; + +namespace LfMerge.Core +{ + public static class LanguageTags + { + /// + /// The canonical form of a language tag: the id LCM knows a writing system by, once it has + /// created one from this tag. + /// + /// Language Forge stores a tag exactly as it was typed. LCM canonicalizes it when the writing + /// system is created -- WritingSystemManager.GetOrSet and Create both do -- and thereafter + /// GetWsFromStr matches that canonical id literally. It is case-insensitive, so a difference + /// of case resolves on its own, but it is not structure-aware, so a tag that loses or absorbs + /// a subtag is never found: + /// + /// qaa-x-qaa-v -> qaa-x-v the private-use section repeats the "qaa" language subtag, + /// and canonicalizing absorbs it + /// th-Thai -> th Thai's suppress-script is dropped + /// + /// LfWsToLcmWs does not need this -- GetOrSet canonicalizes for it -- but the lexicon does: + /// its multitext keys are LF tags being looked up against LCM's canonical ids. + /// + /// + /// The canonical tag, or the tag unchanged when it is empty or cannot be parsed -- + /// IetfLanguageTag.Canonicalize throws on those, and an unparseable tag should go on failing + /// where it failed before rather than here. + /// + public static string Canonical(string tag) + { + if (string.IsNullOrEmpty(tag) || !IetfLanguageTag.IsValid(tag)) + { + return tag; + } + return IetfLanguageTag.Canonicalize(tag); + } + } +} From 27039443b999d69b377ae65662c30a2c89416342 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 14 Aug 2026 11:32:07 +0700 Subject: [PATCH 03/51] Use GetOrSet instead of TryGet in LfWsToLcmWs TryGet matched the writing system id literally while Create canonicalized it, so a tag whose canonical form differs -- "qaa-x-qaa-v" becomes "qaa-x-v", because the private-use section repeats the "qaa" language subtag -- was not found, was created under its canonical id, and collided with the writing system already there: Unable to set writing system 'qaa-x-v' because this id already exists. GetOrSet does the lookup and the creation in one step and canonicalizes for both, so the two halves can no longer disagree. It returns true when it found an existing writing system and false when it created one, and a created one is already in the manager, so the separate Set call goes away and the two branches collapse into one path plus a conditional vernacular/analysis assignment. This implements only half of the TODO it replaces. The other half -- "use the ILgWritingSystemFactory interface and remove one point of FW 8-to-9 API incompatibility" -- does not follow: GetOrSet is declared on WritingSystemManager, not on ILgWritingSystemFactory, which offers only get_Engine, GetWsFromStr, GetStrFromWs, GetIcuLocaleFromWs and GetWritingSystems. The #if FW8_COMPAT blocks therefore stay, and the comment now records why rather than leaving a TODO that cannot be done as written. The vernacular comparison keeps comparing the tag as LF spells it against languageCode as LF spells it. Both are LF's own strings, so a project whose languageCode is itself non-canonical still matches its own input system. Braces added to the single-statement Abbreviation guard, which reads as indentation-scoped without them. Co-Authored-By: Claude Opus 5 (1M context) --- .../ConvertMongoToLcmLexicon.cs | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 70d172fb..b1850028 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -253,36 +253,46 @@ private void LfWsToLcmWs(Dictionary lfWsList) } */ - // TODO: It might be possible to rewrite this code to NOT rely on TryGet() after all, in which case we could - // use the ILgWritingSystemFactory interface and remove one point of FW 8-to-9 API incompatibility. - - if (wsManager.TryGet(lfWs.Tag, out ws)) - { - // The WS does check that a property has a different value before setting it - // (and thus setting IsChanged flag), but for Abbreviation the WS returns - // Language if not set, and it fails to check that. - if (ws.Abbreviation != lfWs.Abbreviation) - ws.Abbreviation = lfWs.Abbreviation; - ws.RightToLeftScript = lfWs.IsRightToLeft; - wsManager.Replace(ws); - } - else + // GetOrSet() does the lookup and the creation in one step, and does both by the + // canonical form of the tag. TryGet() matched the id literally while Create() + // canonicalized, so a tag LCM canonicalizes -- "qaa-x-qaa-v" becomes "qaa-x-v", + // because the private-use section repeats the "qaa" language subtag, and "th-Thai" + // becomes "th", because Thai's suppress-script is dropped -- was not found, was then + // created under its canonical id, and collided with the writing system already there: + // Unable to set writing system 'qaa-x-v' because this id already exists. + // GetOrSet returns true when it found an existing writing system and false when it + // created one (creating adds it to the manager, so no Set() call is needed here). + // + // The rest of the TODO this replaces is not achievable this way: GetOrSet lives on + // WritingSystemManager, not on ILgWritingSystemFactory, so the FW 8-to-9 compatibility + // blocks above and below have to stay. + bool wsAlreadyExisted = wsManager.GetOrSet(lfWs.Tag, out ws); + + // The WS does check that a property has a different value before setting it + // (and thus setting IsChanged flag), but for Abbreviation the WS returns + // Language if not set, and it fails to check that. + if (ws.Abbreviation != lfWs.Abbreviation) { - ws = wsManager.Create(lfWs.Tag); ws.Abbreviation = lfWs.Abbreviation; - ws.RightToLeftScript = lfWs.IsRightToLeft; - wsManager.Set(ws); + } + ws.RightToLeftScript = lfWs.IsRightToLeft; + wsManager.Replace(ws); + if (!wsAlreadyExisted) + { // LF doesn't distinguish between vernacular/analysis WS, so we'll // only assign the project language code to vernacular. // All other WS assigned to analysis. + // Compared as LF spells them, both sides being LF's own strings: a project whose + // languageCode is itself non-canonical (e.g. "th-Thai") still matches its own + // input system, which is spelled the same way. + // TODO: What if our vernacular was Thai, but we added th-ipa? This logic needs to be a bit "fuzzier", really. if (lfWs.Tag.Equals(vernacularLanguageCode)) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); else ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); - } } } From 2088096b245390be840f70d5ac1107d3ef3f6b06 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 14 Aug 2026 14:46:26 +0700 Subject: [PATCH 04/51] Identify vernacular writing systems from the vernacular config fields LF has no vernacular/analysis flag on a writing system; the distinction is conventional, encoded in which config fields use it. LfWsToLcmWs treated exactly one writing system as vernacular -- whichever equalled the project's languageCode -- and filed every other one as analysis. A project whose lexeme field uses more than one writing system therefore lost the distinction: apltw2016's lexeme field uses qaa-x-IPA-dupl1 alongside th, and the phonetic one was made analysis. Take the vernacular writing systems from the fields that are vernacular by convention instead: citationForm, lexeme, etymology, senses.fields.examples.fields.sentence That list was already in MongoConnection, which applies the same convention in the other direction when it writes input systems back out to the config. Rather than have a second copy to keep in step by hand, it moves to MagicStrings.LfVernacularConfigFieldPaths and both directions read it from there. MongoConnection keeps its field name, so its six use sites are unchanged; the type widens to IReadOnlyList so the shared instance is not handed out as mutable. The example sentence path is nested, so the paths are walked a segment at a time. They are spelled as Mongo sees them, where each level of nesting goes through a "fields" document; in the mapped classes that is the Fields dictionary being indexed, so those segments are skipped. Where the config names no vernacular input systems at all, the project's language code is still used, so a project with a config LfMerge cannot read behaves as it did before. Tags are compared case-insensitively, matching LCM's own writing system lookups. This replaces the "needs to be a bit fuzzier" TODO in that method. Note the assignment still only happens for a writing system LfMerge has just created; one that already exists in the FieldWorks project keeps whatever vernacular/analysis membership it has there. That is unchanged behaviour, and it is why the new tests cover the derivation directly: creating a writing system in the shared test project cannot be undone afterwards, since WritingSystemManager has no removal API. Co-Authored-By: Claude Opus 5 (1M context) --- .../VernacularWritingSystemTagsTests.cs | 179 ++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 71 +++++-- src/LfMerge.Core/MagicStrings.cs | 12 ++ .../MongoConnector/MongoConnection.cs | 8 +- 4 files changed, 255 insertions(+), 15 deletions(-) create mode 100644 src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs new file mode 100644 index 00000000..c2197b3e --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs @@ -0,0 +1,179 @@ +using System.Collections.Generic; +using LfMerge.Core.DataConverters; +using LfMerge.Core.LanguageForge.Config; +using NUnit.Framework; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// LF has no vernacular/analysis flag on a writing system: which ones are vernacular is implied + /// by the config fields that use them. These cover that derivation on its own, without an LCM + /// project, since it is a pure function of the config and the project's language code. + /// + public class VernacularWritingSystemTagsTests + { + private static LfProjectConfig ConfigWith(params (string FieldName, string[] InputSystems)[] fields) + { + var entry = new LfConfigFieldList(); + foreach ((string fieldName, string[] inputSystems) in fields) + { + entry.Fields[fieldName] = new LfConfigMultiText { + InputSystems = new List(inputSystems) + }; + } + return new LfProjectConfig { Entry = entry }; + } + + [Test] + public void TakesTheLexemeFieldsInputSystems() + { + var config = ConfigWith(("lexeme", new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); + + Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); + } + + [Test] + public void UnionsLexemeAndCitationFormWithoutDuplicating() + { + var config = ConfigWith( + ("lexeme", new[] { "qaa-x-kal", "qaa-fonipa-x-kal" }), + ("citationForm", new[] { "qaa-x-kal", "seh" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); + + Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "seh" })); + } + + [Test] + public void TakesTheEtymologyFieldsInputSystems() + { + var config = ConfigWith( + ("lexeme", new[] { "qaa-x-kal" }), + ("etymology", new[] { "qaa-fonipa-x-kal" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); + + Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); + } + + /// + /// The example sentence lives at senses.fields.examples.fields.sentence, so this covers the + /// nested path as well as the field itself. + /// + [Test] + public void TakesTheExampleSentenceFieldsInputSystems() + { + var entry = new LfConfigFieldList(); + entry.Fields["lexeme"] = new LfConfigMultiText { InputSystems = new List { "qaa-x-kal" } }; + var examples = new LfConfigFieldList(); + examples.Fields["sentence"] = new LfConfigMultiText { InputSystems = new List { "seh" } }; + examples.Fields["translation"] = new LfConfigMultiText { InputSystems = new List { "en" } }; + var senses = new LfConfigFieldList(); + senses.Fields["examples"] = examples; + senses.Fields["definition"] = new LfConfigMultiText { InputSystems = new List { "fr" } }; + entry.Fields["senses"] = senses; + var config = new LfProjectConfig { Entry = entry }; + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); + + Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "seh" })); + Assert.That(tags, Does.Not.Contain("en"), "the example translation is not a vernacular field"); + Assert.That(tags, Does.Not.Contain("fr"), "the sense definition is not a vernacular field"); + } + + [Test] + public void ToleratesAPartiallyMissingNestedPath() + { + // senses exists but has no examples, so the nested lookup has to give up cleanly. + var entry = new LfConfigFieldList(); + entry.Fields["lexeme"] = new LfConfigMultiText { InputSystems = new List { "qaa-x-kal" } }; + entry.Fields["senses"] = new LfConfigFieldList(); + var config = new LfProjectConfig { Entry = entry }; + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); + + Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal" })); + } + + [Test] + public void IgnoresAnalysisFields() + { + var config = ConfigWith( + ("lexeme", new[] { "qaa-x-kal" }), + ("definition", new[] { "en", "fr" }), + ("gloss", new[] { "en" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); + + Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal" })); + Assert.That(tags, Does.Not.Contain("en")); + } + + /// + /// The case this change is for. apltw2016's lexeme field uses a phonetic writing system + /// alongside the project language, and the old rule -- vernacular is whatever equals + /// languageCode -- filed that phonetic one as analysis. + /// + [Test] + public void TreatsASecondLexemeWritingSystemAsVernacular() + { + var config = ConfigWith(("lexeme", new[] { "qaa-x-IPA-dupl1", "th" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "th"); + + Assert.That(tags, Contains.Item("th")); + Assert.That(tags, Contains.Item("qaa-x-IPA-dupl1"), + "a writing system the lexeme field uses is vernacular even though it is not the language code"); + } + + [Test] + public void MatchesTagsCaseInsensitively() + { + var config = ConfigWith(("lexeme", new[] { "qaa-x-IPA-dupl1" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "th"); + + Assert.That(tags.Contains("qaa-x-ipa-dupl1"), Is.True, + "LCM's own writing system lookups are case-insensitive, so this should be too"); + } + + [TestCase(TestName = "FallsBackToLanguageCodeWhenNoVernacularFieldHasInputSystems")] + public void FallsBackToLanguageCode() + { + // A config with only analysis fields: nothing says what the vernacular is, so the + // project's language code is the best available answer -- the rule this replaced. + var config = ConfigWith(("definition", new[] { "en" })); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "fr"); + + Assert.That(tags, Is.EquivalentTo(new[] { "fr" })); + } + + [Test] + public void FallsBackToLanguageCodeWhenLexemeListsNoInputSystems() + { + var config = ConfigWith(("lexeme", new string[0])); + + var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "fr"); + + Assert.That(tags, Is.EquivalentTo(new[] { "fr" })); + } + + [Test] + public void ToleratesAMissingConfig() + { + Assert.That(ConvertMongoToLcmLexicon.VernacularWritingSystemTags(null, "fr"), + Is.EquivalentTo(new[] { "fr" })); + Assert.That(ConvertMongoToLcmLexicon.VernacularWritingSystemTags(new LfProjectConfig(), "fr"), + Is.EquivalentTo(new[] { "fr" })); + } + + [Test] + public void ReturnsNothingWhenThereIsNoConfigAndNoLanguageCode() + { + Assert.That(ConvertMongoToLcmLexicon.VernacularWritingSystemTags(null, null), Is.Empty); + } + } +} diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index b1850028..37cbb836 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -6,6 +6,7 @@ using System.Text.RegularExpressions; using LfMerge.Core.DataConverters.CanonicalSources; using LfMerge.Core.FieldWorks; +using LfMerge.Core.LanguageForge.Config; using LfMerge.Core.LanguageForge.Model; using LfMergeBridge.LfMergeModel; using LfMerge.Core.Logging; @@ -201,6 +202,58 @@ private IEnumerable GetLexicon(ILfProject project) return Connection.GetRecords(project, MagicStrings.LfCollectionNameForLexicon); } + /// + /// Walks a dotted config field path from , or null if any step is + /// missing. The "fields" segments are the Mongo document that holds a field list's children; + /// in the mapped classes that is the Fields dictionary being indexed, so they are skipped. + /// + private static LfConfigFieldBase FindConfigField(LfConfigFieldList root, string path) + { + LfConfigFieldBase current = root; + foreach (string segment in path.Split('.')) + { + if (segment == "fields") + { + continue; + } + var fieldList = current as LfConfigFieldList; + if (fieldList?.Fields == null || !fieldList.Fields.TryGetValue(segment, out current)) + { + return null; + } + } + return current; + } + + /// + /// The writing system tags LF treats as vernacular: the ones used by the fields in + /// MagicStrings.LfVernacularConfigFieldPaths. A project may well have several -- a phonetic or + /// orthographic alternate alongside the main one -- which is why the project's language code + /// alone is not enough. + /// + /// + /// The tags, compared case-insensitively as LCM's own lookups are. Falls back to the + /// project's language code when the config names no vernacular input systems at all, which + /// is the rule this replaces. + /// + public static HashSet VernacularWritingSystemTags(LfProjectConfig config, string languageCode) + { + var tags = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (string path in MagicStrings.LfVernacularConfigFieldPaths) + { + if (FindConfigField(config?.Entry, path) is LfConfigMultiText multiText && + multiText.InputSystems != null) + { + tags.UnionWith(multiText.InputSystems); + } + } + if (tags.Count == 0 && !string.IsNullOrEmpty(languageCode)) + { + tags.Add(languageCode); + } + return tags; + } + /// /// Converts the list of LF input systems and adds them to Lcm writing systems /// @@ -231,7 +284,7 @@ private void LfWsToLcmWs(Dictionary lfWsList) return; } - string vernacularLanguageCode = ProjectRecord.LanguageCode; + HashSet vernacularTags = VernacularWritingSystemTags(ProjectRecord.Config, ProjectRecord.LanguageCode); // TODO: Split the inside of this foreach() out into its own function foreach (var lfWs in lfWsList.Values) { @@ -280,16 +333,12 @@ private void LfWsToLcmWs(Dictionary lfWsList) if (!wsAlreadyExisted) { - // LF doesn't distinguish between vernacular/analysis WS, so we'll - // only assign the project language code to vernacular. - // All other WS assigned to analysis. - - // Compared as LF spells them, both sides being LF's own strings: a project whose - // languageCode is itself non-canonical (e.g. "th-Thai") still matches its own - // input system, which is spelled the same way. - - // TODO: What if our vernacular was Thai, but we added th-ipa? This logic needs to be a bit "fuzzier", really. - if (lfWs.Tag.Equals(vernacularLanguageCode)) + // LF doesn't distinguish between vernacular/analysis WS, so take the vernacular + // ones from the fields that are vernacular by convention -- lexeme and citation + // form -- and treat everything else as analysis. Matching on the tag as LF spells + // it, both sides being LF's own strings, so a non-canonical spelling still matches + // itself. + if (vernacularTags.Contains(lfWs.Tag)) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); else ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); diff --git a/src/LfMerge.Core/MagicStrings.cs b/src/LfMerge.Core/MagicStrings.cs index eebf34f4..2a6e76e7 100644 --- a/src/LfMerge.Core/MagicStrings.cs +++ b/src/LfMerge.Core/MagicStrings.cs @@ -109,6 +109,18 @@ static MagicStrings() public const string LfFieldNameForDefinition = "definition"; public const string LfFieldNameForExampleSentence = "definition"; + // The LF config fields whose input systems are vernacular writing systems rather than + // analysis ones. LF has no vernacular/analysis flag on a writing system; the distinction is + // conventional, and this list is that convention. Both directions read it so that they + // agree: ConvertMongoToLcmLexicon, turning LF input systems into LCM writing systems, and + // MongoConnection, writing input systems back out to the config. + // Paths are relative to config.entry and are spelled as Mongo sees them, so each level of + // nesting goes through "fields". The custom example writing system is deliberately absent: + // it is a custom field, so its name varies from project to project. + public static IReadOnlyList LfVernacularConfigFieldPaths { get; } = new[] { + "citationForm", "lexeme", "etymology", "senses.fields.examples.fields.sentence" + }; + // Fake language codes used in storing custom GenDate and int fields in Mongo public const string LanguageCodeForGenDateFields = "qaa-Qaad"; public const string LanguageCodeForIntFields = "qaa-Zmth"; diff --git a/src/LfMerge.Core/MongoConnector/MongoConnection.cs b/src/LfMerge.Core/MongoConnector/MongoConnection.cs index e9c55da5..8246ea6f 100644 --- a/src/LfMerge.Core/MongoConnector/MongoConnection.cs +++ b/src/LfMerge.Core/MongoConnector/MongoConnection.cs @@ -41,10 +41,10 @@ public class MongoConnection : IMongoConnection public LfMergeSettings Settings { get { return _settings; } } // List of LF fields which will use vernacular or pronunciation input systems. Hierarchy is config.entry.fields... - // We intentionally aren't setting custom example WS here, since it's a custom field with a custom name - private readonly List _vernacularWsFieldsList = new List { - "citationForm", "lexeme", "etymology", "senses.fields.examples.fields.sentence" - }; + // The vernacular list lives in MagicStrings because ConvertMongoToLcmLexicon reads it too, to + // decide which writing systems become vernacular in LCM; sharing it keeps the two directions + // from drifting apart. (Its comment there covers why the custom example WS is not in it.) + private readonly IReadOnlyList _vernacularWsFieldsList = MagicStrings.LfVernacularConfigFieldPaths; // Pronunciation fields are special: they want the *first* vernacular WS that uses IPA (or is otherwise flagged // as being a pronunciation writing system). private readonly List _pronunciationWsFieldsList = new List { From ba1ac3d93de0f59aeb0cd01fd09316d051fb60cb Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 18 Aug 2026 08:55:51 +0700 Subject: [PATCH 05/51] Treat missing or null DirtySR value as 0 Some older projects had a missing or null DirtySR field, which was causing Mongo to throw an error because we never told it to treat null values as 0 for that int field. It's a simple one-line fix to treat missing values as 0, but treating null values as 0 requires a custom serialization helper. Still simple enough, thankfully. --- .../Model/CustomMongoSerializerForInt32.cs | 24 +++++++++++++++++++ .../LanguageForge/Model/LfLexEntry.cs | 5 +++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs diff --git a/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs b/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs new file mode 100644 index 00000000..21c50e88 --- /dev/null +++ b/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs @@ -0,0 +1,24 @@ +// Copyright (c) 2026 SIL Global +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; + +namespace LfMerge.Core.LanguageForge.Model +{ + public class CustomMongoSerializerForInt32 : Int32Serializer + { + public override int Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + IBsonReader bsonReader = context.Reader; + if (bsonReader.GetCurrentBsonType() == BsonType.Null) + { + bsonReader.ReadNull(); + return 0; + } + return base.Deserialize(context, args); + } + } +} \ No newline at end of file diff --git a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs index 28fb02a6..92ea82e1 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016 SIL International +// Copyright (c) 2016 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -19,6 +19,9 @@ public class LfLexEntry : LfFieldBase, IHasNullableGuid public string MercurialSha { get; set; } public DateTime DateCreated { get; set; } public DateTime DateModified { get; set; } + [BsonDefaultValue(0)] + // CustomMongoSerializerForInt32 converts null to 0 + [BsonSerializer(typeof(CustomMongoSerializerForInt32))] public int DirtySR { get; set ; } // Data properties From 5039fbcb24d00fb19eed0e3bb2db9214ce878095 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 24 Aug 2026 16:54:34 +0700 Subject: [PATCH 06/51] Classify vernacular writing systems per project, not by a fixed list The previous commit took the vernacular writing systems from a fixed list of config fields -- citationForm, lexeme, etymology and the example sentence -- shared with MongoConnection. Checking that against the 2026-07-06 corpus showed one list cannot be right for every project: 741 projects configure etymology with analysis writing systems and only 378 with vernacular ones, so naming etymology vernacular adds "en" to the vernacular list of hundreds of projects that have no English headwords. Classify per project from its own config instead. The lexeme and citation form fields are vernacular by definition and the sense definition and gloss are analysis by definition, so they anchor the decision; every other field carrying input systems -- etymology, the example sentence, custom fields -- is assigned by which anchor its writing systems already appear in. Only a writing system exclusive to one anchor counts as evidence: "en" is in both in many projects and so says nothing about any other field's role. This keeps what the fixed list already got right over the original languageCode rule. flh-flex has languageCode "flh-x-ortho", a writing system only its etymology field uses, so its real lexeme writing system "flh-x-cm" was classified as analysis; khg-yl-flex has the bare tag "khg" while its lexeme uses "khg-CN-x-Yangla" and "khg-fonipa"; odo has languageCode "th", which is not among its writing systems at all, leaving its 24,460 "en" headwords as analysis and the project with no vernacular writing system whatsoever. All three still come out right. Writing systems that overlap neither anchor are treated as vernacular and reported. 11 projects need this; grc-vie-flex is the clearest, a Koine Greek dictionary glossed in English and Vietnamese whose etymologies are in Hebrew and Aramaic -- neither vernacular nor analysis. The FieldWorks fields those feed are vernacular-typed, so calling them analysis would leave the data unreachable, whereas a spare vernacular writing system is inert. Since the two directions no longer share a rule, MongoConnection gets its own list back and MagicStrings.LfVernacularConfigFieldPaths goes away. MongoConnection answers a different question -- which config fields to write the vernacular writing systems into -- and that divergence is now deliberate rather than drift. languageCode survives as a last-resort fallback, for a config LfMerge cannot read at all: without it such a project would end up with no vernacular writing system, which is worse than the rule this replaces. Note this only affects writing systems LfWsToLcmWs creates. A project already synced under an earlier rule keeps the membership its FieldWorks project has. The tests move with the logic, keeping their coverage of the union, case-insensitive matching, the nested example sentence path, apltw2016's second lexeme writing system and the fallbacks. Two of them had described configs that do not match how Mongo spells them, with definition and gloss at the entry level rather than under senses; the anchor paths are strict about that, so those configs are corrected. Co-Authored-By: Claude Opus 5 (1M context) --- .../VernacularClassificationTests.cs | 233 ++++++++++++++++++ .../VernacularWritingSystemTagsTests.cs | 179 -------------- .../ConvertMongoToLcmLexicon.cs | 144 ++++++++--- src/LfMerge.Core/MagicStrings.cs | 12 - .../MongoConnector/MongoConnection.cs | 8 +- 5 files changed, 341 insertions(+), 235 deletions(-) create mode 100644 src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs delete mode 100644 src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs new file mode 100644 index 00000000..7d36ce2a --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -0,0 +1,233 @@ +using System.Collections.Generic; +using LfMerge.Core.DataConverters; +using LfMerge.Core.LanguageForge.Config; +using NUnit.Framework; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// LF has no vernacular/analysis flag on a writing system: which ones are vernacular is derived + /// from the project's own config, not from languageCode. Configs are built by hand here, since + /// this is a pure function needing no Mongo and no LCM project. The fixture must not call + /// MongoConnectionDouble.Initialize -- it throws if another fixture in this assembly has already + /// registered the conventions, and nothing here needs it. + /// + /// Most cases are named after a real project in the 2026-07-06 corpus that has that shape. + /// + public class VernacularClassificationTests + { + private static LfConfigMultiText Field(params string[] inputSystems) + { + return new LfConfigMultiText { InputSystems = new List(inputSystems) }; + } + + /// + /// Builds a config from dotted paths spelled as Mongo spells them, so nesting goes through + /// "fields" ("senses.fields.examples.fields.sentence"). + /// + private static LfProjectConfig Config(params (string Path, LfConfigFieldBase Field)[] fields) + { + var entry = new LfConfigFieldList(); + var senses = new LfConfigFieldList(); + var examples = new LfConfigFieldList(); + foreach ((string path, LfConfigFieldBase field) in fields) + { + if (path.StartsWith("senses.fields.examples.fields.")) + examples.Fields[path.Substring("senses.fields.examples.fields.".Length)] = field; + else if (path.StartsWith("senses.fields.")) + senses.Fields[path.Substring("senses.fields.".Length)] = field; + else + entry.Fields[path] = field; + } + if (examples.Fields.Count > 0) senses.Fields["examples"] = examples; + if (senses.Fields.Count > 0) entry.Fields["senses"] = senses; + return new LfProjectConfig { Entry = entry }; + } + + [Test] + public void LexemeAndCitationFormAreAlwaysVernacular() + { + var config = Config(("lexeme", Field("seh")), ("citationForm", Field("seh-fonipa-x-etic")), + ("senses.fields.definition", Field("en", "pt"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" })); + Assert.That(result.Unresolved, Is.Empty); + } + + [Test] + public void UnionsLexemeAndCitationFormWithoutDuplicating() + { + var config = Config( + ("lexeme", Field("qaa-x-kal", "qaa-fonipa-x-kal")), + ("citationForm", Field("qaa-x-kal", "seh")), + ("senses.fields.definition", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, + Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "seh" })); + } + + [Test] + public void IgnoresTheAnalysisAnchorFields() + { + var config = Config(("lexeme", Field("qaa-x-kal")), + ("senses.fields.definition", Field("en", "fr")), ("senses.fields.gloss", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal" })); + Assert.That(result.Vernacular, Does.Not.Contain("en")); + Assert.That(result.Vernacular, Does.Not.Contain("fr")); + } + + /// + /// apltw2016, the case that started this: its lexeme field uses a phonetic writing system + /// alongside the project language, and the rule this replaces -- vernacular is whatever + /// equals languageCode -- filed that phonetic one as analysis. + /// + [Test] + public void TreatsASecondLexemeWritingSystemAsVernacular() + { + var config = Config(("lexeme", Field("qaa-x-IPA-dupl1", "th"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th"); + Assert.That(result.Vernacular, Contains.Item("th")); + Assert.That(result.Vernacular, Contains.Item("qaa-x-IPA-dupl1"), + "a writing system the lexeme field uses is vernacular even though it is not the language code"); + } + + [Test] + public void MatchesTagsCaseInsensitively() + { + var config = Config(("lexeme", Field("qaa-x-IPA-dupl1"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th"); + Assert.That(result.Vernacular.Contains("qaa-x-ipa-dupl1"), Is.True, + "LCM's own writing system lookups are case-insensitive, so this should be too"); + } + + /// The 741-project shape: etymology written in the analysis language. + [Test] + public void EtymologyInAnAnalysisWritingSystemStaysAnalysis() + { + var config = Config(("lexeme", Field("yog")), ("senses.fields.gloss", Field("en")), + ("etymology", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "yog"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "yog" })); + Assert.That(result.Vernacular, Does.Not.Contain("en")); + } + + /// The 378-project shape, e.g. test-india-sena-01. + [Test] + public void EtymologyInTheVernacularIsVernacular() + { + var config = Config(("lexeme", Field("seh", "seh-fonipa-x-etic")), + ("senses.fields.gloss", Field("en", "pt")), + ("etymology", Field("seh", "seh-fonipa-x-etic"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" })); + } + + /// grc-vie-flex: etymologies in Hebrew and Aramaic, neither vernacular nor analysis. + [Test] + public void WritingSystemsInNeitherAnchorAreVernacularAndReported() + { + var config = Config(("lexeme", Field("grc")), ("senses.fields.gloss", Field("en", "vi")), + ("etymology", Field("hbo", "arc"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "grc"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "grc", "hbo", "arc" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo", "arc" })); + } + + /// + /// With no analysis anchor there is nothing to rule a field out, so every other field's + /// writing systems are unresolved -- and therefore vernacular. + /// + [Test] + public void WithNoAnalysisAnchorEveryOtherFieldIsUnresolved() + { + var config = Config(("lexeme", Field("qaa-x-kal")), ("etymology", Field("qaa-fonipa-x-kal"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-fonipa-x-kal" })); + } + + /// flh-flex: languageCode names a writing system only etymology uses. + [Test] + public void LexemeWinsWhenLanguageCodeDisagrees() + { + var config = Config(("lexeme", Field("flh-x-cm")), ("senses.fields.gloss", Field("en", "id")), + ("etymology", Field("flh-x-ortho"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "flh-x-ortho"); + Assert.That(result.Vernacular, Contains.Item("flh-x-cm")); + } + + /// odo: languageCode "th" is not among the project's writing systems at all. + [Test] + public void LexemeIsVernacularEvenWhenLanguageCodeIsAbsent() + { + var config = Config(("lexeme", Field("en")), + ("senses.fields.definition", Field("qaa-Syrc-IQ-x-syr", "en", "acm"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" })); + } + + /// A writing system in both anchors is no evidence of any other field's role. + [Test] + public void AWritingSystemInBothAnchorsIsNotEvidence() + { + var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("en")), + ("etymology", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "en" }), + "etymology overlaps neither anchor exclusively, so its role is unresolved"); + } + + /// + /// The example sentence is nested, so this covers the recursion into a field list as well as + /// the classification. Its sibling translation field is analysis and must stay that way. + /// + [Test] + public void TheExampleSentenceIsClassifiedLikeAnyOtherField() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.definition", Field("fr")), + ("senses.fields.gloss", Field("en")), + ("senses.fields.examples.fields.sentence", Field("kal")), + ("senses.fields.examples.fields.translation", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Vernacular, Does.Not.Contain("en"), + "the example translation is not a vernacular field"); + Assert.That(result.Unresolved, Is.Empty); + } + + [Test] + public void ToleratesAPartiallyMissingNestedPath() + { + // senses exists but has no children, so the walk has to give up cleanly. + var config = Config(("lexeme", Field("qaa-x-kal")), ("senses", new LfConfigFieldList())); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal" })); + } + + [Test] + public void FallsBackToLanguageCodeWhenTheConfigSaysNothing() + { + Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(null, "fr").Vernacular, + Is.EquivalentTo(new[] { "fr" })); + Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(new LfProjectConfig(), "fr").Vernacular, + Is.EquivalentTo(new[] { "fr" })); + } + + [Test] + public void FallsBackToLanguageCodeWhenLexemeListsNoInputSystems() + { + var config = Config(("lexeme", Field())); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "fr"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "fr" })); + } + + [Test] + public void ReturnsNothingWhenThereIsNoConfigAndNoLanguageCode() + { + Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(null, null).Vernacular, + Is.Empty); + } + } +} diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs deleted file mode 100644 index c2197b3e..00000000 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularWritingSystemTagsTests.cs +++ /dev/null @@ -1,179 +0,0 @@ -using System.Collections.Generic; -using LfMerge.Core.DataConverters; -using LfMerge.Core.LanguageForge.Config; -using NUnit.Framework; - -namespace LfMerge.Core.Tests.Lcm.DataConverters -{ - /// - /// LF has no vernacular/analysis flag on a writing system: which ones are vernacular is implied - /// by the config fields that use them. These cover that derivation on its own, without an LCM - /// project, since it is a pure function of the config and the project's language code. - /// - public class VernacularWritingSystemTagsTests - { - private static LfProjectConfig ConfigWith(params (string FieldName, string[] InputSystems)[] fields) - { - var entry = new LfConfigFieldList(); - foreach ((string fieldName, string[] inputSystems) in fields) - { - entry.Fields[fieldName] = new LfConfigMultiText { - InputSystems = new List(inputSystems) - }; - } - return new LfProjectConfig { Entry = entry }; - } - - [Test] - public void TakesTheLexemeFieldsInputSystems() - { - var config = ConfigWith(("lexeme", new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); - - Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); - } - - [Test] - public void UnionsLexemeAndCitationFormWithoutDuplicating() - { - var config = ConfigWith( - ("lexeme", new[] { "qaa-x-kal", "qaa-fonipa-x-kal" }), - ("citationForm", new[] { "qaa-x-kal", "seh" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); - - Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "seh" })); - } - - [Test] - public void TakesTheEtymologyFieldsInputSystems() - { - var config = ConfigWith( - ("lexeme", new[] { "qaa-x-kal" }), - ("etymology", new[] { "qaa-fonipa-x-kal" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); - - Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); - } - - /// - /// The example sentence lives at senses.fields.examples.fields.sentence, so this covers the - /// nested path as well as the field itself. - /// - [Test] - public void TakesTheExampleSentenceFieldsInputSystems() - { - var entry = new LfConfigFieldList(); - entry.Fields["lexeme"] = new LfConfigMultiText { InputSystems = new List { "qaa-x-kal" } }; - var examples = new LfConfigFieldList(); - examples.Fields["sentence"] = new LfConfigMultiText { InputSystems = new List { "seh" } }; - examples.Fields["translation"] = new LfConfigMultiText { InputSystems = new List { "en" } }; - var senses = new LfConfigFieldList(); - senses.Fields["examples"] = examples; - senses.Fields["definition"] = new LfConfigMultiText { InputSystems = new List { "fr" } }; - entry.Fields["senses"] = senses; - var config = new LfProjectConfig { Entry = entry }; - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); - - Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal", "seh" })); - Assert.That(tags, Does.Not.Contain("en"), "the example translation is not a vernacular field"); - Assert.That(tags, Does.Not.Contain("fr"), "the sense definition is not a vernacular field"); - } - - [Test] - public void ToleratesAPartiallyMissingNestedPath() - { - // senses exists but has no examples, so the nested lookup has to give up cleanly. - var entry = new LfConfigFieldList(); - entry.Fields["lexeme"] = new LfConfigMultiText { InputSystems = new List { "qaa-x-kal" } }; - entry.Fields["senses"] = new LfConfigFieldList(); - var config = new LfProjectConfig { Entry = entry }; - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); - - Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal" })); - } - - [Test] - public void IgnoresAnalysisFields() - { - var config = ConfigWith( - ("lexeme", new[] { "qaa-x-kal" }), - ("definition", new[] { "en", "fr" }), - ("gloss", new[] { "en" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "qaa-x-kal"); - - Assert.That(tags, Is.EquivalentTo(new[] { "qaa-x-kal" })); - Assert.That(tags, Does.Not.Contain("en")); - } - - /// - /// The case this change is for. apltw2016's lexeme field uses a phonetic writing system - /// alongside the project language, and the old rule -- vernacular is whatever equals - /// languageCode -- filed that phonetic one as analysis. - /// - [Test] - public void TreatsASecondLexemeWritingSystemAsVernacular() - { - var config = ConfigWith(("lexeme", new[] { "qaa-x-IPA-dupl1", "th" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "th"); - - Assert.That(tags, Contains.Item("th")); - Assert.That(tags, Contains.Item("qaa-x-IPA-dupl1"), - "a writing system the lexeme field uses is vernacular even though it is not the language code"); - } - - [Test] - public void MatchesTagsCaseInsensitively() - { - var config = ConfigWith(("lexeme", new[] { "qaa-x-IPA-dupl1" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "th"); - - Assert.That(tags.Contains("qaa-x-ipa-dupl1"), Is.True, - "LCM's own writing system lookups are case-insensitive, so this should be too"); - } - - [TestCase(TestName = "FallsBackToLanguageCodeWhenNoVernacularFieldHasInputSystems")] - public void FallsBackToLanguageCode() - { - // A config with only analysis fields: nothing says what the vernacular is, so the - // project's language code is the best available answer -- the rule this replaced. - var config = ConfigWith(("definition", new[] { "en" })); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "fr"); - - Assert.That(tags, Is.EquivalentTo(new[] { "fr" })); - } - - [Test] - public void FallsBackToLanguageCodeWhenLexemeListsNoInputSystems() - { - var config = ConfigWith(("lexeme", new string[0])); - - var tags = ConvertMongoToLcmLexicon.VernacularWritingSystemTags(config, "fr"); - - Assert.That(tags, Is.EquivalentTo(new[] { "fr" })); - } - - [Test] - public void ToleratesAMissingConfig() - { - Assert.That(ConvertMongoToLcmLexicon.VernacularWritingSystemTags(null, "fr"), - Is.EquivalentTo(new[] { "fr" })); - Assert.That(ConvertMongoToLcmLexicon.VernacularWritingSystemTags(new LfProjectConfig(), "fr"), - Is.EquivalentTo(new[] { "fr" })); - } - - [Test] - public void ReturnsNothingWhenThereIsNoConfigAndNoLanguageCode() - { - Assert.That(ConvertMongoToLcmLexicon.VernacularWritingSystemTags(null, null), Is.Empty); - } - } -} diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 37cbb836..1df3952a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -202,56 +202,111 @@ private IEnumerable GetLexicon(ILfProject project) return Connection.GetRecords(project, MagicStrings.LfCollectionNameForLexicon); } + // The two field groups whose vernacular/analysis role is never in doubt. Everything else is + // classified per project by which of these its writing systems already appear in -- a fixed + // list cannot be right for every project: in the 2026-07-06 corpus 741 projects configure + // etymology with analysis writing systems and 378 with vernacular ones. + private static readonly string[] VernacularAnchorFields = { "lexeme", "citationForm" }; + private static readonly string[] AnalysisAnchorFields = { "senses.fields.definition", "senses.fields.gloss" }; + /// - /// Walks a dotted config field path from , or null if any step is - /// missing. The "fields" segments are the Mongo document that holds a field list's children; - /// in the mapped classes that is the Fields dictionary being indexed, so they are skipped. + /// Work out which writing systems this project treats as vernacular, from its own config. + /// + /// The lexeme and citation form fields are vernacular by definition, the sense definition and + /// gloss are analysis by definition. Every other field carrying input systems (etymology, the + /// example sentence, custom fields) is assigned by overlap: a writing system that appears in + /// the vernacular anchors and NOT in the analysis anchors is evidence the field is vernacular, + /// and vice versa. A writing system present in both anchors -- "en" very often is -- is no + /// evidence either way and is ignored for that purpose. + /// + /// This replaces comparing each tag against ProjectRecord.LanguageCode, which named exactly one + /// vernacular writing system and got it wrong whenever languageCode disagreed with the lexeme + /// field: flh-flex has languageCode "flh-x-ortho" (used only by etymology) while its lexeme + /// uses "flh-x-cm", and odo has languageCode "th" which is not among its writing systems at all, + /// leaving its 24,460 "en" headwords classified as analysis. /// - private static LfConfigFieldBase FindConfigField(LfConfigFieldList root, string path) + /// + /// The vernacular tags, and the subset of them that could not be resolved from the config. + /// Unresolved writing systems are treated as VERNACULAR: the FieldWorks fields they feed + /// (etymology form, example sentence) are vernacular-typed, so classifying them as analysis + /// would leave their data with nowhere to go, whereas a spare vernacular writing system is + /// inert. 11 projects in the corpus need this, e.g. grc-vie-flex, whose etymologies are in + /// Hebrew and Aramaic -- neither its vernacular (Greek) nor its analysis (English, Vietnamese). + /// + public static (ISet Vernacular, ISet Unresolved) ClassifyVernacularWritingSystems( + LfProjectConfig config, string languageCode) { - LfConfigFieldBase current = root; - foreach (string segment in path.Split('.')) + var byPath = new Dictionary>(); + CollectInputSystems((config == null) ? null : config.Entry, "", byPath); + + var vernacular = new HashSet(StringComparer.OrdinalIgnoreCase); + var unresolved = new HashSet(StringComparer.OrdinalIgnoreCase); + var vernacularAnchor = Union(byPath, VernacularAnchorFields); + var analysisAnchor = Union(byPath, AnalysisAnchorFields); + vernacular.UnionWith(vernacularAnchor); + + // Only a writing system exclusive to one anchor is evidence of that anchor's role. + var vernacularOnly = new HashSet(vernacularAnchor, StringComparer.OrdinalIgnoreCase); + vernacularOnly.ExceptWith(analysisAnchor); + var analysisOnly = new HashSet(analysisAnchor, StringComparer.OrdinalIgnoreCase); + analysisOnly.ExceptWith(vernacularAnchor); + + var anchors = new HashSet(VernacularAnchorFields.Concat(AnalysisAnchorFields)); + foreach (var field in byPath) { - if (segment == "fields") - { + if (anchors.Contains(field.Key) || field.Value.Count == 0) continue; + if (field.Value.Any(vernacularOnly.Contains)) + { + vernacular.UnionWith(field.Value); } - var fieldList = current as LfConfigFieldList; - if (fieldList?.Fields == null || !fieldList.Fields.TryGetValue(segment, out current)) + else if (!field.Value.Any(analysisOnly.Contains)) { - return null; + // Overlaps neither anchor: nothing in the config says which role this field plays. + vernacular.UnionWith(field.Value); + unresolved.UnionWith(field.Value); } } - return current; + + // Safety net for a config LfMerge cannot read: without this such a project would get no + // vernacular writing system at all, which is worse than the old rule. + if (vernacular.Count == 0 && !string.IsNullOrEmpty(languageCode)) + vernacular.Add(languageCode); + + return (vernacular, unresolved); } - /// - /// The writing system tags LF treats as vernacular: the ones used by the fields in - /// MagicStrings.LfVernacularConfigFieldPaths. A project may well have several -- a phonetic or - /// orthographic alternate alongside the main one -- which is why the project's language code - /// alone is not enough. - /// - /// - /// The tags, compared case-insensitively as LCM's own lookups are. Falls back to the - /// project's language code when the config names no vernacular input systems at all, which - /// is the rule this replaces. - /// - public static HashSet VernacularWritingSystemTags(LfProjectConfig config, string languageCode) + private static ISet Union(IDictionary> byPath, IEnumerable paths) { - var tags = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (string path in MagicStrings.LfVernacularConfigFieldPaths) + var result = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var path in paths) { - if (FindConfigField(config?.Entry, path) is LfConfigMultiText multiText && - multiText.InputSystems != null) - { - tags.UnionWith(multiText.InputSystems); - } + ISet tags; + if (byPath.TryGetValue(path, out tags)) + result.UnionWith(tags); } - if (tags.Count == 0 && !string.IsNullOrEmpty(languageCode)) + return result; + } + + /// + /// Record every config field's input systems, keyed by its dotted path as Mongo spells it, so + /// nesting goes through "fields" ("senses.fields.examples.fields.sentence"). + /// + private static void CollectInputSystems(LfConfigFieldList fieldList, string path, + IDictionary> byPath) + { + if (fieldList == null || fieldList.Fields == null) + return; + foreach (var child in fieldList.Fields) { - tags.Add(languageCode); + string childPath = string.IsNullOrEmpty(path) ? child.Key : path + "." + child.Key; + var multiText = child.Value as LfConfigMultiText; + if (multiText != null && multiText.InputSystems != null) + byPath[childPath] = new HashSet(multiText.InputSystems, StringComparer.OrdinalIgnoreCase); + var nested = child.Value as LfConfigFieldList; + if (nested != null) + CollectInputSystems(nested, childPath + ".fields", byPath); } - return tags; } /// @@ -284,7 +339,16 @@ private void LfWsToLcmWs(Dictionary lfWsList) return; } - HashSet vernacularTags = VernacularWritingSystemTags(ProjectRecord.Config, ProjectRecord.LanguageCode); + // Which writing systems are vernacular is derived from the project's own config rather + // than from languageCode alone; see ClassifyVernacularWritingSystems. + var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, ProjectRecord.LanguageCode); + ISet vernacularTags = classification.Vernacular; + if (classification.Unresolved.Count > 0) + { + Logger.Notice("MongoToLcm: writing system(s) {0} appear only in config fields whose " + + "vernacular/analysis role could not be determined; treating them as vernacular", + string.Join(", ", classification.Unresolved)); + } // TODO: Split the inside of this foreach() out into its own function foreach (var lfWs in lfWsList.Values) { @@ -333,11 +397,11 @@ private void LfWsToLcmWs(Dictionary lfWsList) if (!wsAlreadyExisted) { - // LF doesn't distinguish between vernacular/analysis WS, so take the vernacular - // ones from the fields that are vernacular by convention -- lexeme and citation - // form -- and treat everything else as analysis. Matching on the tag as LF spells - // it, both sides being LF's own strings, so a non-canonical spelling still matches - // itself. + // LF doesn't distinguish between vernacular/analysis WS, so the vernacular ones + // are worked out per project from its config -- see + // ClassifyVernacularWritingSystems -- and everything else is analysis. Matching on + // the tag as LF spells it, both sides being LF's own strings, so a non-canonical + // spelling still matches itself. if (vernacularTags.Contains(lfWs.Tag)) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); else diff --git a/src/LfMerge.Core/MagicStrings.cs b/src/LfMerge.Core/MagicStrings.cs index 2a6e76e7..eebf34f4 100644 --- a/src/LfMerge.Core/MagicStrings.cs +++ b/src/LfMerge.Core/MagicStrings.cs @@ -109,18 +109,6 @@ static MagicStrings() public const string LfFieldNameForDefinition = "definition"; public const string LfFieldNameForExampleSentence = "definition"; - // The LF config fields whose input systems are vernacular writing systems rather than - // analysis ones. LF has no vernacular/analysis flag on a writing system; the distinction is - // conventional, and this list is that convention. Both directions read it so that they - // agree: ConvertMongoToLcmLexicon, turning LF input systems into LCM writing systems, and - // MongoConnection, writing input systems back out to the config. - // Paths are relative to config.entry and are spelled as Mongo sees them, so each level of - // nesting goes through "fields". The custom example writing system is deliberately absent: - // it is a custom field, so its name varies from project to project. - public static IReadOnlyList LfVernacularConfigFieldPaths { get; } = new[] { - "citationForm", "lexeme", "etymology", "senses.fields.examples.fields.sentence" - }; - // Fake language codes used in storing custom GenDate and int fields in Mongo public const string LanguageCodeForGenDateFields = "qaa-Qaad"; public const string LanguageCodeForIntFields = "qaa-Zmth"; diff --git a/src/LfMerge.Core/MongoConnector/MongoConnection.cs b/src/LfMerge.Core/MongoConnector/MongoConnection.cs index 8246ea6f..e9c55da5 100644 --- a/src/LfMerge.Core/MongoConnector/MongoConnection.cs +++ b/src/LfMerge.Core/MongoConnector/MongoConnection.cs @@ -41,10 +41,10 @@ public class MongoConnection : IMongoConnection public LfMergeSettings Settings { get { return _settings; } } // List of LF fields which will use vernacular or pronunciation input systems. Hierarchy is config.entry.fields... - // The vernacular list lives in MagicStrings because ConvertMongoToLcmLexicon reads it too, to - // decide which writing systems become vernacular in LCM; sharing it keeps the two directions - // from drifting apart. (Its comment there covers why the custom example WS is not in it.) - private readonly IReadOnlyList _vernacularWsFieldsList = MagicStrings.LfVernacularConfigFieldPaths; + // We intentionally aren't setting custom example WS here, since it's a custom field with a custom name + private readonly List _vernacularWsFieldsList = new List { + "citationForm", "lexeme", "etymology", "senses.fields.examples.fields.sentence" + }; // Pronunciation fields are special: they want the *first* vernacular WS that uses IPA (or is otherwise flagged // as being a pronunciation writing system). private readonly List _pronunciationWsFieldsList = new List { From 6c45cfddc06e32f8ff027b959d220034794884f8 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 25 Aug 2026 11:37:29 +0700 Subject: [PATCH 07/51] Write MultiUnicode custom fields from LF into LCM SetCustomFieldData had no case for CellarPropertyType.MultiUnicode, so every multilingual text custom field fell through to `default: return false` and its data was dropped. The other direction has handled MultiUnicode all along (ConvertLcmToMongoCustomField line 363), so LfMerge could read one of these fields out of FieldWorks but never write one back in. That asymmetry is invisible in a log. The name lookup succeeds -- the field exists in LCM, so it is not among the remainingFieldNames that produce "Custom field X from LF skipped" -- and the default case returned false without saying anything, carrying a TODO asking for exactly the warning it never got. Migrating thai_jay is what turned it up: its "Top 10 list" field arrived in FieldWorks present but empty for all 537 entries, with nothing in 10,801 lines of LfMerge output to explain why. The comment above the switch is the likely origin, since it lists the types to implement and leaves MultiUnicode out; it is wrong, and is corrected here. This matters more than a missing type usually would, because MultiUnicode is the type every text custom field in a Language Forge project has. LF has never offered a single-string custom field, so nothing LF owns arrives as CellarPropertyType.String -- the one text type that did work. Alternatives LF does not have are cleared rather than left in place. That is safe here specifically because the export writes every alternative LCM holds, via LfMultiText.FromMultiITsString: an alternative missing from LF is one a user deleted there, not one LF never saw. It also matches what LfMultiText.WriteToLcmMultiString does for the built-in multitext fields. Values that have not changed are left untouched, as the String case does, so they stay out of the .fwdata XML and out of the Mercurial commit. MultiString is deliberately not added. GetCustomFieldData cannot produce one either, so an import path for it would be untested code for a shape LfMerge never emits; it now reaches the new warning instead of disappearing. The default case logs a warning, which is what the TODO asked for and what the LCM-to-Mongo direction has always done. A field type going unimplemented is worth a line in the log rather than silence. Tests cover the four cases against the test project's Cust_Single_Line_All, which really is MultiUnicode with English and French alternatives: both alternatives written, a previously emptied field written (the migration case), an alternative LF dropped cleared rather than left stale, and an unidentified writing system skipped. All four fail on the unmodified converter, each showing the field still holding its original "Some custom text". Co-Authored-By: Claude Opus 5 (1M context) --- .../ConvertMongoToLcmCustomFieldTests.cs | 127 ++++++++++++++++++ .../ConvertMongoToLcmCustomField.cs | 65 ++++++++- 2 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs new file mode 100644 index 00000000..46eb72f6 --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs @@ -0,0 +1,127 @@ +// Copyright (c) 2016-2018 SIL International +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +using System; +using LfMerge.Core.DataConverters; +using MongoDB.Bson; +using NUnit.Framework; +using SIL.LCModel; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// Writing custom fields from LF into LCM. + /// + /// Cust_Single_Line_All in the test project is a MultiUnicode field with English and French + /// alternatives. That is the shape every text custom field an LF project owns takes: LF has + /// never offered a single-string custom field, so a field created there is always multilingual + /// even when only one writing system is filled in. + /// + public class ConvertMongoToLcmCustomFieldTests : LcmTestBase + { + private const string MultiTextField = "customField_entry_Cust_Single_Line_All"; + + private static BsonDocument MultiText(params string[] wsThenValue) + { + var multiText = new BsonDocument(); + for (int i = 0; i < wsThenValue.Length; i += 2) + multiText.Add(wsThenValue[i], new BsonDocument("value", wsThenValue[i + 1])); + return multiText; + } + + private ILexEntry TestEntry() + { + var entry = _servLoc.GetInstance().GetObject(Guid.Parse(TestEntryGuidStr)); + Assert.That(entry, Is.Not.Null); + return entry; + } + + private ConvertMongoToLcmCustomField Converter() + { + return new ConvertMongoToLcmCustomField(_cache, _servLoc, + new TestLogger(TestContext.CurrentContext.Test.Name), _wsEn); + } + + /// Reads the entry's custom fields back out through the converter going the other way. + private BsonDocument ReadBack(ILexEntry entry) + { + var reader = new ConvertLcmToMongoCustomField(_cache, _servLoc, + new TestLogger(TestContext.CurrentContext.Test.Name)); + return reader.GetCustomFieldsForThisCmObject(entry, "entry", _listConverters)[0].AsBsonDocument; + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldWriteEveryMultiUnicodeAlternative() + { + // Setup + ILexEntry entry = TestEntry(); + var customFields = new BsonDocument(MultiTextField, + MultiText("en", "Updated English", "fr", "Updated French")); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", customFields, null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written["en"]["value"].AsString, Is.EqualTo("Updated English")); + Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Updated French")); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldWriteAFieldThatWasEmptyInLcm() + { + // The migration case: LF holds the only copy, and LCM has nothing in the field yet. + // Emptying it leaves the alternatives in place holding empty strings rather than removing + // them, so the precondition checks the contents rather than the presence of the key. + ILexEntry entry = TestEntry(); + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "", "fr", "")), null); + foreach (BsonElement alternative in ReadBack(entry)[MultiTextField].AsBsonDocument) + Assert.That(alternative.Value["value"].AsString, Is.Empty, + $"precondition: {alternative.Name} should have been emptied"); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "Brand new")), null); + + // Verify + Assert.That(ReadBack(entry)[MultiTextField]["en"]["value"].AsString, Is.EqualTo("Brand new")); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldClearAnAlternativeLfNoLongerHas() + { + // The reverse converter exports every alternative LCM holds, so one missing from LF is + // one the user deleted there -- not one LF never knew about. + ILexEntry entry = TestEntry(); + Assert.That(ReadBack(entry)[MultiTextField].AsBsonDocument.Contains("fr"), Is.True, + "precondition: the test project's field starts with a French alternative"); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "English only")), null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written["en"]["value"].AsString, Is.EqualTo("English only")); + Assert.That(written.Contains("fr"), Is.False, "French should have been cleared, not left stale"); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldSkipUnidentifiedWritingSystems() + { + // Setup + ILexEntry entry = TestEntry(); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, + MultiText("en", "Kept", "zzz-x-nonesuch", "Dropped")), null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written["en"]["value"].AsString, Is.EqualTo("Kept")); + Assert.That(written.Contains("zzz-x-nonesuch"), Is.False); + } + } +} diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index a8de39d0..ef546e07 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -89,7 +89,9 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui if (fieldName == null) return false; - // Valid field types in LCM are GenDate, Integer, String, OwningAtomic, ReferenceAtomic, and ReferenceCollection, so that's all we implement. + // Valid field types in LCM are GenDate, Integer, String, MultiUnicode, OwningAtomic, + // ReferenceAtomic, and ReferenceCollection, so that's all we implement. MultiUnicode is the + // one every text custom field created in LF has, since LF has no single-string custom field. switch (fieldType) { case CellarPropertyType.GenDate: @@ -356,6 +358,62 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui return true; } + case CellarPropertyType.MultiUnicode: + { + var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); + + // ConvertLcmToMongoCustomField exports every alternative LCM holds, via + // LfMultiText.FromMultiITsString, so an alternative missing here is one that was + // removed in LF rather than one LF never saw. Clearing it is therefore right, and + // matches what LfMultiText.WriteToLcmMultiString does for the built-in multitext + // fields. + var wsIdsToClear = new HashSet(); + ITsMultiString oldValues = data.get_MultiStringProp(hvo, flid); + if (oldValues != null) + { + for (int index = 0; index < oldValues.StringCount; index++) + { + int oldWsId; + oldValues.GetStringFromIndex(index, out oldWsId); + wsIdsToClear.Add(oldWsId); + } + } + + bool changed = false; + foreach (KeyValuePair kv in valueAsMultiText) + { + int wsId = servLoc.WritingSystemFactory.GetWsFromStr(kv.Key); + if (wsId == 0) + { + logger.Warning("Custom field {0}: skipping unidentified writing system {1}", + fieldName, kv.Key); + continue; + } + wsIdsToClear.Remove(wsId); + string text = (kv.Value == null) ? string.Empty : (kv.Value.Value ?? string.Empty); + ITsString newValue = ConvertMongoToLcmTsStrings.SpanStrToTsString( + text, wsId, servLoc.WritingSystemFactory); + ITsString oldValue = data.get_MultiStringAlt(hvo, flid, wsId); + // GetDiffsInTsStrings() returns null when there are no changes, so leaving the + // value alone keeps it out of the .fwdata XML and out of the Mercurial commit. + if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) + continue; + data.SetMultiStringAlt(hvo, flid, wsId, newValue); + changed = true; + } + + foreach (int wsId in wsIdsToClear) + { + ITsString oldValue = data.get_MultiStringAlt(hvo, flid, wsId); + if (oldValue == null || string.IsNullOrEmpty(oldValue.Text)) + continue; + data.SetMultiStringAlt(hvo, flid, wsId, TsStringUtils.EmptyString(wsId)); + changed = true; + } + + return changed; + } + case CellarPropertyType.String: { var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); @@ -379,8 +437,11 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui } default: + logger.Warning( + "Custom field {0} not written to LCM: CellarPropertyType.{1} is not implemented " + + "for the LF to LCM direction, so any data LF holds in it is being dropped", + fieldName, fieldType.ToString()); return false; - // TODO: Maybe issue a proper warning (or error) log message for "field type not recognized"? } } From d90c0b95a8f272697b19131a4faa243b231db697 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 25 Aug 2026 16:23:55 +0700 Subject: [PATCH 08/51] Silence spammy log warning we no longer need Claude determined that this warning is no longer needed; there are indeed some LF fields (usually on the sense level) that have empty but non-null multitexts, but they truly are empty and there is no loss of data involved, so no need for a spammy log warning. --- src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 1df3952a..defea905 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -415,7 +415,8 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool if (input == null) return null; if (input.Count == 0) { - Logger.Warning("BestStringAndWsFromMultiText got a non-null multitext, but it was empty. Empty LF MultiText objects should be nulls in Mongo. Unfortunately, at this point in the code it's hard to know which multitext it was."); + // Some Language Forge fields, like scientificName on senses, have empty but + // non-null multitexts; those should also be empty in LCM. return null; } From e111c604a95e1bac82595e29b012aaa90e17a554 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Sep 2026 14:56:15 +0700 Subject: [PATCH 09/51] Resolve MultiUnicode custom field keys by canonical language tag The MultiUnicode case added to SetCustomFieldData looked each LF key up with GetWsFromStr as LF spells it. LCM knows a writing system only by its canonical id, so a non-canonical key such as "fr-Latn" (canonically "fr") resolved to 0 and was skipped as unidentified. The clearing pass then blanked that writing system's alternative, since nothing from LF had claimed it: the LF value was not merely dropped, the LCM value was deleted. This is the same bug 0c70604 fixed in LfMultiText.WriteToLcmMultiString for the built-in multitext fields; canonicalize the key here the same way. The new test fails without the fix, French having been cleared. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ConvertMongoToLcmCustomFieldTests.cs | 18 ++++++++++++++++++ .../ConvertMongoToLcmCustomField.cs | 7 +++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs index 46eb72f6..71da9b0a 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs @@ -123,5 +123,23 @@ public void SetCustomFieldsForThisCmObject_ShouldSkipUnidentifiedWritingSystems( Assert.That(written["en"]["value"].AsString, Is.EqualTo("Kept")); Assert.That(written.Contains("zzz-x-nonesuch"), Is.False); } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagToTheCanonicalWritingSystem() + { + // LF keeps a tag as typed, while LCM knows the writing system only by its canonical id: + // "fr-Latn" canonicalizes to "fr", French's suppress-script being dropped. Looked up + // literally the key resolves to 0, and French would then be cleared as missing from LF. + ILexEntry entry = TestEntry(); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "English", "fr-Latn", "Français")), null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written.Contains("fr"), Is.True, "French should have been written, not cleared"); + Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Français")); + } } } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index ef546e07..5dd5143a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -382,7 +382,10 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui bool changed = false; foreach (KeyValuePair kv in valueAsMultiText) { - int wsId = servLoc.WritingSystemFactory.GetWsFromStr(kv.Key); + // By the canonical tag, as in LfMultiText.WriteToLcmMultiString: matching LF's + // spelling literally would skip a non-canonical key as unidentified, and the + // clearing pass below would then blank that writing system's text in LCM. + int wsId = servLoc.WritingSystemFactory.GetWsFromStr(LanguageTags.Canonical(kv.Key)); if (wsId == 0) { logger.Warning("Custom field {0}: skipping unidentified writing system {1}", @@ -482,4 +485,4 @@ public void SetCustomFieldsForThisCmObject(ICmObject cmObj, string objectType, B } } } -} \ No newline at end of file +} From 920dc447398279ca4268b16be2c5670491b87f8a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Sep 2026 14:59:25 +0700 Subject: [PATCH 10/51] Classify analysis writing systems too, and let one play both roles ClassifyVernacularWritingSystems returned only a vernacular set, and LfWsToLcmWs filed every new writing system in exactly one list: vernacular if the set held it, analysis otherwise. That got three shapes wrong. A field with a vernacular-only writing system had ALL its writing systems made vernacular, including ones exclusive to the analysis anchors: an etymology configured as [seh, pt] made Portuguese vernacular-only, so a new pt writing system was never added as analysis and pt glosses had nowhere to go. The field's evidence now decides the role of the field, not of every writing system in it; an analysis-only writing system stays analysis. A writing system in both anchors -- "en" in lexeme and gloss alike -- was vernacular, and so never analysis. The classification now returns an analysis set as well, and a new writing system is added to every list it belongs in. A writing system in a field overlapping neither anchor was made vernacular only, on the reasoning that a spare vernacular writing system is inert. It is not: a French-only notes custom field left French out of the analysis list, so an analysis-typed FieldWorks field would not offer it. With no evidence either way such a writing system is now both, which serves vernacular-typed fields (the etymology form, the example sentence) and analysis-typed ones alike. The integration test fails without the LfWsToLcmWs change, the new writing system missing from the current analysis list. This commit also removes the BOM from ConvertMongoToLcmLexicon.cs. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ConvertMongoToLcmWritingSystemsTests.cs | 47 ++++++++++++++ .../VernacularClassificationTests.cs | 58 ++++++++++++++++- .../ConvertMongoToLcmLexicon.cs | 63 ++++++++++++------- 3 files changed, 144 insertions(+), 24 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index b614d4b9..20b61b41 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using LfMerge.Core.LanguageForge.Config; using LfMerge.Core.MongoConnector; using LfMergeBridge.LfMergeModel; using MongoDB.Bson; @@ -53,6 +54,21 @@ private Action AddLfInputSystem(string tag) return () => record.InputSystems.Remove(tag); } + /// + /// Adds a tag to one config field's input systems, found by its path through the entry's + /// field lists, and returns an action that takes it out again. Like the input systems, the + /// config is memoized with the record, so the change must not outlive the test. + /// + private Action AddToConfigField(string tag, params string[] path) + { + LfConfigFieldBase field = _recordFactory.Create(_lfProj).Config.Entry; + foreach (string key in path) + field = ((LfConfigFieldList)field).Fields[key]; + var multiText = (LfConfigMultiText)field; + multiText.InputSystems.Add(tag); + return () => multiText.InputSystems.Remove(tag); + } + private IEnumerable LcmWritingSystemIds() { return _cache.ServiceLocator.WritingSystemManager.WritingSystems.Select(ws => ws.Id); @@ -114,5 +130,36 @@ public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWrit Assert.That(entry.SensesOS[0].Definition.get_String(wsId).Text, Is.EqualTo(newDefinition), "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId); } + + [Test] + public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis() + { + // Setup: a writing system LCM does not have yet, configured for the lexeme and for the + // definition, as "en" so often is. Being vernacular must not stop it being analysis too, + // or the definitions written in it would have no analysis writing system to go to. + const string tag = "qaa-x-both"; + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag); + var restores = new[] { + AddLfInputSystem(tag), + AddToConfigField(tag, "lexeme"), + AddToConfigField(tag, "senses", "definition"), + }; + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + ILangProject langProj = _cache.LanguageProject; + Assert.That(langProj.CurrentVernacularWritingSystems.Select(ws => ws.Id), Contains.Item(tag)); + Assert.That(langProj.CurrentAnalysisWritingSystems.Select(ws => ws.Id), Contains.Item(tag)); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } } } diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 7d36ce2a..0bc99e8b 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -110,6 +110,7 @@ public void EtymologyInAnAnalysisWritingSystemStaysAnalysis() var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "yog"); Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "yog" })); Assert.That(result.Vernacular, Does.Not.Contain("en")); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); } /// The 378-project shape, e.g. test-india-sena-01. @@ -123,20 +124,55 @@ public void EtymologyInTheVernacularIsVernacular() Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" })); } - /// grc-vie-flex: etymologies in Hebrew and Aramaic, neither vernacular nor analysis. + /// + /// A vernacular field is evidence about the field, not about every writing system in it: + /// Portuguese is exclusive to the analysis anchors, so an etymology that also carries the + /// vernacular must not make it vernacular -- a new Portuguese writing system would then be + /// created vernacular-only, and the glosses in it would have no analysis writing system. + /// + [Test] + public void AVernacularFieldDoesNotMakeAnAnalysisOnlyWritingSystemVernacular() + { + var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("en", "pt")), + ("etymology", Field("seh", "pt"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "pt" })); + } + + /// + /// grc-vie-flex: etymologies in Hebrew and Aramaic, neither vernacular nor analysis. With no + /// evidence either way they are both, so neither a vernacular-typed nor an analysis-typed + /// field is left without them. + /// [Test] - public void WritingSystemsInNeitherAnchorAreVernacularAndReported() + public void WritingSystemsInNeitherAnchorAreBothRolesAndReported() { var config = Config(("lexeme", Field("grc")), ("senses.fields.gloss", Field("en", "vi")), ("etymology", Field("hbo", "arc"))); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "grc"); Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "grc", "hbo", "arc" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "vi", "hbo", "arc" })); Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo", "arc" })); } + /// + /// A French-only notes field overlaps neither anchor. French must reach the analysis list, or + /// an analysis-typed FieldWorks custom field would not offer it. + /// + [Test] + public void AnUnresolvedCustomFieldWritingSystemIsAnalysisToo() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("senses.fields.customField_senses_note", Field("fr"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + Assert.That(result.Analysis, Contains.Item("fr")); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "fr" })); + } + /// /// With no analysis anchor there is nothing to rule a field out, so every other field's - /// writing systems are unresolved -- and therefore vernacular. + /// writing systems are unresolved -- and therefore both vernacular and analysis. /// [Test] public void WithNoAnalysisAnchorEveryOtherFieldIsUnresolved() @@ -144,6 +180,7 @@ public void WithNoAnalysisAnchorEveryOtherFieldIsUnresolved() var config = Config(("lexeme", Field("qaa-x-kal")), ("etymology", Field("qaa-fonipa-x-kal"))); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "qaa-fonipa-x-kal" })); Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-fonipa-x-kal" })); } @@ -175,10 +212,24 @@ public void AWritingSystemInBothAnchorsIsNotEvidence() ("etymology", Field("en"))); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en"); Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "en" }), "etymology overlaps neither anchor exclusively, so its role is unresolved"); } + /// + /// The "en" in lexeme and gloss alike: it is vernacular because the lexeme uses it, and it is + /// analysis because the gloss does. Being one must not stop it being the other. + /// + [Test] + public void AWritingSystemInBothAnchorsIsBothVernacularAndAnalysis() + { + var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal", "en" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + /// /// The example sentence is nested, so this covers the recursion into a field list as well as /// the classification. Its sibling translation field is analysis and must stay that way. @@ -194,6 +245,7 @@ public void TheExampleSentenceIsClassifiedLikeAnyOtherField() Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); Assert.That(result.Vernacular, Does.Not.Contain("en"), "the example translation is not a vernacular field"); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "fr" })); Assert.That(result.Unresolved, Is.Empty); } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index defea905..a2270b9a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -210,14 +210,19 @@ private IEnumerable GetLexicon(ILfProject project) private static readonly string[] AnalysisAnchorFields = { "senses.fields.definition", "senses.fields.gloss" }; /// - /// Work out which writing systems this project treats as vernacular, from its own config. + /// Work out which writing systems this project treats as vernacular and which as analysis, + /// from its own config. /// /// The lexeme and citation form fields are vernacular by definition, the sense definition and /// gloss are analysis by definition. Every other field carrying input systems (etymology, the /// example sentence, custom fields) is assigned by overlap: a writing system that appears in /// the vernacular anchors and NOT in the analysis anchors is evidence the field is vernacular, /// and vice versa. A writing system present in both anchors -- "en" very often is -- is no - /// evidence either way and is ignored for that purpose. + /// evidence either way and is ignored for that purpose; it is both vernacular and analysis. + /// + /// Evidence decides the role of the field, not of every writing system in it: a writing + /// system exclusive to the analysis anchors stays analysis-only even in a field classified + /// vernacular, so an etymology configured as [seh, pt] does not make Portuguese vernacular. /// /// This replaces comparing each tag against ProjectRecord.LanguageCode, which named exactly one /// vernacular writing system and got it wrong whenever languageCode disagreed with the lexeme @@ -226,24 +231,32 @@ private IEnumerable GetLexicon(ILfProject project) /// leaving its 24,460 "en" headwords classified as analysis. /// /// - /// The vernacular tags, and the subset of them that could not be resolved from the config. - /// Unresolved writing systems are treated as VERNACULAR: the FieldWorks fields they feed - /// (etymology form, example sentence) are vernacular-typed, so classifying them as analysis - /// would leave their data with nowhere to go, whereas a spare vernacular writing system is - /// inert. 11 projects in the corpus need this, e.g. grc-vie-flex, whose etymologies are in - /// Hebrew and Aramaic -- neither its vernacular (Greek) nor its analysis (English, Vietnamese). + /// The vernacular tags; the analysis tags; and the tags that could not be resolved from the + /// config. The first two sets overlap wherever a writing system plays both roles. A tag in + /// neither set -- one no config field uses -- is analysis, as it always has been. + /// + /// Unresolved writing systems are treated as BOTH vernacular and analysis, since nothing says + /// which they are and a writing system missing from the list a field draws on leaves that + /// field's data with nowhere to go. The FieldWorks fields such a writing system most often + /// feeds (etymology form, example sentence) are vernacular-typed, which is why being only + /// analysis will not do; but a custom field can be analysis-typed, as a French-only notes + /// field would be, which is why being only vernacular will not do either. 11 projects in the + /// corpus need this, e.g. grc-vie-flex, whose etymologies are in Hebrew and Aramaic -- + /// neither its vernacular (Greek) nor its analysis (English, Vietnamese). /// - public static (ISet Vernacular, ISet Unresolved) ClassifyVernacularWritingSystems( - LfProjectConfig config, string languageCode) + public static (ISet Vernacular, ISet Analysis, ISet Unresolved) + ClassifyVernacularWritingSystems(LfProjectConfig config, string languageCode) { var byPath = new Dictionary>(); CollectInputSystems((config == null) ? null : config.Entry, "", byPath); var vernacular = new HashSet(StringComparer.OrdinalIgnoreCase); + var analysis = new HashSet(StringComparer.OrdinalIgnoreCase); var unresolved = new HashSet(StringComparer.OrdinalIgnoreCase); var vernacularAnchor = Union(byPath, VernacularAnchorFields); var analysisAnchor = Union(byPath, AnalysisAnchorFields); vernacular.UnionWith(vernacularAnchor); + analysis.UnionWith(analysisAnchor); // Only a writing system exclusive to one anchor is evidence of that anchor's role. var vernacularOnly = new HashSet(vernacularAnchor, StringComparer.OrdinalIgnoreCase); @@ -258,12 +271,17 @@ public static (ISet Vernacular, ISet Unresolved) ClassifyVernacu continue; if (field.Value.Any(vernacularOnly.Contains)) { - vernacular.UnionWith(field.Value); + vernacular.UnionWith(field.Value.Where(tag => !analysisOnly.Contains(tag))); } - else if (!field.Value.Any(analysisOnly.Contains)) + else if (field.Value.Any(analysisOnly.Contains)) + { + analysis.UnionWith(field.Value); + } + else { // Overlaps neither anchor: nothing in the config says which role this field plays. vernacular.UnionWith(field.Value); + analysis.UnionWith(field.Value); unresolved.UnionWith(field.Value); } } @@ -273,7 +291,7 @@ public static (ISet Vernacular, ISet Unresolved) ClassifyVernacu if (vernacular.Count == 0 && !string.IsNullOrEmpty(languageCode)) vernacular.Add(languageCode); - return (vernacular, unresolved); + return (vernacular, analysis, unresolved); } private static ISet Union(IDictionary> byPath, IEnumerable paths) @@ -343,10 +361,11 @@ private void LfWsToLcmWs(Dictionary lfWsList) // than from languageCode alone; see ClassifyVernacularWritingSystems. var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, ProjectRecord.LanguageCode); ISet vernacularTags = classification.Vernacular; + ISet analysisTags = classification.Analysis; if (classification.Unresolved.Count > 0) { Logger.Notice("MongoToLcm: writing system(s) {0} appear only in config fields whose " - + "vernacular/analysis role could not be determined; treating them as vernacular", + + "vernacular/analysis role could not be determined; treating them as both", string.Join(", ", classification.Unresolved)); } // TODO: Split the inside of this foreach() out into its own function @@ -397,14 +416,16 @@ private void LfWsToLcmWs(Dictionary lfWsList) if (!wsAlreadyExisted) { - // LF doesn't distinguish between vernacular/analysis WS, so the vernacular ones - // are worked out per project from its config -- see - // ClassifyVernacularWritingSystems -- and everything else is analysis. Matching on - // the tag as LF spells it, both sides being LF's own strings, so a non-canonical - // spelling still matches itself. - if (vernacularTags.Contains(lfWs.Tag)) + // LF doesn't distinguish between vernacular/analysis WS, so the roles are worked + // out per project from its config -- see ClassifyVernacularWritingSystems. A + // writing system can play both, "en" in lexeme and gloss alike being the usual + // case, and one no field claims is analysis. Matching on the tag as LF spells it, + // both sides being LF's own strings, so a non-canonical spelling still matches + // itself. + bool isVernacular = vernacularTags.Contains(lfWs.Tag); + if (isVernacular) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); - else + if (!isVernacular || analysisTags.Contains(lfWs.Tag)) ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); } } From 89000580de371df8b5802535de317c09d8a42218 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Sep 2026 15:11:53 +0700 Subject: [PATCH 11/51] Resolve every LF writing system tag through one helper 0c70604 and c91a6c7 canonicalized LF's tags at three GetWsFromStr call sites, but the other places that resolve a tag LF supplied still matched it literally, so each had the non-canonical-tag bug those commits fixed: - BestStringAndWsFromMultiText compared LF's keys with ws.Id. A key such as "fr-Latn" never matched French, so a single-string lexicon field took the text of whichever writing system came next in the search order, or the first key in the document, instead of the preferred one. - The String custom field case resolved the key it picked literally, and gave up on the whole field when that returned 0, so the LF edit never reached LCM. - A span's lang attribute resolved to 0, and building the TsString then threw ArgumentOutOfRangeException ("The specified writing system code is invalid"). - A multi-paragraph's input system and the option list item writing system were resolved literally too. LanguageTags.WsIdFromLfTag is now the one way to resolve an LF tag: the canonical tag first, then the tag as LF spells it, for a writing system LCM might hold under a non-canonical id. Keys are still resolved at lookup rather than rewritten when the multitext is read in, since LF shows text under the tags its own config spells, and rewritten keys would not be shown. LfMultiText gains BestStringAndWsId, which resolves each key to a handle before comparing it with the search order, and whose fallback skips values in writing systems LCM does not know. BestStringAndWsFromMultiText and the String custom field case both use it. The MultiUnicode custom field case re-implemented WriteToLcmMultiString, and the two copies had already drifted: that is how the copy came to lack the canonical lookup that c91a6c7 added. Both now call LfMultiText.WriteToLcm, which reaches LCM through delegates, since a custom field has only ISilDataAccess and not an IMultiAccessorBase. The built-in fields gain two behaviours the custom field copy had: an unchanged alternative is left alone, keeping it out of the .fwdata XML and the Mercurial commit, and a null value is treated as empty rather than throwing. The String custom field and span tests fail without the fix; the others test the new methods directly. All 219 LCM tests pass. The 21 Mercurial and synchronize tests that fail do so because hg init and hg serve cannot run in this environment. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ConvertMongoToLcmCustomFieldTests.cs | 18 +++ .../ConvertMongoToLcmWritingSystemsTests.cs | 65 +++++++++++ .../ConvertMongoToLcmCustomField.cs | 62 +++-------- .../ConvertMongoToLcmLexicon.cs | 15 +-- .../ConvertMongoToLcmOptionList.cs | 4 +- .../ConvertMongoToLcmTsStrings.cs | 4 +- .../LanguageForge/Model/LfMultiText.cs | 103 +++++++++++++++--- src/LfMerge.Core/LanguageTags.cs | 30 ++++- 8 files changed, 221 insertions(+), 80 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs index 71da9b0a..962268f5 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs @@ -20,6 +20,8 @@ namespace LfMerge.Core.Tests.Lcm.DataConverters public class ConvertMongoToLcmCustomFieldTests : LcmTestBase { private const string MultiTextField = "customField_entry_Cust_Single_Line_All"; + // A String field, whose writing system is the magic "first analysis" selector. + private const string SingleStringField = "customField_entry_Cust_Single_Line"; private static BsonDocument MultiText(params string[] wsThenValue) { @@ -141,5 +143,21 @@ public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagToTheCan Assert.That(written.Contains("fr"), Is.True, "French should have been written, not cleared"); Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Français")); } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagInAStringField() + { + // The String case resolved the key it picked with a literal GetWsFromStr too, and gave + // up on the whole field when that returned 0, so the LF edit never reached LCM. + ILexEntry entry = TestEntry(); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(SingleStringField, MultiText("fr-Latn", "Texte")), null); + + // Verify + BsonDocument written = ReadBack(entry)[SingleStringField].AsBsonDocument; + Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Texte")); + } } } diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 20b61b41..f03733ad 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -1,12 +1,15 @@ using System; using System.Collections.Generic; using System.Linq; +using LfMerge.Core.DataConverters; using LfMerge.Core.LanguageForge.Config; +using LfMerge.Core.LanguageForge.Model; using LfMerge.Core.MongoConnector; using LfMergeBridge.LfMergeModel; using MongoDB.Bson; using NUnit.Framework; using SIL.LCModel; +using SIL.LCModel.Core.Text; namespace LfMerge.Core.Tests.Lcm.DataConverters { @@ -131,6 +134,68 @@ public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWrit "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId); } + [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)] + [TestCase(MixedCaseTag, MixedCaseId)] + [TestCase("fr-Latn", "fr")] + public void WsIdFromLfTag_NonCanonicalTag_ResolvesToTheCanonicalWritingSystem(string lfTag, string canonicalId) + { + int expected = _cache.WritingSystemFactory.GetWsFromStr(canonicalId); + Assert.That(expected, Is.Not.EqualTo(0), "{0} should resolve in testlangproj", canonicalId); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, lfTag), Is.EqualTo(expected)); + } + + [Test] + public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() + { + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "zzz-x-nonesuch"), Is.EqualTo(0)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, ""), Is.EqualTo(0)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, null), Is.EqualTo(0)); + } + + [Test] + public void BestStringAndWsId_NonCanonicalKey_StillMatchesThePreferredWritingSystem() + { + // The lexicon's single-string fields compared LF's keys with ws.Id, so "fr-Latn" never + // matched French and the English value was chosen although French comes first. + int wsFr = _cache.WritingSystemFactory.GetWsFromStr("fr"); + var multiText = new LfMultiText { + { "en", LfStringField.FromString("English") }, + { "fr-Latn", LfStringField.FromString("Français") }, + }; + + KeyValuePair best = multiText.BestStringAndWsId(new[] { wsFr, _wsEn }, _cache.WritingSystemFactory); + + Assert.That(best.Key, Is.EqualTo(wsFr)); + Assert.That(best.Value, Is.EqualTo("Français")); + } + + [Test] + public void BestStringAndWsId_FallsBackToTheFirstValueInAKnownWritingSystem() + { + // An unidentified key comes first, but a value LCM has no writing system for is no use. + var multiText = new LfMultiText { + { "zzz-x-nonesuch", LfStringField.FromString("Unplaceable") }, + { "en", LfStringField.FromString("English") }, + }; + + KeyValuePair best = multiText.BestStringAndWsId(new int[0], _cache.WritingSystemFactory); + + Assert.That(best.Key, Is.EqualTo(_wsEn)); + Assert.That(best.Value, Is.EqualTo("English")); + } + + [Test] + public void SpanStrToTsString_NonCanonicalSpanLang_UsesTheCanonicalWritingSystem() + { + int wsFr = _cache.WritingSystemFactory.GetWsFromStr("fr"); + + var tss = ConvertMongoToLcmTsStrings.SpanStrToTsString( + "English français", _wsEn, _cache.WritingSystemFactory); + + Assert.That(tss.RunCount, Is.EqualTo(2)); + Assert.That(tss.get_WritingSystem(1), Is.EqualTo(wsFr)); + } + [Test] public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index 5dd5143a..6391176b 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -178,7 +178,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui if (multiPara.InputSystem == null) wsId = lcmMetaData.GetFieldWs(flid); else - wsId = servLoc.WritingSystemFactory.GetWsFromStr(multiPara.InputSystem); + wsId = LanguageTags.WsIdFromLfTag(servLoc.WritingSystemFactory, multiPara.InputSystem); ConvertUtilities.SetCustomStTextValues(text, multiPara.Paragraphs, wsId); return true; @@ -362,12 +362,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui { var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); - // ConvertLcmToMongoCustomField exports every alternative LCM holds, via - // LfMultiText.FromMultiITsString, so an alternative missing here is one that was - // removed in LF rather than one LF never saw. Clearing it is therefore right, and - // matches what LfMultiText.WriteToLcmMultiString does for the built-in multitext - // fields. - var wsIdsToClear = new HashSet(); + var existingWsIds = new List(); ITsMultiString oldValues = data.get_MultiStringProp(hvo, flid); if (oldValues != null) { @@ -375,57 +370,28 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui { int oldWsId; oldValues.GetStringFromIndex(index, out oldWsId); - wsIdsToClear.Add(oldWsId); + existingWsIds.Add(oldWsId); } } - bool changed = false; - foreach (KeyValuePair kv in valueAsMultiText) - { - // By the canonical tag, as in LfMultiText.WriteToLcmMultiString: matching LF's - // spelling literally would skip a non-canonical key as unidentified, and the - // clearing pass below would then blank that writing system's text in LCM. - int wsId = servLoc.WritingSystemFactory.GetWsFromStr(LanguageTags.Canonical(kv.Key)); - if (wsId == 0) - { - logger.Warning("Custom field {0}: skipping unidentified writing system {1}", - fieldName, kv.Key); - continue; - } - wsIdsToClear.Remove(wsId); - string text = (kv.Value == null) ? string.Empty : (kv.Value.Value ?? string.Empty); - ITsString newValue = ConvertMongoToLcmTsStrings.SpanStrToTsString( - text, wsId, servLoc.WritingSystemFactory); - ITsString oldValue = data.get_MultiStringAlt(hvo, flid, wsId); - // GetDiffsInTsStrings() returns null when there are no changes, so leaving the - // value alone keeps it out of the .fwdata XML and out of the Mercurial commit. - if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) - continue; - data.SetMultiStringAlt(hvo, flid, wsId, newValue); - changed = true; - } - - foreach (int wsId in wsIdsToClear) - { - ITsString oldValue = data.get_MultiStringAlt(hvo, flid, wsId); - if (oldValue == null || string.IsNullOrEmpty(oldValue.Text)) - continue; - data.SetMultiStringAlt(hvo, flid, wsId, TsStringUtils.EmptyString(wsId)); - changed = true; - } - - return changed; + // The same writer as the built-in multitext fields, reaching LCM through + // ISilDataAccess because a custom field has no IMultiAccessorBase. + return valueAsMultiText.WriteToLcm(existingWsIds, + wsId => data.get_MultiStringAlt(hvo, flid, wsId), + (wsId, tss) => data.SetMultiStringAlt(hvo, flid, wsId, tss), + servLoc.WritingSystemFactory, + tag => logger.Warning("Custom field {0}: skipping unidentified writing system {1}", + fieldName, tag)); } case CellarPropertyType.String: { var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); int wsIdForField = lcmMetaData.GetFieldWs(flid); - string wsStrForField = servLoc.WritingSystemFactory.GetStrFromWs(wsIdForField); - KeyValuePair kv = valueAsMultiText.BestStringAndWs(new string[] { wsStrForField }); - string foundWs = kv.Key ?? string.Empty; + KeyValuePair kv = valueAsMultiText.BestStringAndWsId( + new[] { wsIdForField }, servLoc.WritingSystemFactory); + int foundWsId = kv.Key; string foundData = kv.Value ?? string.Empty; - int foundWsId = servLoc.WritingSystemFactory.GetWsFromStr(foundWs); if (foundWsId == 0) return false; // Skip any unidentified writing systems ITsString oldValue = data.get_StringProp(hvo, flid); diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index a2270b9a..e843a3e3 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -448,15 +448,12 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool // _analysisWsIdsAndNamesInSearchOrder : // _vernacularWsIdsAndNamesInSearchOrder; - foreach (ILgWritingSystem ws in wsesToSearch) - { - LfStringField field; - if (input.TryGetValue(ws.Id, out field) && field != null && !field.IsEmpty) - { -// Logger.Debug("Returning TsString from {0} for writing system {1}", field.Value, ws.Id); - return new Tuple(field.Value, ws.Handle); - } - } + // By handle, not by comparing LF's keys with ws.Id, which would miss a key LF spells + // differently from LCM. Falls back to the first value in any writing system LCM knows. + KeyValuePair best = input.BestStringAndWsId( + wsesToSearch.Select(ws => ws.Handle), ServiceLocator.WritingSystemFactory); + if (best.Value != null) + return new Tuple(best.Value, best.Key); // Last-ditch option: just grab the first non-empty string we can find KeyValuePair kv = input.WsIdAndFirstNonEmptyString(Cache); diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs index e33a4f96..1edbc320 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -289,7 +289,7 @@ public void PopulateCmPossibilityFromOptionListItem(ICmPossibility poss, LfOptio // Should only be called when NO canonical item can be found if (wsStr == null) wsStr = "en"; // TODO: Set this from LF user's writing system rather than English, once LF allows setting interface languages - int wsId = poss.Cache.WritingSystemFactory.GetWsFromStr(wsStr); + int wsId = LanguageTags.WsIdFromLfTag(poss.Cache.WritingSystemFactory, wsStr); poss.Abbreviation.set_String(wsId, item.Abbreviation); poss.Name.set_String(wsId, item.Value); } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs index cd1b1b7f..21d01d33 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -89,7 +89,7 @@ public static ITsString SpanStrToTsString(string source, int mainWs, ILgWritingS builder.ClearProps(); // Make sure there aren't leftover properties from previous run // To remove a string property, you set it to null, so we can just use StyleName directly whether or not it's null. builder.SetStrPropValue((int)FwTextPropType.ktptNamedStyle, run.StyleName); - int runWs = (run.Lang == null) ? mainWs : wsf.GetWsFromStr(run.Lang); + int runWs = (run.Lang == null) ? mainWs : LanguageTags.WsIdFromLfTag(wsf, run.Lang); builder.SetIntPropValues((int)FwTextPropType.ktptWs, (int)FwTextPropVar.ktpvDefault, runWs); // We don't care about Guids in this function, so run.Guid is ignored // But we do need to set any other int or string properties that were in the original diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 74000edf..9dcde533 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Linq; @@ -117,10 +117,39 @@ public KeyValuePair WsIdAndFirstNonEmptyString(LcmCache cache) KeyValuePair kv = FirstNonEmptyKeyValue(); if (kv.Key == null) return new KeyValuePair(); ILgWritingSystemFactory wsManager = cache.ServiceLocator.WritingSystemManager; - int wsId = wsManager.GetWsFromStr(LanguageTags.Canonical(kv.Key)); + int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); return new KeyValuePair(wsId, kv.Value); } + /// + /// The value for the first writing system in wsSearchOrder that has a non-empty one, falling + /// back to the first non-empty value in any writing system LCM knows, together with that + /// writing system's handle. Keys are resolved to handles before they are compared, so a key + /// LF spells differently from LCM's id for the writing system still matches it. + /// + /// The handle and the value, or (0, null) when no non-empty value resolves. + public KeyValuePair BestStringAndWsId(IEnumerable wsSearchOrder, ILgWritingSystemFactory wsManager) + { + var resolved = new List>(); + foreach (KeyValuePair kv in this) + { + if (kv.Value == null || kv.Value.IsEmpty) + continue; + int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); + if (wsId != 0) + resolved.Add(new KeyValuePair(wsId, kv.Value.Value)); + } + foreach (int wsId in wsSearchOrder) + { + foreach (KeyValuePair candidate in resolved) + { + if (candidate.Key == wsId) + return candidate; + } + } + return resolved.FirstOrDefault(); + } + public KeyValuePair FirstNonEmptyKeyValue() { KeyValuePair result = this.FirstOrDefault(kv => kv.Value != null && !String.IsNullOrEmpty(kv.Value.Value)); @@ -129,28 +158,68 @@ public KeyValuePair FirstNonEmptyKeyValue() new KeyValuePair(result.Key, result.Value.Value); } - public void WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager) + public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager) { if (dest == null) - return; - HashSet destWsIdsToClear = new HashSet(dest.AvailableWritingSystemIds); + return false; + return WriteToLcm(dest.AvailableWritingSystemIds, dest.get_String, dest.set_String, wsManager); + } + + /// + /// Writes every alternative into an LCM multistring, then clears each alternative LCM holds + /// that has no key here. The built-in multitext fields and the MultiUnicode custom fields + /// both come through here, reaching LCM through an IMultiAccessorBase and through + /// ISilDataAccess respectively, which is why LCM is reached through delegates. + /// + /// Clearing is right because the LCM-to-Mongo direction exports every alternative LCM holds: + /// one missing from LF is one a user removed there, not one LF never saw. + /// + /// Each key is resolved with LanguageTags.WsIdFromLfTag. Resolving it any other way would + /// skip a non-canonical key as unidentified, and the clearing pass would then blank that + /// writing system's text in LCM: the data would not merely be dropped but deleted. + /// + /// An alternative whose value has not changed is left alone, so it stays out of the .fwdata + /// XML and out of the Mercurial commit. + /// + /// The writing systems LCM currently holds alternatives for. + /// Reads LCM's alternative for a writing system. + /// Writes LCM's alternative for a writing system. + /// Resolves LF's keys to LCM writing systems. + /// Told each key that resolves to no writing system. + /// Whether anything in LCM changed. + public bool WriteToLcm(IEnumerable existingWsIds, Func getAlternative, + Action setAlternative, ILgWritingSystemFactory wsManager, + Action onUnidentifiedTag = null) + { + var wsIdsToClear = new HashSet(existingWsIds); + bool changed = false; foreach (KeyValuePair kv in this) { - // By the canonical tag: LF may hold a non-canonical spelling of the id LCM knows the - // writing system by, and GetWsFromStr matches that id literally. Without this the value - // is skipped as unidentified AND the clearing pass below blanks whatever LCM had for - // that writing system, so the data is not merely dropped but deleted. - int wsId = wsManager.GetWsFromStr(LanguageTags.Canonical(kv.Key)); - if (wsId == 0) continue; // Skip any unidentified writing systems - string value = kv.Value.Value; - ITsString tss = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(value, wsId, wsManager); - dest.set_String(wsId, tss); - destWsIdsToClear.Remove(wsId); + int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); + if (wsId == 0) + { + onUnidentifiedTag?.Invoke(kv.Key); + continue; + } + wsIdsToClear.Remove(wsId); + string text = (kv.Value == null) ? string.Empty : (kv.Value.Value ?? string.Empty); + ITsString newValue = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(text, wsId, wsManager); + ITsString oldValue = getAlternative(wsId); + // GetDiffsInTsStrings() returns null when there are no changes + if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) + continue; + setAlternative(wsId, newValue); + changed = true; } - foreach (int wsId in destWsIdsToClear) + foreach (int wsId in wsIdsToClear) { - dest.set_String(wsId, string.Empty); + ITsString oldValue = getAlternative(wsId); + if (oldValue == null || string.IsNullOrEmpty(oldValue.Text)) + continue; + setAlternative(wsId, TsStringUtils.EmptyString(wsId)); + changed = true; } + return changed; } } diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs index 187fe7ec..ac792193 100644 --- a/src/LfMerge.Core/LanguageTags.cs +++ b/src/LfMerge.Core/LanguageTags.cs @@ -1,3 +1,4 @@ +using SIL.LCModel.Core.KernelInterfaces; using SIL.WritingSystems; namespace LfMerge.Core @@ -18,8 +19,9 @@ public static class LanguageTags /// and canonicalizing absorbs it /// th-Thai -> th Thai's suppress-script is dropped /// - /// LfWsToLcmWs does not need this -- GetOrSet canonicalizes for it -- but the lexicon does: - /// its multitext keys are LF tags being looked up against LCM's canonical ids. + /// LfWsToLcmWs does not need this -- GetOrSet canonicalizes for it -- but every other lookup + /// of an LF tag does, and should go through rather than calling + /// this and GetWsFromStr itself. /// /// /// The canonical tag, or the tag unchanged when it is empty or cannot be parsed -- @@ -34,5 +36,29 @@ public static string Canonical(string tag) } return IetfLanguageTag.Canonicalize(tag); } + + /// + /// The handle of the LCM writing system a Language Forge tag names, or 0 when there is none. + /// + /// Every place that resolves a tag LF supplied -- a multitext key, a span's lang attribute, + /// a multi-paragraph's input system -- goes through here, so that a non-canonical spelling + /// resolves the same way everywhere. The canonical tag is tried first, since that is the id + /// LCM gives every writing system it creates; the tag as LF spells it is tried second, for a + /// writing system LCM somehow holds under a non-canonical id. + /// + public static int WsIdFromLfTag(ILgWritingSystemFactory wsf, string tag) + { + if (string.IsNullOrEmpty(tag)) + { + return 0; + } + string canonical = Canonical(tag); + int wsId = wsf.GetWsFromStr(canonical); + if (wsId == 0 && canonical != tag) + { + wsId = wsf.GetWsFromStr(tag); + } + return wsId; + } } } From e69494b370ec0c26779e1884dfd9d46e89be80f7 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Sep 2026 11:26:42 +0700 Subject: [PATCH 12/51] Count the text each writing system holds, field by field ClassifyVernacularWritingSystems decides whether a writing system is vernacular or analysis from the project config alone, and the config is weaker evidence than it looks: Language Forge routinely offers every input system in every field, so a tag configured for the lexeme field may have no headword in it anywhere. Scanning the 2026-07-06 corpus, 750 (project, writing system) pairs are configured somewhere but hold no text at all. Counting the text says what a writing system is really being used for. This adds the counting pass on its own; nothing consumes it yet. Fields are keyed by the dotted path the config spells them with, nesting through "fields" ("senses.fields.examples.fields.sentence"), so the counts line up with the paths classification already works in. Every built-in multitext field is counted, not just the ones classification consults today. Deleted entries are skipped: they are tombstones Language Forge keeps so the deletion can be replayed, and their text is not evidence that anyone is using that writing system. Whitespace counts as empty. Tags are matched case-insensitively, as every LCM writing system lookup is, because the config and the lexicon do not always agree on the case of a tag. Co-Authored-By: Claude Opus 5 --- .../LfWritingSystemUsageTests.cs | 150 +++++++++++++++ .../DataConverters/LfWritingSystemUsage.cs | 171 ++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs create mode 100644 src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs new file mode 100644 index 00000000..d5da98e1 --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) 2016-2018 SIL International +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +using System.Collections.Generic; +using LfMerge.Core.DataConverters; +using LfMerge.Core.LanguageForge.Model; +using NUnit.Framework; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// Counting the text a writing system actually holds, field by field. This is a pure function + /// over the Language Forge model, so the fixture needs neither Mongo nor an LCM project. It + /// must not call MongoConnectionDouble.Initialize, which throws once another fixture in this + /// assembly has registered the conventions. + /// + public class LfWritingSystemUsageTests + { + private static LfMultiText Text(params string[] tagThenValue) + { + var result = new LfMultiText(); + for (int i = 0; i < tagThenValue.Length; i += 2) + result[tagThenValue[i]] = new LfStringField { Value = tagThenValue[i + 1] }; + return result; + } + + private static LfLexEntry Entry(LfMultiText lexeme, params LfSense[] senses) + { + return new LfLexEntry { Lexeme = lexeme, Senses = new List(senses) }; + } + + private static LfSense Sense(LfMultiText gloss, params LfExample[] examples) + { + return new LfSense { Gloss = gloss, Examples = new List(examples) }; + } + + [Test] + public void FromLexicon_CountsOneEntryPerNonEmptyAlternative() + { + var lexicon = new[] { + Entry(Text("qaa-x-kal", "first", "en", "one")), + Entry(Text("qaa-x-kal", "second")), + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.EntriesCounted, Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_DoesNotCountEmptyOrWhitespaceOrMissingText() + { + // A field Language Forge offers but nobody has filled in is exactly what the counts are + // meant to tell apart from a field that is really in use. + var lexicon = new[] { + Entry(Text("qaa-x-kal", "", "en", " ", "fr", "real")), + new LfLexEntry { Lexeme = null }, + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.EntriesCounted, Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(0)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(0)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "fr"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_SkipsDeletedEntries() + { + // Deleted entries are tombstones kept so the deletion can be replayed; their text is + // not evidence that anyone is using that writing system. + var lexicon = new[] { + Entry(Text("qaa-x-kal", "kept")), + new LfLexEntry { Lexeme = Text("qaa-x-kal", "gone"), IsDeleted = true }, + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.EntriesCounted, Is.EqualTo(1)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_CountsNestedSenseAndExampleFields() + { + var lexicon = new[] { + Entry(Text("kal", "word"), + Sense(Text("en", "meaning"), + new LfExample { Sentence = Text("kal", "a sentence"), Translation = Text("en", "a translation") }, + new LfExample { Sentence = Text("kal", "another") })), + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Gloss, "en"), Is.EqualTo(1)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.ExampleSentence, "kal"), Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.ExampleTranslation, "en"), Is.EqualTo(1)); + } + + [Test] + public void NonEmptyCount_SumsOverSeveralFields() + { + var lexicon = new[] { + Entry(Text("kal", "word"), + Sense(null, new LfExample { Sentence = Text("kal", "a sentence") })), + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.NonEmptyCount( + new[] { LfWritingSystemUsage.Lexeme, LfWritingSystemUsage.ExampleSentence }, "kal"), + Is.EqualTo(2)); + } + + [Test] + public void NonEmptyCount_MatchesTagsCaseInsensitively() + { + // Every LCM writing system lookup is case-insensitive, and the config and the lexicon do + // not always agree on the case of a tag. + var lexicon = new[] { Entry(Text("qaa-x-IPA-dupl1", "text")) }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-ipa-dupl1"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_ToleratesAnEmptyOrNullLexicon() + { + Assert.That(LfWritingSystemUsage.FromLexicon(null).EntriesCounted, Is.EqualTo(0)); + Assert.That(LfWritingSystemUsage.FromLexicon(new LfLexEntry[0]).EntriesCounted, Is.EqualTo(0)); + Assert.That(LfWritingSystemUsage.FromLexicon(new LfLexEntry[0]) + .NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(0)); + } + + [Test] + public void TagsWithText_ListsOnlyTheWritingSystemsThatHoldText() + { + var lexicon = new[] { Entry(Text("kal", "word", "en", "", "fr", "mot")) }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.TagsWithText(LfWritingSystemUsage.Lexeme), Is.EquivalentTo(new[] { "kal", "fr" })); + Assert.That(usage.TagsWithText(LfWritingSystemUsage.Gloss), Is.Empty); + } + } +} diff --git a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs new file mode 100644 index 00000000..1a4e9a79 --- /dev/null +++ b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs @@ -0,0 +1,171 @@ +// Copyright (c) 2016-2018 SIL International +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) +using System.Collections.Generic; +using LfMerge.Core.LanguageForge.Model; + +namespace LfMerge.Core.DataConverters +{ + /// + /// How much text each writing system actually holds in each Language Forge field, counted over + /// a project's lexicon. + /// + /// The project config says which writing systems a field OFFERS, which is a much weaker claim + /// than it looks: Language Forge routinely offers every input system in every field, so a tag + /// configured for the lexeme field may well have no headword in it anywhere. Counting the text + /// tells us what a writing system is really being used for, which is what + /// ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems needs to tell a vernacular writing + /// system from an analysis one. + /// + /// Fields are keyed by the dotted path the config spells them with, nesting through "fields" + /// ("senses.fields.examples.fields.sentence"), so the counts line up with the paths that + /// classification already works in. + /// + public class LfWritingSystemUsage + { + // Field paths, spelled as the config spells them. + public const string Lexeme = "lexeme"; + public const string CitationForm = "citationForm"; + public const string Definition = "senses.fields.definition"; + public const string Gloss = "senses.fields.gloss"; + public const string ExampleSentence = "senses.fields.examples.fields.sentence"; + public const string ExampleTranslation = "senses.fields.examples.fields.translation"; + public const string Note = "note"; + public const string LiteralMeaning = "literalMeaning"; + + private readonly Dictionary> _countsByPath = + new Dictionary>(); + + /// How many entries were counted; zero means we have no evidence to go on. + public int EntriesCounted { get; private set; } + + /// The number of non-empty strings this writing system holds in this field. + public int NonEmptyCount(string fieldPath, string tag) + { + Dictionary byTag; + int count; + if (_countsByPath.TryGetValue(fieldPath, out byTag) && byTag.TryGetValue(tag, out count)) + return count; + return 0; + } + + /// + /// The total number of non-empty strings this writing system holds across the given fields. + /// + public int NonEmptyCount(IEnumerable fieldPaths, string tag) + { + int total = 0; + foreach (string path in fieldPaths) + total += NonEmptyCount(path, tag); + return total; + } + + /// The writing systems that hold any text in this field. + public IEnumerable TagsWithText(string fieldPath) + { + Dictionary byTag; + if (!_countsByPath.TryGetValue(fieldPath, out byTag)) + return new string[0]; + return byTag.Keys; + } + + /// + /// Counts the text in every built-in multitext field of every entry. Deleted entries are + /// skipped: they are tombstones Language Forge keeps so the deletion can be replayed, and + /// their text is not evidence of anything. + /// + public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon) + { + var usage = new LfWritingSystemUsage(); + if (lexicon == null) + return usage; + foreach (LfLexEntry entry in lexicon) + { + if (entry == null || entry.IsDeleted) + continue; + usage.EntriesCounted++; + usage.Add(Lexeme, entry.Lexeme); + usage.Add(CitationForm, entry.CitationForm); + usage.Add(Note, entry.Note); + usage.Add(LiteralMeaning, entry.LiteralMeaning); + usage.Add("cvPattern", entry.CvPattern); + usage.Add("entryBibliography", entry.EntryBibliography); + usage.Add("entryRestrictions", entry.EntryRestrictions); + usage.Add("etymology", entry.Etymology); + usage.Add("etymologyComment", entry.EtymologyComment); + usage.Add("etymologyGloss", entry.EtymologyGloss); + usage.Add("etymologySource", entry.EtymologySource); + usage.Add("pronunciation", entry.Pronunciation); + usage.Add("summaryDefinition", entry.SummaryDefinition); + usage.Add("tone", entry.Tone); + if (entry.Senses == null) + continue; + foreach (LfSense sense in entry.Senses) + { + if (sense == null) + continue; + usage.Add(Definition, sense.Definition); + usage.Add(Gloss, sense.Gloss); + usage.Add("senses.fields.anthropologyNote", sense.AnthropologyNote); + usage.Add("senses.fields.discourseNote", sense.DiscourseNote); + usage.Add("senses.fields.encyclopedicNote", sense.EncyclopedicNote); + usage.Add("senses.fields.generalNote", sense.GeneralNote); + usage.Add("senses.fields.grammarNote", sense.GrammarNote); + usage.Add("senses.fields.phonologyNote", sense.PhonologyNote); + usage.Add("senses.fields.scientificName", sense.ScientificName); + usage.Add("senses.fields.semanticsNote", sense.SemanticsNote); + usage.Add("senses.fields.senseBibliography", sense.SenseBibliography); + usage.Add("senses.fields.senseImportResidue", sense.SenseImportResidue); + usage.Add("senses.fields.senseRestrictions", sense.SenseRestrictions); + usage.Add("senses.fields.sociolinguisticsNote", sense.SociolinguisticsNote); + usage.Add("senses.fields.source", sense.Source); + if (sense.Examples != null) + { + foreach (LfExample example in sense.Examples) + { + if (example == null) + continue; + usage.Add(ExampleSentence, example.Sentence); + usage.Add(ExampleTranslation, example.Translation); + usage.Add("senses.fields.examples.fields.reference", example.Reference); + } + } + if (sense.Pictures != null) + { + foreach (LfPicture picture in sense.Pictures) + { + if (picture != null) + usage.Add("senses.fields.pictures", picture.Caption); + } + } + } + } + return usage; + } + + /// + /// Counts one field's non-empty alternatives. Whitespace counts as empty; span markup is + /// not stripped first, since a string made only of markup carries no text to begin with. + /// + private void Add(string fieldPath, LfMultiText text) + { + if (text == null) + return; + foreach (KeyValuePair kv in text) + { + if (kv.Value == null || string.IsNullOrWhiteSpace(kv.Value.Value)) + continue; + Dictionary byTag; + if (!_countsByPath.TryGetValue(fieldPath, out byTag)) + { + // Case-insensitively, as every LCM writing system lookup is: the config and the + // lexicon do not always agree on the case of a tag, and they mean the same one. + byTag = new Dictionary(System.StringComparer.OrdinalIgnoreCase); + _countsByPath[fieldPath] = byTag; + } + int count; + byTag.TryGetValue(kv.Key, out count); + byTag[kv.Key] = count + 1; + } + } + } +} From 9badbf2d26cfd50e8eec44c911648b571bd39ae5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Sep 2026 11:30:00 +0700 Subject: [PATCH 13/51] Classify a writing system by where its text actually is The config says which writing systems a field OFFERS, and Language Forge routinely offers every input system in every field. ClassifyVernacularWriting- Systems believed it, so a project whose lexeme field lists "en" alongside the vernacular got "en" classified vernacular even when not one headword is written in it. That is the shape behind most of the corpus's unresolved writing systems: 168 of the 323 have lexeme and gloss offering exactly the same input systems, so neither anchor has any exclusive tag and every other field becomes unresolved. Where the lexicon has text to show for a writing system, that text now decides, and the config only answers for writing systems with nothing written in them. By share, not by presence. A few strays are not evidence of a second role: in brb-flex-2022 the phonetic writing system has 4,030 headwords and one stray string in an example reference, and FieldWorks has it as vernacular only, while in cmo-khm-flex the vernacular has 7,815 vernacular strings against 300 in glosses and is likewise vernacular only. The one writing system those projects really do list as both sits at 28%, so 10% is a round number in the middle of the gap. What no text can settle, language often can. A writing system nothing else has placed still has a language, and the other writing systems spelling that language have usually been placed already: a phonetic or audio spelling of the vernacular is vernacular too, and a regional spelling of the analysis language is analysis. The language of a tag, for this purpose, is its language subtag plus any private-use subtags that name the language rather than a variant of it. Script and region come out, so "en" and "en-GB" are one language. The private-use subtags have to stay, because under "qaa" -- the code for a language with no code of its own, which Language Forge projects lean on heavily -- they ARE the language: "qaa-x-kal" and "qaa-x-hbo" are no more the same language than "fr" and "de". The phonetic markers, the audio marker and the "dupl1" suffixes FieldWorks appends all come out, so "qaa-x-kal", "qaa-fonipa-x-kal" and "qaa-Zxxx-x-kal-audio" are one language. A language both roles already use settles nothing, for the same reason a writing system in both anchors is no evidence of a field's role. Checked against the four FieldWorks repositories on hand, whose LanguageProject.langproj records the real vernacular and analysis lists: this gets 18 of 18 writing systems right, where the config-only rule gets 15 right and 2 wrong (cmo-khm-flex's km-KH and xkk-flex-2022's en, both wrongly vernacular). Across the 2026-07-06 corpus it takes the projects with at least one unresolved writing system from 323 to 69, and the unresolved (project, writing system) pairs from 419 to 71. A writing system the config forgot but the lexicon uses is now classified too, rather than falling through as neither. Counting means a second streaming pass over the Mongo lexicon before any writing system is created. GetLexicon yields from a cursor, so the lexicon does not all sit in memory at once. WithNoAnalysisAnchorEveryOtherFieldIsUnresolved covered a phonetic spelling of the lexeme's own language, which language affinity now settles, so it grows a writing system of an unrelated language to keep making its original point. Co-Authored-By: Claude Opus 5 --- .../VernacularClassificationTests.cs | 228 +++++++++++++++++- .../ConvertMongoToLcmLexicon.cs | 199 +++++++++++++-- 2 files changed, 403 insertions(+), 24 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 0bc99e8b..03896ec4 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -1,6 +1,8 @@ +using System; using System.Collections.Generic; using LfMerge.Core.DataConverters; using LfMerge.Core.LanguageForge.Config; +using LfMerge.Core.LanguageForge.Model; using NUnit.Framework; namespace LfMerge.Core.Tests.Lcm.DataConverters @@ -44,6 +46,37 @@ private static LfProjectConfig Config(params (string Path, LfConfigFieldBase Fie return new LfProjectConfig { Entry = entry }; } + /// + /// One entry holding text at the given field path, so a lexicon of them gives a writing + /// system a known amount of text in a field of known role. + /// + private static LfLexEntry EntryWithTextAt(string fieldPath, string tag) + { + var text = new LfMultiText { { tag, LfStringField.FromString("text") } }; + var entry = new LfLexEntry(); + switch (fieldPath) + { + case LfWritingSystemUsage.Lexeme: entry.Lexeme = text; break; + case LfWritingSystemUsage.CitationForm: entry.CitationForm = text; break; + case LfWritingSystemUsage.Definition: + entry.Senses = new List { new LfSense { Definition = text } }; break; + case LfWritingSystemUsage.Gloss: + entry.Senses = new List { new LfSense { Gloss = text } }; break; + case "etymology": entry.Etymology = text; break; + default: throw new ArgumentException("no test entry shape for " + fieldPath); + } + return entry; + } + + private static LfWritingSystemUsage Usage(params (string Path, string Tag, int Count)[] spec) + { + var lexicon = new List(); + foreach ((string path, string tag, int count) in spec) + for (int i = 0; i < count; i++) + lexicon.Add(EntryWithTextAt(path, tag)); + return LfWritingSystemUsage.FromLexicon(lexicon); + } + [Test] public void LexemeAndCitationFormAreAlwaysVernacular() { @@ -172,16 +205,21 @@ public void AnUnresolvedCustomFieldWritingSystemIsAnalysisToo() /// /// With no analysis anchor there is nothing to rule a field out, so every other field's - /// writing systems are unresolved -- and therefore both vernacular and analysis. + /// writing systems are unresolved -- and therefore both vernacular and analysis. Unless + /// their language has already been spoken for: qaa-fonipa-x-kal is a phonetic spelling of + /// the same language as the lexeme's qaa-x-kal, so it is vernacular like its relative, + /// while Hebrew keeps no such company and stays in doubt. /// [Test] public void WithNoAnalysisAnchorEveryOtherFieldIsUnresolved() { - var config = Config(("lexeme", Field("qaa-x-kal")), ("etymology", Field("qaa-fonipa-x-kal"))); + var config = Config(("lexeme", Field("qaa-x-kal")), + ("etymology", Field("qaa-fonipa-x-kal", "hbo"))); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); - Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal" })); - Assert.That(result.Analysis, Is.EquivalentTo(new[] { "qaa-fonipa-x-kal" })); - Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-fonipa-x-kal" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" })); + Assert.That(result.Vernacular, + Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "hbo" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "hbo" })); } /// flh-flex: languageCode names a writing system only etymology uses. @@ -275,6 +313,186 @@ public void FallsBackToLanguageCodeWhenLexemeListsNoInputSystems() Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "fr" })); } + /// + /// The case that motivates counting at all: the config offers "en" for the lexeme field, so + /// the config alone calls it vernacular, but no headword is written in it. + /// + [Test] + public void AWritingSystemConfiguredForTheLexemeButUnusedThereIsNotVernacular() + { + var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 100), + (LfWritingSystemUsage.Gloss, "en", 100)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + Assert.That(result.Vernacular, Does.Not.Contain("en"), + "nothing is written in en in the lexeme field, so it is no evidence of vernacular"); + } + + /// + /// brb-flex-2022: the phonetic writing system has 4,030 headwords and one stray string in an + /// example reference, and FieldWorks has it as vernacular only. + /// + [Test] + public void AFewStrayStringsDoNotMakeAVernacularWritingSystemAnalysisAsWell() + { + var config = Config(("lexeme", Field("seh-fonipa-x-etic")), ("senses.fields.gloss", Field("en", "seh-fonipa-x-etic"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "seh-fonipa-x-etic", 4030), + (LfWritingSystemUsage.Gloss, "seh-fonipa-x-etic", 1), + (LfWritingSystemUsage.Gloss, "en", 4000)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh", usage); + + Assert.That(result.Vernacular, Contains.Item("seh-fonipa-x-etic")); + Assert.That(result.Analysis, Does.Not.Contain("seh-fonipa-x-etic")); + Assert.That(result.Analysis, Contains.Item("en")); + } + + /// + /// brb-flex-2022 again: the vernacular is also glossed in, 72/28, and FieldWorks really does + /// list it in both. + /// + [Test] + public void AWritingSystemUsedSubstantiallyForBothRolesGetsBoth() + { + var config = Config(("lexeme", Field("brb")), ("senses.fields.gloss", Field("brb", "en"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "brb", 7256), + (LfWritingSystemUsage.Gloss, "brb", 2801)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "brb", usage); + + Assert.That(result.Vernacular, Contains.Item("brb")); + Assert.That(result.Analysis, Contains.Item("brb")); + Assert.That(result.Unresolved, Is.Empty, "28% is a real second role, not an unknown one"); + } + + [Test] + public void TextOutweighsTheConfigInBothDirections() + { + // Configured the wrong way round: the lexeme offers "en" and the gloss offers "kal", + // but every headword is in kal and every gloss in en. + var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("kal"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 50), + (LfWritingSystemUsage.Gloss, "en", 50)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en", usage); + + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + + [Test] + public void AWritingSystemUsedOnlyInAFieldOfUnknownRoleIsStillUnresolved() + { + // Text in the etymology says nothing: the corpus has 33 projects writing etymologies in + // the vernacular and 50 in an analysis language. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("etymology", Field("hbo"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), ("etymology", "hbo", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" })); + } + + [Test] + public void AWritingSystemTheConfigForgotIsClassifiedFromItsTextAnyway() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Lexeme, "kal-fonipa", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Contains.Item("kal-fonipa")); + } + + [Test] + public void AnEmptyLexiconLeavesTheConfigInCharge() + { + var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en"))); + + var withoutData = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", + LfWritingSystemUsage.FromLexicon(new LfLexEntry[0])); + var configOnly = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + + Assert.That(withoutData.Vernacular, Is.EquivalentTo(configOnly.Vernacular)); + Assert.That(withoutData.Analysis, Is.EquivalentTo(configOnly.Analysis)); + Assert.That(withoutData.Vernacular, Contains.Item("en")); + } + + /// + /// A writing system with nothing written in it and no company still has a language, and the + /// other writing systems of that language have already been placed. + /// + [TestCase("seh", "seh-fonipa-x-etic", true, TestName = "AffinityPhoneticVariantOfTheVernacular")] + [TestCase("qaa-x-kal", "qaa-Zxxx-x-kal-audio", true, TestName = "AffinityAudioVariantOfTheVernacular")] + [TestCase("qaa-x-kal", "qaa-x-kal-dupl1", true, TestName = "AffinityDuplicateOfTheVernacular")] + public void AWritingSystemOfTheSameLanguageAsTheVernacularIsVernacular( + string vernacularTag, string relative, bool isVernacular) + { + var config = Config(("lexeme", Field(vernacularTag)), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field(relative))); + var usage = Usage((LfWritingSystemUsage.Lexeme, vernacularTag, 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, vernacularTag, usage); + + Assert.That(result.Unresolved, Is.Empty); + Assert.That(result.Vernacular.Contains(relative), Is.EqualTo(isVernacular)); + Assert.That(result.Analysis.Contains(relative), Is.EqualTo(!isVernacular)); + } + + [Test] + public void AWritingSystemOfTheSameLanguageAsTheAnalysisLanguageIsAnalysis() + { + // Script and region are not the language: "en-GB" is English. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field("en-GB"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.Empty); + Assert.That(result.Analysis, Contains.Item("en-GB")); + Assert.That(result.Vernacular, Does.Not.Contain("en-GB")); + } + + /// + /// Under "qaa" the private-use subtags are the language, so two of them are two languages. + /// + [Test] + public void ADifferentPrivateUseLanguageIsNotTheSameLanguage() + { + var config = Config(("lexeme", Field("qaa-x-kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field("qaa-x-hbo"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "qaa-x-kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-x-hbo" })); + } + + [Test] + public void ALanguageUsedForBothRolesSettlesNothing() + { + // seh is the vernacular and seh-fonipa is glossed in, so the language says nothing. + var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("seh-fonipa")), + ("customField_entry_Extra", Field("seh-Latn"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "seh", 10), + (LfWritingSystemUsage.Gloss, "seh-fonipa", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "seh-Latn" })); + } + [Test] public void ReturnsNothingWhenThereIsNoConfigAndNoLanguageCode() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index e843a3e3..a6e4ed94 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -139,9 +139,15 @@ public ConversionError RunConversion() // Logger.Debug("Running \"fake\" MtFComments, should see comments show up below:"); var entryObjectIdToGuidMappings = Connection.GetGuidsByObjectIdForCollection(LfProject, MagicStrings.LfCollectionNameForLexicon); EntryCounts.Reset(); + // Counting the lexicon before any writing system is created, because which list a new + // writing system belongs in depends on what it is actually used for. This is a second + // streaming pass over Mongo; GetLexicon yields from a cursor, so it does not all sit in + // memory at once. + LfWritingSystemUsage wsUsage = LfWritingSystemUsage.FromLexicon(GetLexicon(LfProject)); + // Update writing systems from project config input systems. Won't commit till the end UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", Cache.ActionHandlerAccessor, () => - LfWsToLcmWs(ProjectRecord.InputSystems)); + LfWsToLcmWs(ProjectRecord.InputSystems, wsUsage)); // Set English ws handle again in case it changed _wsEn = ServiceLocator.WritingSystemFactory.GetWsFromStr("en"); @@ -224,15 +230,21 @@ private IEnumerable GetLexicon(ILfProject project) /// system exclusive to the analysis anchors stays analysis-only even in a field classified /// vernacular, so an etymology configured as [seh, pt] does not make Portuguese vernacular. /// + /// Where the lexicon has text to show for a writing system, that text decides instead: a + /// writing system holding nearly all of its text in the vernacular anchors is vernacular + /// whatever the config offers, and vice versa. See . + /// /// This replaces comparing each tag against ProjectRecord.LanguageCode, which named exactly one /// vernacular writing system and got it wrong whenever languageCode disagreed with the lexeme /// field: flh-flex has languageCode "flh-x-ortho" (used only by etymology) while its lexeme /// uses "flh-x-cm", and odo has languageCode "th" which is not among its writing systems at all, /// leaving its 24,460 "en" headwords classified as analysis. /// + /// + /// How much text each writing system holds in each field, or null to go on the config alone. + /// /// - /// The vernacular tags; the analysis tags; and the tags that could not be resolved from the - /// config. The first two sets overlap wherever a writing system plays both roles. A tag in + /// The vernacular tags; the analysis tags; and the tags that could not be resolved. The first two sets overlap wherever a writing system plays both roles. A tag in /// neither set -- one no config field uses -- is analysis, as it always has been. /// /// Unresolved writing systems are treated as BOTH vernacular and analysis, since nothing says @@ -245,7 +257,8 @@ private IEnumerable GetLexicon(ILfProject project) /// neither its vernacular (Greek) nor its analysis (English, Vietnamese). /// public static (ISet Vernacular, ISet Analysis, ISet Unresolved) - ClassifyVernacularWritingSystems(LfProjectConfig config, string languageCode) + ClassifyVernacularWritingSystems(LfProjectConfig config, string languageCode, + LfWritingSystemUsage usage = null) { var byPath = new Dictionary>(); CollectInputSystems((config == null) ? null : config.Entry, "", byPath); @@ -253,39 +266,51 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre var vernacular = new HashSet(StringComparer.OrdinalIgnoreCase); var analysis = new HashSet(StringComparer.OrdinalIgnoreCase); var unresolved = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Writing systems whose text has already answered for them. They keep that answer: the + // config below may offer them for fields nobody writes in, and must not overrule it. + var decidedByText = ClassifyByText(usage, byPath, vernacular, analysis); + var vernacularAnchor = Union(byPath, VernacularAnchorFields); var analysisAnchor = Union(byPath, AnalysisAnchorFields); - vernacular.UnionWith(vernacularAnchor); - analysis.UnionWith(analysisAnchor); + vernacular.UnionWith(vernacularAnchor.Where(tag => !decidedByText.Contains(tag))); + analysis.UnionWith(analysisAnchor.Where(tag => !decidedByText.Contains(tag))); - // Only a writing system exclusive to one anchor is evidence of that anchor's role. - var vernacularOnly = new HashSet(vernacularAnchor, StringComparer.OrdinalIgnoreCase); - vernacularOnly.ExceptWith(analysisAnchor); - var analysisOnly = new HashSet(analysisAnchor, StringComparer.OrdinalIgnoreCase); - analysisOnly.ExceptWith(vernacularAnchor); + // Only a writing system exclusive to one role is evidence of a field's role. + var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase); + vernacularOnly.ExceptWith(analysis); + var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); + analysisOnly.ExceptWith(vernacular); var anchors = new HashSet(VernacularAnchorFields.Concat(AnalysisAnchorFields)); foreach (var field in byPath) { if (anchors.Contains(field.Key) || field.Value.Count == 0) continue; + // A writing system its own text has spoken for is still evidence of what this field + // is for, but nothing here may change the answer it was given. + var undecided = field.Value.Where(tag => !decidedByText.Contains(tag)).ToList(); + if (undecided.Count == 0) + continue; if (field.Value.Any(vernacularOnly.Contains)) { - vernacular.UnionWith(field.Value.Where(tag => !analysisOnly.Contains(tag))); + vernacular.UnionWith(undecided.Where(tag => !analysisOnly.Contains(tag))); } else if (field.Value.Any(analysisOnly.Contains)) { - analysis.UnionWith(field.Value); + analysis.UnionWith(undecided); } else { - // Overlaps neither anchor: nothing in the config says which role this field plays. - vernacular.UnionWith(field.Value); - analysis.UnionWith(field.Value); - unresolved.UnionWith(field.Value); + // Overlaps neither role: nothing says which role this field plays. + vernacular.UnionWith(undecided); + analysis.UnionWith(undecided); + unresolved.UnionWith(undecided); } } + ResolveByLanguageAffinity(vernacular, analysis, unresolved); + // Safety net for a config LfMerge cannot read: without this such a project would get no // vernacular writing system at all, which is worse than the old rule. if (vernacular.Count == 0 && !string.IsNullOrEmpty(languageCode)) @@ -294,6 +319,140 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre return (vernacular, analysis, unresolved); } + /// + /// Settles what is left by language. A writing system plays the same role as the others + /// that spell the same language: a phonetic or audio spelling of the vernacular is + /// vernacular too, and a regional spelling of the analysis language is analysis. + /// + private static void ResolveByLanguageAffinity(ISet vernacular, ISet analysis, + ISet unresolved) + { + if (unresolved.Count == 0) + return; + + // Unresolved writing systems sit in both sets, so they are in neither of these and + // cannot vote for each other. + var vernacularLanguages = new HashSet(StringComparer.OrdinalIgnoreCase); + var analysisLanguages = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (string tag in vernacular) + { + if (!analysis.Contains(tag)) + vernacularLanguages.Add(LanguageOf(tag)); + } + foreach (string tag in analysis) + { + if (!vernacular.Contains(tag)) + analysisLanguages.Add(LanguageOf(tag)); + } + + foreach (string tag in unresolved.ToList()) + { + string language = LanguageOf(tag); + bool withVernacular = vernacularLanguages.Contains(language); + bool withAnalysis = analysisLanguages.Contains(language); + if (withVernacular == withAnalysis) + continue; // The language is spoken for by both roles, or by neither. + if (withVernacular) + analysis.Remove(tag); + else + vernacular.Remove(tag); + unresolved.Remove(tag); + } + } + + // Subtags that mark a variant of a language rather than a language of its own. FieldWorks + // also appends "dupl1", "dupl2" and so on when a project needs a second writing system for + // the same thing. + private static readonly HashSet VariantMarkers = new HashSet( + new[] { "fonipa", "etic", "emic", "audio" }, StringComparer.OrdinalIgnoreCase); + + /// + /// The language a tag names, as far as telling one project's writing systems apart goes: + /// the language subtag plus any private-use subtags that name the language rather than a + /// variant of it. Script and region are left out, so "en" and "en-GB" are one language. + /// + /// The private-use subtags have to stay, because under "qaa" -- the code for a language + /// with no code of its own, which Language Forge projects lean on heavily -- they ARE the + /// language: "qaa-x-kal" and "qaa-x-hbo" are no more the same language than "fr" and "de". + /// So "qaa-x-kal", "qaa-fonipa-x-kal" and "qaa-Zxxx-x-kal-audio" all come out "qaa-kal", + /// while "seh" and "seh-fonipa-x-etic" both come out "seh". + /// + private static string LanguageOf(string tag) + { + if (string.IsNullOrEmpty(tag)) + return tag; + string[] parts = tag.Split('-'); + var language = new List { parts[0] }; + int privateUse = Array.FindIndex(parts, 1, + part => part.Equals("x", StringComparison.OrdinalIgnoreCase)); + if (privateUse >= 0) + { + for (int i = privateUse + 1; i < parts.Length; i++) + { + if (VariantMarkers.Contains(parts[i]) || + parts[i].StartsWith("dupl", StringComparison.OrdinalIgnoreCase)) + continue; + language.Add(parts[i]); + } + } + return string.Join("-", language); + } + + /// + /// A writing system holding at most this share of its text in the fields of one role is + /// taken to belong to the other role alone; in between, it plays both. + /// + /// A few strays are not evidence of a second role. In brb-flex-2022 the phonetic writing + /// system has 4,030 headwords and one stray string in an example reference, and FieldWorks + /// has it as vernacular only; in cmo-khm-flex the vernacular has 7,815 vernacular strings + /// against 300 in glosses, and is likewise vernacular only. The one writing system the four + /// FieldWorks projects on hand really do list as BOTH sits at 28%, so anything between 4% + /// and 28% separates the two, and 10% is the round number in the middle. + /// + public const double MinorityShare = 0.10; + + /// + /// Classifies every writing system the lexicon has text for, by where that text sits. + /// Adds them to and , and returns + /// the ones it answered for. + /// + private static ISet ClassifyByText(LfWritingSystemUsage usage, + IDictionary> byPath, ISet vernacular, ISet analysis) + { + var decided = new HashSet(StringComparer.OrdinalIgnoreCase); + if (usage == null || usage.EntriesCounted == 0) + return decided; + + // Every writing system the config mentions, plus any the config forgot but the lexicon + // uses anyway -- those need classifying just as much. + var candidates = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (ISet tags in byPath.Values) + candidates.UnionWith(tags); + foreach (string path in VernacularAnchorFields.Concat(AnalysisAnchorFields)) + candidates.UnionWith(usage.TagsWithText(path)); + + foreach (string tag in candidates) + { + int vernacularText = usage.NonEmptyCount(VernacularAnchorFields, tag); + int analysisText = usage.NonEmptyCount(AnalysisAnchorFields, tag); + int total = vernacularText + analysisText; + if (total == 0) + continue; // No text in a field of known role, so its text says nothing. + double analysisShare = (double)analysisText / total; + if (analysisShare <= MinorityShare) + vernacular.Add(tag); + else if (analysisShare >= 1.0 - MinorityShare) + analysis.Add(tag); + else + { + vernacular.Add(tag); + analysis.Add(tag); + } + decided.Add(tag); + } + return decided; + } + private static ISet Union(IDictionary> byPath, IEnumerable paths) { var result = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -331,7 +490,8 @@ private static void CollectInputSystems(LfConfigFieldList fieldList, string path /// Converts the list of LF input systems and adds them to Lcm writing systems /// /// List of LF input systems. - private void LfWsToLcmWs(Dictionary lfWsList) + private void LfWsToLcmWs(Dictionary lfWsList, + LfWritingSystemUsage usage = null) { // Between FW 8.2 and 9, a few classes and interfaces were renamed. The ones most relevant here are // IWritingSystemManager (interface was removed and replaced with the WritingSystemManager concrete class), @@ -359,7 +519,8 @@ private void LfWsToLcmWs(Dictionary lfWsList) // Which writing systems are vernacular is derived from the project's own config rather // than from languageCode alone; see ClassifyVernacularWritingSystems. - var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, ProjectRecord.LanguageCode); + var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, + ProjectRecord.LanguageCode, usage); ISet vernacularTags = classification.Vernacular; ISet analysisTags = classification.Analysis; if (classification.Unresolved.Count > 0) From a6a5810c45780789621b3bc3ab6e202c7dff6226 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Sep 2026 11:32:46 +0700 Subject: [PATCH 14/51] Take the example sentence, translation, note and literal meaning as evidence Only the lexeme, citation form, definition and gloss counted as fields whose role FieldWorks fixes. Four more do: an example sentence is vernacular-typed, and an example translation, an entry note and a literal meaning are analysis- typed. Counting the 2026-07-06 corpus for each field, the projects whose text there is exclusively one role against exclusively the other: example sentence 332 vernacular : 5 analysis example translation 1 vernacular : 320 analysis entry note 0 vernacular : 255 analysis literal meaning 4 vernacular : 136 analysis Taking them as evidence takes the projects with at least one unresolved writing system from 69 to 61, and still gets all 18 writing systems in the four FieldWorks repositories on hand right. The pronunciation field is deliberately left out although it looks vernacular at 114 projects to 16. That is 12% analysis, above the share a single writing system may hold in the other role and still count as playing one role alone, so projects evidently use the field for more than one thing and it cannot be trusted to settle anything. The field paths are now the constants LfWritingSystemUsage already defines, so the counting pass and the classification cannot drift apart on a spelling. Co-Authored-By: Claude Opus 5 --- .../VernacularClassificationTests.cs | 61 +++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 31 ++++++++-- 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 03896ec4..528bbcca 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -62,6 +62,17 @@ private static LfLexEntry EntryWithTextAt(string fieldPath, string tag) entry.Senses = new List { new LfSense { Definition = text } }; break; case LfWritingSystemUsage.Gloss: entry.Senses = new List { new LfSense { Gloss = text } }; break; + case LfWritingSystemUsage.ExampleSentence: + entry.Senses = new List { + new LfSense { Examples = new List { new LfExample { Sentence = text } } } }; + break; + case LfWritingSystemUsage.ExampleTranslation: + entry.Senses = new List { + new LfSense { Examples = new List { new LfExample { Translation = text } } } }; + break; + case LfWritingSystemUsage.Note: entry.Note = text; break; + case LfWritingSystemUsage.LiteralMeaning: entry.LiteralMeaning = text; break; + case "pronunciation": entry.Pronunciation = text; break; case "etymology": entry.Etymology = text; break; default: throw new ArgumentException("no test entry shape for " + fieldPath); } @@ -425,6 +436,56 @@ public void AnEmptyLexiconLeavesTheConfigInCharge() Assert.That(withoutData.Vernacular, Contains.Item("en")); } + /// + /// FieldWorks fixes the role of these fields, and the corpus agrees: counting the projects + /// whose text in each is exclusively one role, the example sentence runs 332 vernacular to 5 + /// analysis, and the translation, entry note and literal meaning run the other way. + /// + [TestCase("senses.fields.examples.fields.sentence", true)] + [TestCase("senses.fields.examples.fields.translation", false)] + [TestCase("note", false)] + [TestCase("literalMeaning", false)] + public void TheFieldsFieldWorksFixesTheRoleOfAreEvidenceToo(string fieldPath, bool isVernacular) + { + // Only the field under test carries any text for "xyz", so nothing else can decide it. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + (fieldPath, Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (fieldPath, "xyz", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Does.Not.Contain("xyz")); + if (isVernacular) + { + Assert.That(result.Vernacular, Contains.Item("xyz")); + Assert.That(result.Analysis, Does.Not.Contain("xyz")); + } + else + { + Assert.That(result.Analysis, Contains.Item("xyz")); + Assert.That(result.Vernacular, Does.Not.Contain("xyz")); + } + } + + /// + /// The pronunciation field is left out on purpose. It looks vernacular in the corpus at 114 + /// projects to 16, but 12% is above the share a single writing system may hold in the other + /// role and still count as one thing, so projects evidently use it for more than one. + /// + [Test] + public void ThePronunciationFieldIsNotEvidenceOfAnything() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("pronunciation", Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), ("pronunciation", "xyz", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Contains.Item("xyz")); + } + /// /// A writing system with nothing written in it and no company still has a language, and the /// other writing systems of that language have already been placed. diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index a6e4ed94..afecef10 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -208,12 +208,31 @@ private IEnumerable GetLexicon(ILfProject project) return Connection.GetRecords(project, MagicStrings.LfCollectionNameForLexicon); } - // The two field groups whose vernacular/analysis role is never in doubt. Everything else is - // classified per project by which of these its writing systems already appear in -- a fixed - // list cannot be right for every project: in the 2026-07-06 corpus 741 projects configure - // etymology with analysis writing systems and 378 with vernacular ones. - private static readonly string[] VernacularAnchorFields = { "lexeme", "citationForm" }; - private static readonly string[] AnalysisAnchorFields = { "senses.fields.definition", "senses.fields.gloss" }; + // The field groups whose vernacular/analysis role is never in doubt, because FieldWorks fixes + // it: a headword and an example sentence are vernacular, a gloss and a translation are not. + // Everything else is classified per project by which of these its writing systems appear in + // -- a fixed list cannot be right for every project: in the 2026-07-06 corpus 741 projects + // configure etymology with analysis writing systems and 378 with vernacular ones. + // + // Counting that corpus, for each field, the projects whose text there is exclusively + // vernacular against exclusively analysis: the example sentence is 332 to 5, the example + // translation 1 to 320, the entry note 0 to 255 and the literal meaning 4 to 136. + // + // The pronunciation field is deliberately NOT here. It looks vernacular at 114 to 16, but + // that is 12% analysis, above the MinorityShare a single writing system would have to stay + // under to count as one role alone, so projects evidently use it for more than one thing. + private static readonly string[] VernacularAnchorFields = { + LfWritingSystemUsage.Lexeme, + LfWritingSystemUsage.CitationForm, + LfWritingSystemUsage.ExampleSentence, + }; + private static readonly string[] AnalysisAnchorFields = { + LfWritingSystemUsage.Definition, + LfWritingSystemUsage.Gloss, + LfWritingSystemUsage.ExampleTranslation, + LfWritingSystemUsage.Note, + LfWritingSystemUsage.LiteralMeaning, + }; /// /// Work out which writing systems this project treats as vernacular and which as analysis, From 99315f02e52513c2cff9808d4f85040ab802236e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Sep 2026 12:37:28 +0700 Subject: [PATCH 15/51] Settle a writing system by the company it keeps The field loop takes its evidence as it stood before the loop began, so a field holding only writing systems the loop itself has settled looks like evidence of nothing, and everything in it goes back into doubt. A writing system settled by one custom field can therefore be dragged into doubt by another. A second pass settles those again from the company they keep: a custom field holding the vernacular and one writing system nothing has placed is being used for vernacular content, so the unplaced one is vernacular too. Only unanimous company counts, for the same reason a writing system in both anchors is no evidence of a field's role. This takes the projects with at least one unresolved writing system from 61 to 57, and still gets all 18 writing systems in the four FieldWorks repositories on hand right. The pass deliberately does not chain: the company is counted once, before any of it is settled, so a writing system whose only company was itself in doubt stays unresolved rather than inheriting a second-hand answer. Co-Authored-By: Claude Opus 5 --- .../VernacularClassificationTests.cs | 52 +++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 45 ++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 528bbcca..b39673f7 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -486,6 +486,58 @@ public void ThePronunciationFieldIsNotEvidenceOfAnything() Assert.That(result.Unresolved, Contains.Item("xyz")); } + /// + /// A writing system can be settled by one custom field and then dragged back into doubt by + /// another: the field loop takes its evidence as it stood before the loop began, so a field + /// holding only writing systems it has settled since looks like evidence of nothing. The + /// company the writing system keeps settles it again. + /// + [TestCase("kal", true)] + [TestCase("en", false)] + public void AWritingSystemDraggedIntoDoubtIsSettledByTheCompanyItKeeps(string companion, bool isVernacular) + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Settled", Field(companion, "abc")), + ("customField_entry_Doubtful", Field("abc", "xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Does.Not.Contain("abc"), + "abc shares a field with {0}, which is settled, so abc is settled too", companion); + Assert.That(result.Vernacular.Contains("abc"), Is.EqualTo(isVernacular)); + Assert.That(result.Analysis.Contains("abc"), Is.EqualTo(!isVernacular)); + } + + [Test] + public void AWritingSystemWhoseCompanyIsItselfUnsettledStaysUnresolved() + { + // xyz keeps company only with abc, and abc was in doubt when the company was counted. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Settled", Field("kal", "abc")), + ("customField_entry_Doubtful", Field("abc", "xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "xyz" })); + } + + [Test] + public void AWritingSystemWithNoCompanyAtAllStaysUnresolved() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Note", Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "xyz" })); + } + /// /// A writing system with nothing written in it and no company still has a language, and the /// other writing systems of that language have already been placed. diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index afecef10..d1e7699d 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -328,6 +328,7 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre } } + ResolveByCompanions(byPath, vernacular, analysis, unresolved); ResolveByLanguageAffinity(vernacular, analysis, unresolved); // Safety net for a config LfMerge cannot read: without this such a project would get no @@ -338,6 +339,50 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre return (vernacular, analysis, unresolved); } + /// + /// Settles what is left by the company a writing system keeps. A writing system nothing has + /// settled still shares its fields with others, and those may all have been settled: a + /// custom field holding the vernacular and one unplaced writing system is being used for + /// vernacular content, so the unplaced one is vernacular too. + /// + /// Only unanimous company counts. A field holding writing systems of both roles says + /// nothing, which is the same reason a writing system in both anchors is no evidence of a + /// field's role. + /// + private static void ResolveByCompanions(IDictionary> byPath, + ISet vernacular, ISet analysis, ISet unresolved) + { + if (unresolved.Count == 0) + return; + + // Unresolved writing systems sit in both sets, so they are in neither of these and + // cannot vote for each other. + var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase); + vernacularOnly.ExceptWith(analysis); + var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); + analysisOnly.ExceptWith(vernacular); + + foreach (string tag in unresolved.ToList()) + { + var companions = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var field in byPath) + { + if (field.Value.Contains(tag)) + companions.UnionWith(field.Value); + } + companions.Remove(tag); + bool withVernacular = companions.Any(vernacularOnly.Contains); + bool withAnalysis = companions.Any(analysisOnly.Contains); + if (withVernacular == withAnalysis) + continue; // No company, or company of both roles: still nothing to go on. + if (withVernacular) + analysis.Remove(tag); + else + vernacular.Remove(tag); + unresolved.Remove(tag); + } + } + /// /// Settles what is left by language. A writing system plays the same role as the others /// that spell the same language: a phonetic or audio spelling of the vernacular is From 323391a220b722ba81f78c9bf27247fe21d27831 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Sep 2026 09:00:44 +0700 Subject: [PATCH 16/51] Do not put an already-placed writing system in doubt The field loop's no-evidence branch put every writing system in a field into doubt, including ones something had already placed. Before the companions pass that cost nothing but a log line, since a doubtful writing system goes in both lists anyway. With it, it strips a role. Take a config whose lexeme offers [seh, en], whose gloss and definition offer [en], and whose general note offers [en], with nothing written in en yet. The anchors correctly put en in both lists. The note holds nothing exclusive to one role, so the loop also put en in doubt; the companions pass then found that the only single-role company en keeps is seh, in the lexeme field, and removed en from the analysis list. A new "en" writing system was created vernacular only, and English glosses and definitions lost their analysis writing system. A field with no evidence now puts in doubt only the writing systems nothing else has placed. One already placed keeps its place. In the 2026-07-06 corpus five writing systems were losing a role this way. The fix also stops flagging the 47 empty projects whose "en" is offered in every field, both anchors included, so the projects with a writing system in doubt go from 57 to 10. It still gets all 18 writing systems in the four FieldWorks repositories on hand right. Two tests described the old behaviour. AWritingSystemInBothAnchorsIsNotEvidence expected en itself to be in doubt; it now gives the etymology a second, unplaced writing system to keep making its point. The companions tests relied on a writing system being dragged back into doubt, which no longer happens, so they now list the doubtful field first -- fields are visited in config order -- and the "company is itself unsettled" case uses two writing systems with no other company at all. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 54 +++++++++++++------ .../ConvertMongoToLcmLexicon.cs | 14 +++-- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index b39673f7..da74463f 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -253,17 +253,20 @@ public void LexemeIsVernacularEvenWhenLanguageCodeIsAbsent() Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" })); } - /// A writing system in both anchors is no evidence of any other field's role. + /// + /// A writing system in both anchors is no evidence of any other field's role, so the Hebrew + /// sharing the etymology with it is in doubt. "en" itself is not: the anchors placed it. + /// [Test] public void AWritingSystemInBothAnchorsIsNotEvidence() { var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("en")), - ("etymology", Field("en"))); + ("etymology", Field("en", "hbo"))); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en"); - Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" })); - Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); - Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "en" }), - "etymology overlaps neither anchor exclusively, so its role is unresolved"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en", "hbo" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "hbo" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" }), + "etymology overlaps neither anchor exclusively, so the writing system it alone places is in doubt"); } /// @@ -487,18 +490,18 @@ public void ThePronunciationFieldIsNotEvidenceOfAnything() } /// - /// A writing system can be settled by one custom field and then dragged back into doubt by - /// another: the field loop takes its evidence as it stood before the loop began, so a field - /// holding only writing systems it has settled since looks like evidence of nothing. The - /// company the writing system keeps settles it again. + /// The field loop takes its evidence as it stood before the loop began, so a field it visits + /// BEFORE the one that settles a writing system sees no evidence and puts that writing system + /// in doubt. The company the writing system keeps settles it again. The doubtful field is + /// listed first on purpose: fields are visited in config order. /// [TestCase("kal", true)] [TestCase("en", false)] - public void AWritingSystemDraggedIntoDoubtIsSettledByTheCompanyItKeeps(string companion, bool isVernacular) + public void AWritingSystemInDoubtIsSettledByTheCompanyItKeeps(string companion, bool isVernacular) { var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), - ("customField_entry_Settled", Field(companion, "abc")), - ("customField_entry_Doubtful", Field("abc", "xyz"))); + ("customField_entry_Doubtful", Field("abc", "xyz")), + ("customField_entry_Settled", Field(companion, "abc"))); var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10)); @@ -511,18 +514,35 @@ public void AWritingSystemDraggedIntoDoubtIsSettledByTheCompanyItKeeps(string co } [Test] - public void AWritingSystemWhoseCompanyIsItselfUnsettledStaysUnresolved() + public void WritingSystemsWhoseOnlyCompanyIsEachOtherStayUnresolved() { - // xyz keeps company only with abc, and abc was in doubt when the company was counted. var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), - ("customField_entry_Settled", Field("kal", "abc")), ("customField_entry_Doubtful", Field("abc", "xyz"))); var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10)); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); - Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "xyz" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "abc", "xyz" })); + } + + /// + /// "en" is in both anchors, so it plays both roles. Turning up alone in a note, a field + /// that gives no evidence, must not put it in doubt: if it did, the company it keeps in the + /// lexeme field (the vernacular) would then strip it of its analysis role, and a new "en" + /// writing system would be created vernacular only. + /// + [Test] + public void AWritingSystemAlreadyPlacedIsNotPutInDoubtByAFieldWithNoEvidence() + { + var config = Config(("lexeme", Field("seh", "en")), ("senses.fields.gloss", Field("en")), + ("senses.fields.definition", Field("en")), ("senses.fields.generalNote", Field("en"))); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + + Assert.That(result.Unresolved, Is.Empty); + Assert.That(result.Vernacular, Contains.Item("en")); + Assert.That(result.Analysis, Contains.Item("en")); } [Test] diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index d1e7699d..90dc6fb6 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -321,10 +321,16 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre } else { - // Overlaps neither role: nothing says which role this field plays. - vernacular.UnionWith(undecided); - analysis.UnionWith(undecided); - unresolved.UnionWith(undecided); + // Overlaps neither role: nothing says which role this field plays. That puts in + // doubt only the writing systems nothing else has placed. One already placed keeps + // its place: "en" in both the lexeme and the gloss is not made doubtful by also + // turning up in a note, and if it were, the passes below could strip one of its + // roles on the strength of the company it keeps there. + var unplaced = undecided + .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag)).ToList(); + vernacular.UnionWith(unplaced); + analysis.UnionWith(unplaced); + unresolved.UnionWith(unplaced); } } From c66141f0922e25b7b87f47a65bfe166c02d4e45b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Sep 2026 09:04:37 +0700 Subject: [PATCH 17/51] Leave a single-string field alone when LF's text cannot be placed BestStringAndWsFromMultiText ended with a "last-ditch" fallback: take the first non-empty value whatever writing system it is in. Once 74e1684 gave it a fallback that already takes the first value in any writing system LCM has, the last-ditch branch was reached only when no key resolved at all, and then it always returned writing system 0. Building a string in writing system 0 throws ("The specified writing system code is invalid"), and the per-entry catch in RunConversion then abandons the entry half-written: every field converted before the throw is kept, and every field after it is skipped. Returning null instead, as the review suggested, would be worse. Every caller assigns the result straight into the LCM property, so null clears the field, deleting what LCM holds on account of text that could not be placed. So the field now keeps its current value, and the log says which writing systems' text had nowhere to go. Null, clearing the field, still means what it always meant: LF holds no text there. The callers pass the current value in. The picture caption had the same hole: a caption in no writing system LCM has came back as writing system 0 and threw in GetOrCreatePictureByGuid. It now starts the picture with an empty caption in the default analysis writing system, and SetMultiStringFrom fills in every alternative that can be placed. WsIdAndFirstNonEmptyString, which fed both, is gone. The new test fails without this: the entry is abandoned at the scientific name, so the sociolinguistics note converted after it never arrives. The etymology-source line changed here is compiled only for DBVERSION_7000068, which does not build against the current SIL.LCModel for reasons unrelated to this change (ILexEntry.EtymologyOA and ILexEtymology.Source are gone), so it is unverified. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 37 ++++++++ .../ConvertMongoToLcmLexicon.cs | 86 +++++++++++++------ .../LanguageForge/Model/LfMultiText.cs | 9 -- 3 files changed, 96 insertions(+), 36 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index f03733ad..72159104 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -134,6 +134,43 @@ public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWrit "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId); } + [Test] + public void SingleStringField_TextInNoKnownWritingSystem_KeepsTheLcmValueAndSyncsTheRest() + { + // Setup: LCM holds a scientific name, and then LF holds one only in a writing system LCM + // has never heard of. Building that into a string in writing system 0 threw, abandoning + // the rest of the entry half-written; returning null instead would clear the field and + // delete what LCM holds. Neither is acceptable. + const string original = "Homo sapiens"; + const string laterNote = "A note set after the scientific name"; + var data = new SampleData(); + BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument; + sense["scientificName"] = new BsonDocument { { "en", new BsonDocument { { "value", original } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + SutMongoToLcm.Run(_lfProj); + + sense["scientificName"] = new BsonDocument { + { "zzz-x-nonesuch", new BsonDocument { { "value", "Unplaceable" } } } }; + // Converted after the scientific name, so it only arrives if the entry is not abandoned. + sense["sociolinguisticsNote"] = new BsonDocument { + { "en", new BsonDocument { { "value", laterNote } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow.AddMinutes(1); + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + var entry = _cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)) as ILexEntry; + Assert.That(entry, Is.Not.Null); + ILexSense lcmSense = entry.SensesOS[0]; + Assert.That(lcmSense.ScientificName.Text, Is.EqualTo(original), + "text that cannot be placed must not replace or clear what LCM holds"); + Assert.That(lcmSense.SocioLinguisticsNote.get_String(_wsEn).Text, Is.EqualTo(laterNote), + "the rest of the entry should still have been converted"); + } + [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)] [TestCase(MixedCaseTag, MixedCaseId)] [TestCase("fr-Latn", "fr")] diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 90dc6fb6..63b72f8c 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -662,6 +662,11 @@ private void LfWsToLcmWs(Dictionary lfWsList, } } + /// + /// The best string for a single-string LCM field, and the writing system it is in. Null when + /// LF holds no text in any writing system LCM has -- which is not the same as holding no text + /// at all; see . + /// private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool isAnalysisField = true) { if (input == null) return null; @@ -683,31 +688,50 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool // differently from LCM. Falls back to the first value in any writing system LCM knows. KeyValuePair best = input.BestStringAndWsId( wsesToSearch.Select(ws => ws.Handle), ServiceLocator.WritingSystemFactory); - if (best.Value != null) - return new Tuple(best.Value, best.Key); - - // Last-ditch option: just grab the first non-empty string we can find - KeyValuePair kv = input.WsIdAndFirstNonEmptyString(Cache); - if (kv.Value == null) return null; -// Logger.Debug("Returning first non-empty TsString from {0} for writing system with ID {1}", -// kv.Value, kv.Key); - return new Tuple(kv.Value, kv.Key); + if (best.Value == null) + return null; + return new Tuple(best.Value, best.Key); } - private ITsString BestTsStringFromMultiText(LfMultiText input, bool isAnalysisField = true) + /// + /// The value to give a single-string LCM field, whose current value is + /// . + /// + /// Null, clearing the field, when LF holds no text for it. But when LF holds text and every + /// alternative is in a writing system LCM does not have, there is nowhere to put it, and the + /// field keeps its current value. Neither alternative will do: building the string in + /// writing system 0 throws, which abandons the rest of the entry half-written, and clearing + /// the field would delete what LCM holds on account of text that could not be placed. + /// + private ITsString BestTsStringFromMultiText(LfMultiText input, ITsString current, bool isAnalysisField = true) { Tuple stringAndWsId = BestStringAndWsFromMultiText(input, isAnalysisField); - if (stringAndWsId == null) - return null; - return ConvertMongoToLcmTsStrings.SpanStrToTsString(stringAndWsId.Item1, stringAndWsId.Item2, ServiceLocator.WritingSystemFactory); + if (stringAndWsId != null) + return ConvertMongoToLcmTsStrings.SpanStrToTsString(stringAndWsId.Item1, stringAndWsId.Item2, ServiceLocator.WritingSystemFactory); + return NothingToPlace(input) ? null : current; } - private string BestStringFromMultiText(LfMultiText input, bool isAnalysisField = true) + /// As , for a plain string field. + private string BestStringFromMultiText(LfMultiText input, string current, bool isAnalysisField = true) { Tuple stringAndWsId = BestStringAndWsFromMultiText(input, isAnalysisField); - if (stringAndWsId == null) - return null; - return stringAndWsId.Item1; + if (stringAndWsId != null) + return stringAndWsId.Item1; + return NothingToPlace(input) ? null : current; + } + + /// + /// Whether LF holds no text at all here. When it does hold text that nevertheless could not + /// be placed, says so in the log, since that text is not reaching FieldWorks. + /// + private bool NothingToPlace(LfMultiText input) + { + if (input == null || input.IsEmpty) + return true; + Logger.Warning("MongoToLcm: text in writing system(s) {0} has nowhere to go, since LCM has none " + + "of them; leaving the field as it was", + string.Join(", ", input.Where(kv => kv.Value != null && !kv.Value.IsEmpty).Select(kv => kv.Key))); + return false; } // This GetOrCreate() function takes an extra out parameter so we can correctly update @@ -1066,7 +1090,7 @@ private void LfExampleToLcmExample(LfExample lfExample, ILexSense owner) // LcmExample.Guid, // LcmExample.Hvo // ); - LcmExample.Reference = BestTsStringFromMultiText(lfExample.Reference); + LcmExample.Reference = BestTsStringFromMultiText(lfExample.Reference, LcmExample.Reference); ICmTranslation t = FindOrCreateTranslationByGuid(lfExample.TranslationGuid, LcmExample, _freeTranslationType); SetMultiStringFrom(t.Translation, lfExample.Translation); @@ -1092,9 +1116,17 @@ private void LfPictureToLcmPicture(LfPicture lfPicture, ILexSense owner) string caption = ""; if (lfPicture.Caption != null) { - KeyValuePair kv = lfPicture.Caption.WsIdAndFirstNonEmptyString(Cache); - captionWs = kv.Key; - caption = kv.Value; + // A caption in no writing system LCM has would come back as writing system 0, which + // cannot be built into a string; the picture then starts with an empty caption, and + // SetMultiStringFrom below fills in every alternative that can be placed. + KeyValuePair kv = lfPicture.Caption.BestStringAndWsId( + ServiceLocator.LanguageProject.AnalysisWritingSystems.Select(ws => ws.Handle), + ServiceLocator.WritingSystemFactory); + if (kv.Value != null) + { + captionWs = kv.Key; + caption = kv.Value; + } } // Lcm expects internal pictures in a certain path. If an external path already @@ -1162,19 +1194,19 @@ private void LfSenseToLcmSense(LfSense lfSense, ILexEntry owner) SetMultiStringFrom(LcmSense.PhonologyNote, lfSense.PhonologyNote); // LcmSense.ReversalEntriesRC = lfSense.ReversalEntries; // TODO: More complex than // that. Handle it correctly. Maybe. - LcmSense.ScientificName = BestTsStringFromMultiText(lfSense.ScientificName); + LcmSense.ScientificName = BestTsStringFromMultiText(lfSense.ScientificName, LcmSense.ScientificName); ListConverters[SemDomListCode].UpdatePossibilitiesFromStringArray(LcmSense.SemanticDomainsRC, lfSense.SemanticDomain); SetMultiStringFrom(LcmSense.SemanticsNote, lfSense.SemanticsNote); SetMultiStringFrom(LcmSense.Bibliography, lfSense.SenseBibliography); // lfSense.SenseId; // TODO: What do I do with this one? - LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue); + LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue, LcmSense.ImportResidue); SetMultiStringFrom(LcmSense.Restrictions, lfSense.SenseRestrictions); LcmSense.SenseTypeRA = ListConverters[SenseTypeListCode].FromStringField(lfSense.SenseType); SetMultiStringFrom(LcmSense.SocioLinguisticsNote, lfSense.SociolinguisticsNote); - LcmSense.Source = BestTsStringFromMultiText(lfSense.Source); + LcmSense.Source = BestTsStringFromMultiText(lfSense.Source, LcmSense.Source); LcmSense.StatusRA = ListConverters[StatusListCode].FromStringArrayFieldWithOneCase(lfSense.Status); ListConverters[UsageTypeListCode].UpdatePossibilitiesFromStringArray(LcmSense.UsageTypesRC, lfSense.Usages); @@ -1325,7 +1357,7 @@ private void SetEtymologyFields(ILexEntry LcmEntry, LfLexEntry lfEntry) SetMultiStringFrom(LcmEtymology.Gloss, lfEntry.EtymologyGloss); if (lfEntry.EtymologySource != null) #if DBVERSION_7000068 - LcmEtymology.Source = BestStringFromMultiText(lfEntry.EtymologySource); + LcmEtymology.Source = BestStringFromMultiText(lfEntry.EtymologySource, LcmEtymology.Source); #else SetMultiStringFrom(LcmEtymology.LanguageNotes, lfEntry.EtymologySource); #endif @@ -1367,8 +1399,8 @@ private void SetPronunciation(ILexEntry LcmEntry, LfLexEntry lfEntry) LcmPronunciation = GetInstance().Create(); LcmEntry.PronunciationsOS.Add(LcmPronunciation); } - LcmPronunciation.CVPattern = BestTsStringFromMultiText(lfEntry.CvPattern); - LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone); + LcmPronunciation.CVPattern = BestTsStringFromMultiText(lfEntry.CvPattern, LcmPronunciation.CVPattern); + LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone, LcmPronunciation.Tone); SetMultiStringFrom(LcmPronunciation.Form, lfEntry.Pronunciation); LcmPronunciation.LocationRA = (ICmLocation)ListConverters[LocationListCode].FromStringField(lfEntry.Location); diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 9dcde533..2da775df 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -112,15 +112,6 @@ public string FirstNonEmptyString() return (result == null) ? null : result.Value; } - public KeyValuePair WsIdAndFirstNonEmptyString(LcmCache cache) - { - KeyValuePair kv = FirstNonEmptyKeyValue(); - if (kv.Key == null) return new KeyValuePair(); - ILgWritingSystemFactory wsManager = cache.ServiceLocator.WritingSystemManager; - int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); - return new KeyValuePair(wsId, kv.Value); - } - /// /// The value for the first writing system in wsSearchOrder that has a non-empty one, falling /// back to the first non-empty value in any writing system LCM knows, together with that From af04bb8c9335eb96090281693f7347244ab44e4a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Sep 2026 09:07:06 +0700 Subject: [PATCH 18/51] Resolve an LF tag by its exact Id first, then by LanguageTag LCM knows a writing system by its Id, which is whatever its .ldml file is named, and also has its LanguageTag, which is always canonical. In older projects the two differ. brb-flex-2022 holds "km-Khmr-KH.ldml", whose identity is km plus KH, so LCM holds it as Id "km-Khmr-KH" with LanguageTag "km-KH"; so do spt-flex's "hi-Deva-IN" (hi-IN) and "spt-Tibt-x-tib" (spt-x-tib). LfMerge writes LF's multitext keys by Id and the project's input systems by LanguageTag, so LF can hand back either spelling. WsIdFromLfTag tried the canonical form first, which deletes data in the brb-flex-2022 layout. That project also holds a second, unused writing system whose Id is "km-KH". Text keyed by the real writing system's Id, "km-Khmr-KH", canonicalized to "km-KH" and went to the unused one, and WriteToLcm's clearing pass would then blank the real writing system's text, as missing from LF, in every entry LF sent back. (This branch has never run against that project, and LF has never edited it, so nothing has been lost.) It now tries, in turn: 1. The tag as LF spells it, as an Id -- the spelling LfMerge itself exports. 2. Its canonical form, as an Id. 3. Its canonical form as a LanguageTag, which finds spt-flex's "hi-Deva-IN" for the "hi-IN" in LF's config, where before it found nothing and the edit was dropped. Should several writing systems share the LanguageTag, the one created first wins. The two new tests reproduce those layouts and fail without this: the first resolves to the unused writing system, the second to nothing. Trying the exact Id first is never worse than develop, which only ever tried it. One consequence is worth knowing. cmo-khm-flex's LF entries still carry a few stale keys spelled "km-Khmr-kh", from before the project's Khmer writing system was renamed "km-KH"; they resolve, as they do on develop, to the unused "km-Khmr-KH" writing system LfMerge created in 2022 rather than to "km-KH". Sending them to the writing system in the project's lists instead needs the LanguageProject, which this function does not have, and needs a rule for which of two keys resolving to one writing system wins, which does not exist yet. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 45 ++++++++++++++++++ src/LfMerge.Core/LanguageTags.cs | 47 ++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 72159104..59eefcc0 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -10,6 +10,7 @@ using NUnit.Framework; using SIL.LCModel; using SIL.LCModel.Core.Text; +using SIL.LCModel.Core.WritingSystems; namespace LfMerge.Core.Tests.Lcm.DataConverters { @@ -181,6 +182,50 @@ public void WsIdFromLfTag_NonCanonicalTag_ResolvesToTheCanonicalWritingSystem(st Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, lfTag), Is.EqualTo(expected)); } + /// + /// Adds a writing system whose Id is spelled differently from its LanguageTag, the way LCM + /// loads an older .ldml file: brb-flex-2022's "km-Khmr-KH.ldml" has km plus KH for its + /// identity, so it is held as Id "km-Khmr-KH" with LanguageTag "km-KH". The repository + /// keeps an Id as given whenever it is equivalent to the LanguageTag. + /// + private CoreWritingSystemDefinition AddWritingSystemWithId(string id, string languageTag) + { + WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager; + CoreWritingSystemDefinition ws = wsManager.Create(languageTag); + ws.Id = id; + wsManager.Set(ws); + Assert.That(ws.Id, Is.EqualTo(id), "precondition: LCM should hold the writing system as {0}", id); + Assert.That(ws.LanguageTag, Is.EqualTo(languageTag)); + return ws; + } + + [Test] + public void WsIdFromLfTag_ExactIdWinsOverAnotherWritingSystemWithTheCanonicalId() + { + // The brb-flex-2022 layout: the real writing system under a non-canonical Id, and an + // unused second one whose Id is the canonical form. Text keyed by the real one's Id is + // the real one's text; resolving the canonical form first sent it to the other, and the + // clearing pass then deleted the real writing system's text as missing from LF. + CoreWritingSystemDefinition real = AddWritingSystemWithId("de-Latn-AT", "de-AT"); + CoreWritingSystemDefinition stray = _cache.ServiceLocator.WritingSystemManager.Set("de-AT"); + Assert.That(stray.Handle, Is.Not.EqualTo(real.Handle)); + + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "de-Latn-AT"), Is.EqualTo(real.Handle)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "de-AT"), Is.EqualTo(stray.Handle)); + } + + [Test] + public void WsIdFromLfTag_CanonicalTagFindsAWritingSystemWhoseIdIsSpelledOtherwise() + { + // The spt-flex layout: LCM holds "hi-Deva-IN" and LF's input systems call it "hi-IN", + // which is neither the Id nor its canonical form spelled as an Id. + CoreWritingSystemDefinition real = AddWritingSystemWithId("fr-Latn-CA", "fr-CA"); + Assert.That(_cache.WritingSystemFactory.GetWsFromStr("fr-CA"), Is.EqualTo(0), + "precondition: no writing system has the Id fr-CA"); + + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "fr-CA"), Is.EqualTo(real.Handle)); + } + [Test] public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() { diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs index ac792193..3d34d9a0 100644 --- a/src/LfMerge.Core/LanguageTags.cs +++ b/src/LfMerge.Core/LanguageTags.cs @@ -1,4 +1,7 @@ +using System; +using System.Linq; using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.WritingSystems; using SIL.WritingSystems; namespace LfMerge.Core @@ -42,9 +45,22 @@ public static string Canonical(string tag) /// /// Every place that resolves a tag LF supplied -- a multitext key, a span's lang attribute, /// a multi-paragraph's input system -- goes through here, so that a non-canonical spelling - /// resolves the same way everywhere. The canonical tag is tried first, since that is the id - /// LCM gives every writing system it creates; the tag as LF spells it is tried second, for a - /// writing system LCM somehow holds under a non-canonical id. + /// resolves the same way everywhere. + /// + /// LCM knows a writing system by its Id, which is whatever its .ldml file is named, and also + /// has its LanguageTag, which is always canonical. The two differ in older projects: + /// brb-flex-2022 holds "km-Khmr-KH.ldml", whose identity is km plus KH, so its Id is + /// "km-Khmr-KH" and its LanguageTag "km-KH". LfMerge writes LF's multitext keys by Id and the + /// project's input systems by LanguageTag, so LF can hand back either spelling. Tried in turn: + /// + /// 1. The tag as LF spells it, as an Id. This must come first. brb-flex-2022 also holds a + /// second, unused writing system whose Id is "km-KH"; trying the canonical form first + /// sent the text under "km-Khmr-KH" there, and WriteToLcm then cleared the real + /// writing system's text as missing from LF. + /// 2. Its canonical form, as an Id: the Id LCM gives any writing system it creates. + /// 3. Its canonical form as a LanguageTag, for an Id spelled some other way: + /// spt-flex's "hi-IN" is the writing system LCM holds as "hi-Deva-IN". Should several + /// writing systems share the LanguageTag, the one created first wins. /// public static int WsIdFromLfTag(ILgWritingSystemFactory wsf, string tag) { @@ -52,13 +68,30 @@ public static int WsIdFromLfTag(ILgWritingSystemFactory wsf, string tag) { return 0; } + int wsId = wsf.GetWsFromStr(tag); + if (wsId != 0) + { + return wsId; + } string canonical = Canonical(tag); - int wsId = wsf.GetWsFromStr(canonical); - if (wsId == 0 && canonical != tag) + if (!string.Equals(canonical, tag, StringComparison.OrdinalIgnoreCase)) { - wsId = wsf.GetWsFromStr(tag); + wsId = wsf.GetWsFromStr(canonical); + if (wsId != 0) + { + return wsId; + } + } + var wsManager = wsf as WritingSystemManager; + if (wsManager == null) + { + return 0; } - return wsId; + CoreWritingSystemDefinition byLanguageTag = wsManager.WritingSystems + .Where(ws => string.Equals(ws.LanguageTag, canonical, StringComparison.OrdinalIgnoreCase)) + .OrderBy(ws => ws.Handle) + .FirstOrDefault(); + return (byLanguageTag == null) ? 0 : byLanguageTag.Handle; } } } From 7d64415716c6a6a8c406c0e2b0aaecf97f711602 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Sep 2026 11:14:04 +0700 Subject: [PATCH 19/51] Find an input system's writing system before creating one LfWsToLcmWs looked each LF input system up with GetOrSet, which looks a tag up only as an Id, and created a writing system when that found nothing. LfMerge exports input systems by LanguageTag, while LCM knows a writing system by its Id, and in older projects the two differ: spt-flex's input system "hi-IN" is the writing system LCM holds as "hi-Deva-IN". On the next sync GetOrSet would not find it, would create a second writing system "hi-IN" and add it to the current lists. WsIdFromLfTag, which every multitext key goes through, then matches that duplicate exactly, so text typed into LF's hi-IN field goes to a writing system FieldWorks shows as new and empty rather than to hi-Deva-IN. The input system is now found the way every other LF tag is found, with WsIdFromLfTag, and a writing system is created only when that finds nothing. The new test fails without this: the duplicate appears among the project's writing systems. This was already the case on develop, which also used GetOrSet here. The LanguageTags comment claiming GetOrSet made the resolver unnecessary for LfWsToLcmWs was wrong, and is corrected. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 28 +++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 35 +++++++++++-------- src/LfMerge.Core/LanguageTags.cs | 6 ++-- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 59eefcc0..7f0103b1 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -226,6 +226,34 @@ public void WsIdFromLfTag_CanonicalTagFindsAWritingSystemWhoseIdIsSpelledOtherwi Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "fr-CA"), Is.EqualTo(real.Handle)); } + [Test] + public void LfWsToLcmWs_InputSystemSpelledAsTheLanguageTag_DoesNotDuplicateTheWritingSystem() + { + // The spt-flex layout: LCM holds "hi-Deva-IN", whose LanguageTag is "hi-IN", and LF's + // input systems -- which LfMerge exports by LanguageTag -- call it "hi-IN". GetOrSet looks + // only at Ids, so it created a second writing system "hi-IN" and put it in the lists. + CoreWritingSystemDefinition real = AddWritingSystemWithId("pt-Latn-AO", "pt-AO"); + Action restore = AddLfInputSystem("pt-AO"); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + Assert.That(LcmWritingSystemIds(), Does.Not.Contain("pt-AO"), + "no duplicate writing system should be created for the input system"); + Assert.That(_cache.ServiceLocator.WritingSystemManager.WritingSystems + .Count(ws => ws.LanguageTag == "pt-AO"), Is.EqualTo(1)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "pt-AO"), Is.EqualTo(real.Handle), + "LF text keyed \"pt-AO\" should go to the existing writing system"); + } + finally + { + restore(); + } + } + [Test] public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 63b72f8c..ab4cfc39 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -620,20 +620,27 @@ private void LfWsToLcmWs(Dictionary lfWsList, } */ - // GetOrSet() does the lookup and the creation in one step, and does both by the - // canonical form of the tag. TryGet() matched the id literally while Create() - // canonicalized, so a tag LCM canonicalizes -- "qaa-x-qaa-v" becomes "qaa-x-v", - // because the private-use section repeats the "qaa" language subtag, and "th-Thai" - // becomes "th", because Thai's suppress-script is dropped -- was not found, was then - // created under its canonical id, and collided with the writing system already there: - // Unable to set writing system 'qaa-x-v' because this id already exists. - // GetOrSet returns true when it found an existing writing system and false when it - // created one (creating adds it to the manager, so no Set() call is needed here). - // - // The rest of the TODO this replaces is not achievable this way: GetOrSet lives on - // WritingSystemManager, not on ILgWritingSystemFactory, so the FW 8-to-9 compatibility - // blocks above and below have to stay. - bool wsAlreadyExisted = wsManager.GetOrSet(lfWs.Tag, out ws); + // Find the writing system the way every other LF tag is found before creating one. + // LfMerge exports input systems by LanguageTag, while LCM knows a writing system by + // its Id, and in older projects the two differ: spt-flex's input system "hi-IN" is + // the writing system LCM holds as "hi-Deva-IN". GetOrSet looks a tag up only as an + // Id, so it did not find it, created a duplicate "hi-IN" and put it in the current + // lists; WsIdFromLfTag, which every multitext key goes through, then matched the + // duplicate exactly, and text typed in LF went to it rather than to hi-Deva-IN. + int existingHandle = LanguageTags.WsIdFromLfTag(wsManager, lfWs.Tag); + bool wsAlreadyExisted; + if (existingHandle != 0) + { + ws = wsManager.Get(existingHandle); + wsAlreadyExisted = true; + } + else + { + // Nothing to find, so create it. GetOrSet creates under the canonical form of the + // tag -- "qaa-x-qaa-v" becomes "qaa-x-v", "th-Thai" becomes "th" -- and adds the + // writing system to the manager, so no Set() call is needed here. + wsAlreadyExisted = wsManager.GetOrSet(lfWs.Tag, out ws); + } // The WS does check that a property has a different value before setting it // (and thus setting IsChanged flag), but for Abbreviation the WS returns diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs index 3d34d9a0..2963dc83 100644 --- a/src/LfMerge.Core/LanguageTags.cs +++ b/src/LfMerge.Core/LanguageTags.cs @@ -22,9 +22,9 @@ public static class LanguageTags /// and canonicalizing absorbs it /// th-Thai -> th Thai's suppress-script is dropped /// - /// LfWsToLcmWs does not need this -- GetOrSet canonicalizes for it -- but every other lookup - /// of an LF tag does, and should go through rather than calling - /// this and GetWsFromStr itself. + /// Every lookup of an LF tag, LfWsToLcmWs's included, should go through + /// rather than calling this and GetWsFromStr itself. GetOrSet is + /// no substitute: it looks a tag up only as an Id. /// /// /// The canonical tag, or the tag unchanged when it is empty or cannot be parsed -- From 9491facbd8f14159ad75871c39a59044a44bb0e6 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Sep 2026 11:15:24 +0700 Subject: [PATCH 20/51] Stop re-registering existing writing systems on every sync After applying an input system's abbreviation and direction, LfWsToLcmWs called wsManager.Replace(ws). ws is already the manager's own object, so the edits are in place without it, and Replace can only do nothing or do harm. It looks the writing system up by LanguageTag, which in older projects is not the Id LCM knows it by, and then: - If no writing system has that LanguageTag as its Id, Replace falls through to Set(ws), which gives the writing system a fresh handle mid-sync. That happens whenever an input system is spelled by an Id that is not its LanguageTag, as in projects LfMerge set up under FieldWorks 8, which exported input systems by Id: to judge from those exports, aeb-ra-flex holds "aeb-Arab" (LanguageTag "aeb") and nod-wlist-flex holds "th-Thai" (LanguageTag "th"). - If another writing system has it -- brb-flex-2022's unused "km-KH" beside the real "km-Khmr-KH" -- Replace removes that one from the manager and gives its handle to ws. The call is gone. The new test covers both layouts, and fails without this because the writing system's handle changes; the eviction in the second layout is from reading Replace, since the test stops at the handle. The test also checks that the input system's abbreviation is still applied. This was already the case on develop. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 41 +++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 7 +++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 7f0103b1..837bb23d 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -254,6 +254,47 @@ public void LfWsToLcmWs_InputSystemSpelledAsTheLanguageTag_DoesNotDuplicateTheWr } } + /// + /// An input system spelled by Id -- as every project LfMerge set up under FieldWorks 8 has + /// them -- for a writing system whose Id differs from its LanguageTag. Replace looked the + /// writing system up by LanguageTag, which either found nothing and re-registered it under a + /// fresh handle, or found the unused writing system with that LanguageTag as its Id and + /// evicted it, the real one taking over its handle. + /// + [TestCase("es-Latn-MX", "es-MX", false)] + [TestCase("it-Latn-CH", "it-CH", true)] + public void LfWsToLcmWs_ExistingWritingSystemWhoseIdIsNotItsLanguageTag_KeepsItsHandle( + string id, string languageTag, bool withUnusedNamesake) + { + WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager; + CoreWritingSystemDefinition real = AddWritingSystemWithId(id, languageTag); + int realHandle = real.Handle; + CoreWritingSystemDefinition namesake = withUnusedNamesake ? wsManager.Set(languageTag) : null; + int namesakeHandle = withUnusedNamesake ? namesake.Handle : 0; + Action restore = AddLfInputSystem(id); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + Assert.That(real.Handle, Is.EqualTo(realHandle), "the writing system should keep its handle"); + Assert.That(wsManager.GetWsFromStr(id), Is.EqualTo(realHandle)); + Assert.That(real.Abbreviation, Is.EqualTo(id), + "the input system's properties should still have been applied"); + if (withUnusedNamesake) + { + Assert.That(wsManager.GetWsFromStr(languageTag), Is.EqualTo(namesakeHandle), + "the other writing system with that LanguageTag should be left alone"); + } + } + finally + { + restore(); + } + } + [Test] public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index ab4cfc39..d92b3933 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -650,7 +650,12 @@ private void LfWsToLcmWs(Dictionary lfWsList, ws.Abbreviation = lfWs.Abbreviation; } ws.RightToLeftScript = lfWs.IsRightToLeft; - wsManager.Replace(ws); + // No wsManager.Replace(ws): ws is the manager's own object, so the edits above are + // already in place. Replace looks the writing system up by LanguageTag, and where + // the Id is spelled otherwise that lookup does harm. If it finds nothing, Replace + // re-registers ws under a fresh handle mid-sync; if it finds another writing system + // with that LanguageTag -- brb-flex-2022's unused "km-KH" beside "km-Khmr-KH" -- + // Replace evicts that one and gives ws its handle. if (!wsAlreadyExisted) { From 2377f89f22c5e74713b2872e964e84a94dc36adc Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Sep 2026 13:32:36 +0700 Subject: [PATCH 21/51] Spell input systems the way multitext keys are spelled LfMerge spelled a writing system two ways in Language Forge. Every multitext key comes from GetStrFromWs, which gives the writing system's Id. The project's input systems, its languageCode, the config's field lists at initial clone and the custom-field config came from its LanguageTag, which is always canonical, while the Id is whatever the .ldml file is named. In older projects the two differ: brb-flex-2022's Khmer is Id "km-Khmr-KH" with LanguageTag "km-KH", and che-flex's vernacular is Id "ce-la" with LanguageTag "ce-LA". LF's editor matches a field's input systems against an entry's keys exactly and case-sensitively, so it showed all of that text as empty: in the 2026-07-06 mongodump, twelve projects are affected, che-flex's 18,423 headwords among them. All of them are now spelled by GetStrFromWs too, through LfTagOf, so the input systems and the keys cannot disagree. FieldWorks 8 builds did the same, by Id; the three #if FW8_COMPAT blocks here existed only to choose between the two spellings, and FW8_COMPAT is never defined, so they are gone. Spelling by Id also agrees with LCM, which looks every writing system up by Id, so a key LF hands back resolves at the first step of WsIdFromLfTag. The project records already spelled by LanguageTag need re-spelling once, since LfMerge rewrites the input systems and custom-field config on every sync but built-in field config only at initial clone. That is done outside LfMerge, by a mongosh script. The new test fails without this: the input system is exported as "nl-BE" while the gloss is keyed "nl-Latn-BE". Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 36 ++++++++++++++++ .../ConvertLcmToMongoCustomField.cs | 11 +++-- .../ConvertLcmToMongoLexicon.cs | 43 ++++++++++--------- 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 837bb23d..e7741d91 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -9,6 +9,7 @@ using MongoDB.Bson; using NUnit.Framework; using SIL.LCModel; +using SIL.LCModel.Infrastructure; using SIL.LCModel.Core.Text; using SIL.LCModel.Core.WritingSystems; @@ -295,6 +296,41 @@ public void LfWsToLcmWs_ExistingWritingSystemWhoseIdIsNotItsLanguageTag_KeepsIts } } + /// + /// LF's editor matches a field's input systems against an entry's keys exactly, so the two + /// must be spelled alike. The keys are spelled by Id; the input systems were spelled by + /// LanguageTag, so brb-flex-2022's Khmer text, keyed "km-Khmr-KH" under an input system + /// called "km-KH", showed as empty. + /// + [Test] + public void LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEverywhere() + { + // Setup: an analysis writing system held under a non-canonical Id, with a gloss in it. + const string id = "nl-Latn-BE"; + const string gloss = "Nederlandse glos"; + CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, "nl-BE"); + Guid entryGuid = Guid.Parse(TestEntryGuidStr); + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + { + _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); + var lcmEntry = (ILexEntry)_cache.ServiceLocator.GetObject(entryGuid); + lcmEntry.SensesOS[0].Gloss.set_String(ws.Handle, gloss); + }); + + // Exercise + SutLcmToMongo.Run(_lfProj); + + // Verify + Dictionary inputSystems = _conn.GetInputSystems(_lfProj); + Assert.That(inputSystems.Keys, Contains.Item(id)); + Assert.That(inputSystems[id].Tag, Is.EqualTo(id)); + Assert.That(inputSystems.Keys, Does.Not.Contain("nl-BE")); + LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid); + Assert.That(lfEntry.Senses[0].Gloss.Keys, Contains.Item(id), + "the multitext key and the input system should be spelled alike"); + Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo(gloss)); + } + [Test] public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() { diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs index 3a736398..2aed584f 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -173,11 +173,10 @@ Dictionary lfCustomFieldTypes wsesForThisField = WritingSystemServices.GetWritingSystemList(cache, fieldWs, forceIncludeEnglish: false); break; } -#if FW8_COMPAT - IEnumerable inputSystems = wsesForThisField.Select(LcmWs => LcmWs.Id); -#else - IEnumerable inputSystems = wsesForThisField.Select(LcmWs => LcmWs.LanguageTag); -#endif + // Spelled as every multitext key is, by GetStrFromWs: LF matches a field's input + // systems against an entry's keys exactly. See ConvertLcmToMongoLexicon.LfTagOf. + IEnumerable inputSystems = wsesForThisField.Select( + LcmWs => servLoc.WritingSystemManager.GetStrFromWs(LcmWs.Handle)); // GetWritingSystemList returns all analysis WSes even when asked for just one, so if this // is a single-line custom field, trim the WSes down to just the first one if (lfCustomFieldType.StartsWith("Single")) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index 413bdd9a..5c1950f7 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -14,6 +14,7 @@ using MongoDB.Bson; using SIL.LCModel; using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.WritingSystems; using SIL.Progress; namespace LfMerge.Core.DataConverters @@ -65,15 +66,9 @@ public ConvertLcmToMongoLexicon(ILfProject lfProject, ILogger logger, IMongoConn // Reconcile writing systems from Lcm and Mongo Dictionary lfWsList = LcmWsToLfWs(); - #if FW8_COMPAT - List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(ws => ws.Id).ToList(); - List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(ws => ws.Id).ToList(); - List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(ws => ws.Id).ToList(); - #else - List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(ws => ws.LanguageTag).ToList(); - List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(ws => ws.LanguageTag).ToList(); - List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(ws => ws.LanguageTag).ToList(); - #endif + List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(LfTagOf).ToList(); + List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(LfTagOf).ToList(); + List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(LfTagOf).ToList(); Connection.SetInputSystems(LfProject, lfWsList, VernacularWss, AnalysisWss, PronunciationWss); ListConverters = new Dictionary(); @@ -589,6 +584,22 @@ private static string LcmPictureFilenameToLfPictureFilename(string LcmInternalFi /// Converts Lcm writing systems to LF input systems /// /// The list of LF input systems. + /// + /// How Language Forge spells an LCM writing system: exactly as every multitext key LfMerge + /// writes is spelled, since those come from GetStrFromWs too (see LfMultiText). That is the + /// writing system's Id, falling back to its LanguageTag should it have no Id. + /// + /// Input systems and the config's field lists must be spelled the same way, because LF's + /// editor matches a field's input systems against an entry's keys exactly and + /// case-sensitively. They were spelled by LanguageTag, which in older projects is not the + /// Id: brb-flex-2022's Khmer is Id "km-Khmr-KH" with LanguageTag "km-KH", and che-flex's + /// vernacular is Id "ce-la" with LanguageTag "ce-LA", so LF showed all of that text as empty. + /// + private string LfTagOf(CoreWritingSystemDefinition ws) + { + return ServiceLocator.WritingSystemManager.GetStrFromWs(ws.Handle); + } + private Dictionary LcmWsToLfWs() { // Using var here so that we'll stay compatible with both FW 8 and 9 (the type of these two lists changed between 8 and 9). @@ -605,20 +616,12 @@ private Dictionary LcmWsToLfWs() Abbreviation = LcmWs.Abbreviation, IsRightToLeft = LcmWs.RightToLeftScript, LanguageName = LcmWs.LanguageName, - #if FW8_COMPAT - Tag = LcmWs.Id, - #else - Tag = LcmWs.LanguageTag, - #endif + Tag = LfTagOf(LcmWs), VernacularWS = vernacularWSList.Contains(LcmWs), AnalysisWS = analysisWSList.Contains(LcmWs) }; - #if FW8_COMPAT - lfWsList[LcmWs.Id] = lfWs; - #else - lfWsList[LcmWs.LanguageTag] = lfWs; - #endif + lfWsList[lfWs.Tag] = lfWs; } return lfWsList; } From 3bde15d2f8c6e69b9d16b6f0c5d191b909e8b8dc Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 2 Oct 2026 11:12:12 +0700 Subject: [PATCH 22/51] Leave out a listed writing system that LCM does not have A project's current vernacular, analysis and pronunciation lists are stored as strings of Ids. If one names an Id LCM has no writing system for, the export threw KeyNotFoundException before writing anything to Language Forge: LINQ's Select and ToList read a list through its indexer or CopyTo, and liblcm's versions of both throw for such an Id. Only its enumerator tolerates one, by yielding null. The three lists are now read with foreach, and an Id with no writing system is left out with a warning. This makes the pronunciation list safe, which nothing else in the export reads. A vernacular list in that state would still fail, inside liblcm's own GetWritingSystemList when the custom-field config is written; FLEx would hit the same thing, so such a project is unlikely to exist. The new test fails without this, with the KeyNotFoundException above. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 17 ++++++++++ .../ConvertLcmToMongoLexicon.cs | 31 +++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index e7741d91..1b449eb8 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -331,6 +331,23 @@ public void LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEver Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo(gloss)); } + [Test] + public void LcmToMongo_PronunciationListNamingAWritingSystemLcmDoesNotHave_LeavesItOut() + { + // Setup: a pronunciation list naming an Id LCM has no writing system for. (The same in + // the vernacular list would still fail, inside liblcm's own GetWritingSystemList.) + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + _cache.LanguageProject.CurPronunWss = "zzz-x-nonesuch"); + + // Exercise + SutLcmToMongo.Run(_lfProj); + + // Verify + Dictionary inputSystems = _conn.GetInputSystems(_lfProj); + Assert.That(inputSystems.Keys, Contains.Item(_cache.LanguageProject.DefaultVernacularWritingSystem.Id)); + Assert.That(inputSystems.Keys, Does.Not.Contain("zzz-x-nonesuch")); + } + [Test] public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() { diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index 5c1950f7..97720c71 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -66,9 +66,9 @@ public ConvertLcmToMongoLexicon(ILfProject lfProject, ILogger logger, IMongoConn // Reconcile writing systems from Lcm and Mongo Dictionary lfWsList = LcmWsToLfWs(); - List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(LfTagOf).ToList(); - List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(LfTagOf).ToList(); - List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(LfTagOf).ToList(); + List VernacularWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentVernacularWritingSystems, "vernacular"); + List AnalysisWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems, "analysis"); + List PronunciationWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems, "pronunciation"); Connection.SetInputSystems(LfProject, lfWsList, VernacularWss, AnalysisWss, PronunciationWss); ListConverters = new Dictionary(); @@ -600,6 +600,31 @@ private string LfTagOf(CoreWritingSystemDefinition ws) return ServiceLocator.WritingSystemManager.GetStrFromWs(ws.Handle); } + /// + /// The LF tags of one of the project's current writing-system lists, leaving out any Id the + /// list names that LCM has no writing system for, rather than ending the transfer. + /// + /// The list is read with foreach on purpose. liblcm's enumerator yields null for such an + /// Id, but its indexer and CopyTo throw KeyNotFoundException, and LINQ's Select and ToList + /// use those on an IList. + /// + private List LfTagsOf(IEnumerable wss, string listName) + { + var tags = new List(); + int missing = 0; + foreach (CoreWritingSystemDefinition ws in wss) + { + if (ws == null) + missing++; + else + tags.Add(LfTagOf(ws)); + } + if (missing > 0) + Logger.Warning("LcmToMongo: the current {0} writing systems list names {1} writing system(s) that LCM does not have; leaving them out", + listName, missing); + return tags; + } + private Dictionary LcmWsToLfWs() { // Using var here so that we'll stay compatible with both FW 8 and 9 (the type of these two lists changed between 8 and 9). From c373e0370fe32486449e65b006e7bf5b9ed8dfb0 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 2 Oct 2026 11:13:12 +0700 Subject: [PATCH 23/51] Keep the language of a tag that is private-use from the start LanguageOf looked for the private-use "x" from the second subtag on, so a tag that begins with it, such as "x-kal", came out as just "x". Every such tag then counted as one language, and language affinity could place "x-hbo" in the same role as "x-kal". The 2026-10-01 corpus has two tags like this, "x-IPA" and "x-v". The search now starts at the first subtag, so "x-kal" comes out "x-kal" and "x-kal-dupl1" still matches it. The new test fails without this: "x-hbo" is classified with "x-kal" instead of being left unresolved. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 18 ++++++++++++++++++ .../DataConverters/ConvertMongoToLcmLexicon.cs | 5 +++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index da74463f..ca399afa 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -565,6 +565,7 @@ public void AWritingSystemWithNoCompanyAtAllStaysUnresolved() [TestCase("seh", "seh-fonipa-x-etic", true, TestName = "AffinityPhoneticVariantOfTheVernacular")] [TestCase("qaa-x-kal", "qaa-Zxxx-x-kal-audio", true, TestName = "AffinityAudioVariantOfTheVernacular")] [TestCase("qaa-x-kal", "qaa-x-kal-dupl1", true, TestName = "AffinityDuplicateOfTheVernacular")] + [TestCase("x-kal", "x-kal-dupl1", true, TestName = "AffinityDuplicateOfAWhollyPrivateUseVernacular")] public void AWritingSystemOfTheSameLanguageAsTheVernacularIsVernacular( string vernacularTag, string relative, bool isVernacular) { @@ -612,6 +613,23 @@ public void ADifferentPrivateUseLanguageIsNotTheSameLanguage() Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-x-hbo" })); } + /// + /// A tag can be private-use from its first subtag (the corpus has "x-IPA" and "x-v"), and + /// then too the private-use subtags are the language. + /// + [Test] + public void AWhollyPrivateUseTagIsNotTheSameLanguageAsAnother() + { + var config = Config(("lexeme", Field("x-kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field("x-hbo"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "x-kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "x-kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "x-hbo" })); + } + [Test] public void ALanguageUsedForBothRolesSettlesNothing() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index d92b3933..48a4cf95 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -445,7 +445,8 @@ private static void ResolveByLanguageAffinity(ISet vernacular, ISet private static string LanguageOf(string tag) { @@ -453,7 +454,7 @@ private static string LanguageOf(string tag) return tag; string[] parts = tag.Split('-'); var language = new List { parts[0] }; - int privateUse = Array.FindIndex(parts, 1, + int privateUse = Array.FindIndex(parts, part => part.Equals("x", StringComparison.OrdinalIgnoreCase)); if (privateUse >= 0) { From d61854495b454ad4041142710f46f292cd18f571 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Fri, 2 Oct 2026 11:14:46 +0700 Subject: [PATCH 24/51] Report a custom field LfMerge cannot write once per sync A custom field whose type the LF to LCM direction does not implement, such as an owning collection or sequence, logged its warning for every entry, sense or example that had it, so a project with 10,000 senses logged it 10,000 times on every sync. That is the same kind of log spam d90c0b9 removed. The converter now remembers which fields it has reported and warns about each once. One converter is made per transfer, so that is once per field per sync. The new test fails without this: three entries give three warnings. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmCustomFieldTests.cs | 21 +++++++++++++++++++ .../ConvertMongoToLcmCustomField.cs | 12 +++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs index 962268f5..3773e8e2 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs @@ -2,6 +2,7 @@ // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; +using System.Linq; using LfMerge.Core.DataConverters; using MongoDB.Bson; using NUnit.Framework; @@ -159,5 +160,25 @@ public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagInAStrin BsonDocument written = ReadBack(entry)[SingleStringField].AsBsonDocument; Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Texte")); } + + [Test] + public void SetCustomFieldData_FieldOfATypeItCannotWrite_IsReportedOncePerSyncNotOncePerObject() + { + // Setup: an owning sequence, a type the LF to LCM direction does not implement, on + // three entries. The test project has no custom field of that type, so a built-in + // field stands in for one; the writer does not tell them apart. + var logger = new TestLogger(TestContext.CurrentContext.Test.Name); + var converter = new ConvertMongoToLcmCustomField(_cache, _servLoc, logger, _wsEn); + var entries = _servLoc.GetInstance().AllInstances().Take(3).ToList(); + Assert.That(entries, Has.Count.EqualTo(3)); + + // Exercise + foreach (ILexEntry entry in entries) + Assert.That(converter.SetCustomFieldData(entry.Hvo, LexEntryTags.kflidSenses, new BsonString("LF data"), null), Is.False); + + // Verify + int reports = logger.Messages.Split('\n').Count(line => line.Contains("not written to LCM")); + Assert.That(reports, Is.EqualTo(1)); + } } } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index 6391176b..aa4c9ce0 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -25,6 +25,9 @@ public class ConvertMongoToLcmCustomField private IFwMetaDataCacheManaged lcmMetaData; private ILogger logger; private int wsEn; + // Fields already reported as not writable, so each is reported once per sync rather than + // once for every entry, sense or example that has it + private readonly HashSet unwritableFieldsReported = new HashSet(); public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn) { @@ -406,10 +409,11 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui } default: - logger.Warning( - "Custom field {0} not written to LCM: CellarPropertyType.{1} is not implemented " - + "for the LF to LCM direction, so any data LF holds in it is being dropped", - fieldName, fieldType.ToString()); + if (unwritableFieldsReported.Add(flid)) + logger.Warning( + "Custom field {0} not written to LCM: CellarPropertyType.{1} is not implemented " + + "for the LF to LCM direction, so any data LF holds in it is being dropped", + fieldName, fieldType.ToString()); return false; } } From 7b76f8fe8947fde42912f559d176f90deb65e4ac Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 10:33:48 +0700 Subject: [PATCH 25/51] Do not count a field offering both roles as vernacular A config field that is not an anchor is classified by the writing systems it offers that belong to one role only. A field offering a vernacular-only and an analysis-only writing system side by side took the vernacular branch, only because that branch was checked first, and the other writing systems it offered became vernacular. That is the opposite of the rule ResolveByCompanions follows, where only unanimous company counts. Such a field is now evidence of neither role. Writing systems it offers that nothing else has placed are left unresolved, as for a field overlapping neither role, and the companions and language-affinity passes then decide them where they can. In the 2026-10-01 corpus 166 projects have 1,381 fields like this, but only 4 projects and 5 writing systems change, measured with a simulation of this classifier. Compared with the roles FLEx gives them (the inputSystems record LfMerge exports): - kam-flex: "es" and "fr", in a wordlist field, were vernacular and are now both; FLEx has them as analysis. - mni-flex: "zh" was vernacular and is now both, as FLEx has it. - msea-thai-flex: "th-x-latin" was vernacular and is now both; FLEx has it as vernacular. - xin-flex: "xin", which LCM does not have yet, was vernacular and is now analysis, through language affinity with its analysis writing system "xin-GT-fonipa". It holds only pronunciation text, so vernacular was the better answer there; the project's writing systems look mislabelled. Only xin-flex is affected in practice: the others already exist in LCM, and classification only decides the lists for a writing system LCM does not have. The new test fails without this: "fr" is classified vernacular-only instead of left unresolved. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 17 +++++++++++ .../ConvertMongoToLcmLexicon.cs | 29 ++++++++++++------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index ca399afa..be6235f5 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -184,6 +184,23 @@ public void AVernacularFieldDoesNotMakeAnAnalysisOnlyWritingSystemVernacular() Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "pt" })); } + /// + /// A field offering a vernacular-only and an analysis-only writing system side by side says + /// nothing about the role of the others it offers. It used to count as vernacular, because + /// that was checked first, which made kam-flex's Spanish and French vernacular; FLEx has + /// both as analysis. + /// + [Test] + public void AFieldOfferingBothRolesIsNotEvidenceOfEither() + { + var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("en")), + ("customField_senses_Wordlist", Field("seh", "en", "fr"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "fr" })); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "fr" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "fr" })); + } + /// /// grc-vie-flex: etymologies in Hebrew and Aramaic, neither vernacular nor analysis. With no /// evidence either way they are both, so neither a vernacular-typed nor an analysis-typed diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 48a4cf95..b06d7caf 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -245,9 +245,10 @@ private IEnumerable GetLexicon(ILfProject project) /// and vice versa. A writing system present in both anchors -- "en" very often is -- is no /// evidence either way and is ignored for that purpose; it is both vernacular and analysis. /// - /// Evidence decides the role of the field, not of every writing system in it: a writing - /// system exclusive to the analysis anchors stays analysis-only even in a field classified - /// vernacular, so an etymology configured as [seh, pt] does not make Portuguese vernacular. + /// Evidence decides the role of the field, not of every writing system in it, and it has to + /// point one way: a field offering writing systems exclusive to each role, such as an + /// etymology configured as [seh, pt], is evidence of neither. So it does not make Portuguese + /// vernacular, nor anything else it offers that the anchors have not placed. /// /// Where the lexicon has text to show for a writing system, that text decides instead: a /// writing system holding nearly all of its text in the vernacular anchors is vernacular @@ -311,21 +312,27 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre var undecided = field.Value.Where(tag => !decidedByText.Contains(tag)).ToList(); if (undecided.Count == 0) continue; - if (field.Value.Any(vernacularOnly.Contains)) + bool withVernacular = field.Value.Any(vernacularOnly.Contains); + bool withAnalysis = field.Value.Any(analysisOnly.Contains); + if (withVernacular && !withAnalysis) { - vernacular.UnionWith(undecided.Where(tag => !analysisOnly.Contains(tag))); + vernacular.UnionWith(undecided); } - else if (field.Value.Any(analysisOnly.Contains)) + else if (withAnalysis && !withVernacular) { analysis.UnionWith(undecided); } else { - // Overlaps neither role: nothing says which role this field plays. That puts in - // doubt only the writing systems nothing else has placed. One already placed keeps - // its place: "en" in both the lexeme and the gloss is not made doubtful by also - // turning up in a note, and if it were, the passes below could strip one of its - // roles on the strength of the company it keeps there. + // Overlaps neither role, or both: nothing says which role this field plays. A field + // offering a vernacular-only and an analysis-only writing system side by side is + // no more evidence for one role than the other, just as only unanimous company + // counts in ResolveByCompanions. + // + // That puts in doubt only the writing systems nothing else has placed. One already + // placed keeps its place: "en" in both the lexeme and the gloss is not made + // doubtful by also turning up in a note, and if it were, the passes below could + // strip one of its roles on the strength of the company it keeps there. var unplaced = undecided .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag)).ToList(); vernacular.UnionWith(unplaced); From 26aaee830e6bbc437ad49ab9cdaa1b313f07fc18 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 11:46:54 +0700 Subject: [PATCH 26/51] Make a writing system with pronunciation text vernacular FieldWorks draws its pronunciation writing systems from the vernacular ones: liblcm's InitializePronunciationWritingSystems considers no others. But the classifier ignored the pronunciation field's text altogether, since the field is not an anchor, so a writing system used only for pronunciations got whatever role its company or language suggested. xin-flex's "xin", which holds 226 pronunciations and nothing else, became analysis-only after 7b76f8f. A writing system with no text in the anchor fields but some in the pronunciation field is now vernacular. That only adds the role, it does not settle it, so the config can still add analysis: the field stays out of the anchors, because 12% of the projects writing in it write analysis there, and a stray pronunciation string should not take an analysis role away. Measured with a simulation of this classifier on the 2026-10-01 corpus, against the roles FLEx gives the 564 writing systems LfMerge's export records: 516 agree, up from 515, and none gets worse. Projects with an unresolved writing system go from 10 to 7. 18 writing systems in 18 projects change, 8 of them in Send/Receive projects, nearly all phonetic ones becoming vernacular. In dtlat-flex, Swedish, configured for the gloss with a single pronunciation string, becomes both rather than vernacular only. This is the part of the four-set design measured best. The rest of it -- "both" only for a genuinely split writing system, with a tie-break for one offered in both anchor groups with no text -- lost five roles FLEx gives, and adding to FLEx's pronunciation list is left out, since LfMerge only reads it. ThePronunciationFieldIsNotEvidenceOfAnything asserted that pronunciation text leaves a writing system unresolved; it is now ThePronunciationFieldIsNotAnAnchor, for a writing system merely offered there. The two new tests fail without this. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 50 +++++++++++++++++-- .../ConvertMongoToLcmLexicon.cs | 17 ++++++- .../DataConverters/LfWritingSystemUsage.cs | 3 +- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index be6235f5..fbee577e 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -72,7 +72,7 @@ private static LfLexEntry EntryWithTextAt(string fieldPath, string tag) break; case LfWritingSystemUsage.Note: entry.Note = text; break; case LfWritingSystemUsage.LiteralMeaning: entry.LiteralMeaning = text; break; - case "pronunciation": entry.Pronunciation = text; break; + case LfWritingSystemUsage.Pronunciation: entry.Pronunciation = text; break; case "etymology": entry.Etymology = text; break; default: throw new ArgumentException("no test entry shape for " + fieldPath); } @@ -489,23 +489,63 @@ public void TheFieldsFieldWorksFixesTheRoleOfAreEvidenceToo(string fieldPath, bo } /// - /// The pronunciation field is left out on purpose. It looks vernacular in the corpus at 114 + /// The pronunciation field is not an anchor. It looks vernacular in the corpus at 114 /// projects to 16, but 12% is above the share a single writing system may hold in the other - /// role and still count as one thing, so projects evidently use it for more than one. + /// role and still count as one thing, so projects evidently use it for more than one. Being + /// offered there, with nothing written in it, therefore settles nothing. /// [Test] - public void ThePronunciationFieldIsNotEvidenceOfAnything() + public void ThePronunciationFieldIsNotAnAnchor() { var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), ("pronunciation", Field("xyz"))); var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), - (LfWritingSystemUsage.Gloss, "en", 10), ("pronunciation", "xyz", 10)); + (LfWritingSystemUsage.Gloss, "en", 10)); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); Assert.That(result.Unresolved, Contains.Item("xyz")); } + /// + /// But text in it makes a writing system vernacular, since FieldWorks draws its + /// pronunciation writing systems from the vernacular ones. xin-flex's "xin" holds 226 + /// pronunciations and nothing else. + /// + [Test] + public void PronunciationTextMakesAWritingSystemVernacular() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("pronunciation", Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "xyz", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Does.Not.Contain("xyz")); + Assert.That(result.Vernacular, Contains.Item("xyz")); + Assert.That(result.Analysis, Does.Not.Contain("xyz")); + } + + /// + /// Pronunciation text only adds the vernacular role; it does not take away an analysis role + /// the config gives. In dtlat-flex a single pronunciation string in Swedish would otherwise + /// make Swedish, configured for the gloss, vernacular only. + /// + [Test] + public void PronunciationTextDoesNotTakeAwayAnAnalysisRole() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en", "sv")), + ("pronunciation", Field("kal", "sv"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "sv", 1)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Contains.Item("sv")); + Assert.That(result.Analysis, Contains.Item("sv")); + } + /// /// The field loop takes its evidence as it stood before the loop began, so a field it visits /// BEFORE the one that settles a writing system sees no evidence and puts that writing system diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index b06d7caf..b1b31ecd 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -221,6 +221,7 @@ private IEnumerable GetLexicon(ILfProject project) // The pronunciation field is deliberately NOT here. It looks vernacular at 114 to 16, but // that is 12% analysis, above the MinorityShare a single writing system would have to stay // under to count as one role alone, so projects evidently use it for more than one thing. + // Its text still counts for something in ClassifyByText, though: see there. private static readonly string[] VernacularAnchorFields = { LfWritingSystemUsage.Lexeme, LfWritingSystemUsage.CitationForm, @@ -493,6 +494,14 @@ private static string LanguageOf(string tag) /// Classifies every writing system the lexicon has text for, by where that text sits. /// Adds them to and , and returns /// the ones it answered for. + /// + /// A writing system whose only text is in the pronunciation field is made vernacular, + /// because FieldWorks draws its pronunciation writing systems from the vernacular ones + /// (InitializePronunciationWritingSystems considers no others). That is not an answer, only + /// a role it must have, so the config can still add analysis: the field is not an anchor + /// (see VernacularAnchorFields), and a stray pronunciation string should not take a + /// writing system's analysis role away. In the 2026-10-01 corpus this gives xin-flex's + /// "xin", which holds 226 pronunciations and nothing else, the vernacular role it needs. /// private static ISet ClassifyByText(LfWritingSystemUsage usage, IDictionary> byPath, ISet vernacular, ISet analysis) @@ -508,6 +517,7 @@ private static ISet ClassifyByText(LfWritingSystemUsage usage, candidates.UnionWith(tags); foreach (string path in VernacularAnchorFields.Concat(AnalysisAnchorFields)) candidates.UnionWith(usage.TagsWithText(path)); + candidates.UnionWith(usage.TagsWithText(LfWritingSystemUsage.Pronunciation)); foreach (string tag in candidates) { @@ -515,7 +525,12 @@ private static ISet ClassifyByText(LfWritingSystemUsage usage, int analysisText = usage.NonEmptyCount(AnalysisAnchorFields, tag); int total = vernacularText + analysisText; if (total == 0) - continue; // No text in a field of known role, so its text says nothing. + { + // No text in a field of known role, so its text decides nothing. + if (usage.NonEmptyCount(LfWritingSystemUsage.Pronunciation, tag) > 0) + vernacular.Add(tag); + continue; + } double analysisShare = (double)analysisText / total; if (analysisShare <= MinorityShare) vernacular.Add(tag); diff --git a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs index 1a4e9a79..65b8a631 100644 --- a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs +++ b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs @@ -31,6 +31,7 @@ public class LfWritingSystemUsage public const string ExampleTranslation = "senses.fields.examples.fields.translation"; public const string Note = "note"; public const string LiteralMeaning = "literalMeaning"; + public const string Pronunciation = "pronunciation"; private readonly Dictionary> _countsByPath = new Dictionary>(); @@ -94,7 +95,7 @@ public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon) usage.Add("etymologyComment", entry.EtymologyComment); usage.Add("etymologyGloss", entry.EtymologyGloss); usage.Add("etymologySource", entry.EtymologySource); - usage.Add("pronunciation", entry.Pronunciation); + usage.Add(Pronunciation, entry.Pronunciation); usage.Add("summaryDefinition", entry.SummaryDefinition); usage.Add("tone", entry.Tone); if (entry.Senses == null) From 7ae3016ce99e9b45efc49a0b33983f22ec1857db Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 11:55:05 +0700 Subject: [PATCH 27/51] Count the lexicon only when a writing system is new to LCM Every Mongo to LCM transfer read the whole lexicon twice: once to count the text in each field for classifying writing systems, once for the entries themselves. The classification is only used to put a writing system LCM does not have yet into its lists, which after a project's first sync almost never happens, so on nearly every sync the first pass was thrown away. For a project like spt-flex that is 18,600 entries read for nothing, every time. LfWsToLcmWs is now given a way to count the lexicon rather than the count, and classifies when the first writing system new to LCM turns up. The classification depends only on LF's config and text, so creating writing systems before then cannot change it. With no new writing system the lexicon is not counted, and the notice about writing systems whose role could not be determined, which was logged on every sync, is not logged either. The counting pass also counted about thirty fields when classification reads only nine: the vernacular and analysis anchors and the pronunciation field. It now counts just those, so classification gives the same answers. The test double for Mongo now counts lexicon reads. The new test that a transfer with no new writing system reads the lexicon once fails without this: it reads it twice. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 35 +++++++++++ src/LfMerge.Core.Tests/TestDoubles.cs | 4 ++ .../ConvertMongoToLcmLexicon.cs | 61 ++++++++++++------- .../DataConverters/LfWritingSystemUsage.cs | 54 ++++------------ 4 files changed, 89 insertions(+), 65 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 1b449eb8..5ddb74e7 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -430,5 +430,40 @@ public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis() restore(); } } + + /// + /// Classifying a writing system counts the text in the whole lexicon, a second pass over + /// Mongo. Only a writing system new to LCM is classified, so with none the lexicon is read + /// once, for the entries themselves. + /// + [Test] + public void MongoToLcm_NoInputSystemNewToLcm_ReadsTheLexiconOnce() + { + int readsBefore = _conn.LexiconReads; + + SutMongoToLcm.Run(_lfProj); + + Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(1)); + } + + [Test] + public void MongoToLcm_InputSystemNewToLcm_CountsTheLexiconToo() + { + const string tag = "qaa-x-new"; + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag); + Action restore = AddLfInputSystem(tag); + try + { + int readsBefore = _conn.LexiconReads; + + SutMongoToLcm.Run(_lfProj); + + Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(2)); + } + finally + { + restore(); + } + } } } diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index e8667743..b6d73622 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -241,11 +241,15 @@ public IEnumerable GetRecords(ILfProject project, string c return GetRecords(project, collectionName).Where(filter.Compile()); } + /// How many times the lexicon has been read, each read being a pass over Mongo. + public int LexiconReads { get; private set; } + public IEnumerable GetRecords(ILfProject project, string collectionName) { switch (collectionName) { case MagicStrings.LfCollectionNameForLexicon: + LexiconReads++; return (IEnumerable)GetLfLexEntries(); case MagicStrings.LfCollectionNameForOptionLists: return (IEnumerable)GetLfOptionLists(); diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index b1b31ecd..415c11ba 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -139,15 +139,12 @@ public ConversionError RunConversion() // Logger.Debug("Running \"fake\" MtFComments, should see comments show up below:"); var entryObjectIdToGuidMappings = Connection.GetGuidsByObjectIdForCollection(LfProject, MagicStrings.LfCollectionNameForLexicon); EntryCounts.Reset(); - // Counting the lexicon before any writing system is created, because which list a new - // writing system belongs in depends on what it is actually used for. This is a second - // streaming pass over Mongo; GetLexicon yields from a cursor, so it does not all sit in - // memory at once. - LfWritingSystemUsage wsUsage = LfWritingSystemUsage.FromLexicon(GetLexicon(LfProject)); - - // Update writing systems from project config input systems. Won't commit till the end + // Update writing systems from project config input systems. Won't commit till the end. + // Which list a new writing system belongs in depends on what it is actually used for, so + // the lexicon is counted -- a second streaming pass over Mongo, which GetLexicon yields + // from a cursor -- but only if some input system is new to LCM. UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", Cache.ActionHandlerAccessor, () => - LfWsToLcmWs(ProjectRecord.InputSystems, wsUsage)); + LfWsToLcmWs(ProjectRecord.InputSystems, () => LfWritingSystemUsage.FromLexicon(GetLexicon(LfProject)))); // Set English ws handle again in case it changed _wsEn = ServiceLocator.WritingSystemFactory.GetWsFromStr("en"); @@ -583,8 +580,12 @@ private static void CollectInputSystems(LfConfigFieldList fieldList, string path /// Converts the list of LF input systems and adds them to Lcm writing systems /// /// List of LF input systems. + /// + /// Counts the text in the lexicon, for classifying a writing system new to LCM. Called only + /// if there is one, since it reads the whole lexicon. + /// private void LfWsToLcmWs(Dictionary lfWsList, - LfWritingSystemUsage usage = null) + Func countUsage = null) { // Between FW 8.2 and 9, a few classes and interfaces were renamed. The ones most relevant here are // IWritingSystemManager (interface was removed and replaced with the WritingSystemManager concrete class), @@ -610,18 +611,11 @@ private void LfWsToLcmWs(Dictionary lfWsList, return; } - // Which writing systems are vernacular is derived from the project's own config rather - // than from languageCode alone; see ClassifyVernacularWritingSystems. - var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, - ProjectRecord.LanguageCode, usage); - ISet vernacularTags = classification.Vernacular; - ISet analysisTags = classification.Analysis; - if (classification.Unresolved.Count > 0) - { - Logger.Notice("MongoToLcm: writing system(s) {0} appear only in config fields whose " - + "vernacular/analysis role could not be determined; treating them as both", - string.Join(", ", classification.Unresolved)); - } + // Worked out when the first writing system new to LCM turns up, and not at all if none + // does, which after a project's first sync is nearly always: only a new writing system is + // put in a list, and working out which ones counts the whole lexicon. It depends only on + // LF's config and text, so the writing systems created before then cannot change it. + (ISet Vernacular, ISet Analysis, ISet Unresolved)? classification = null; // TODO: Split the inside of this foreach() out into its own function foreach (var lfWs in lfWsList.Values) { @@ -688,15 +682,36 @@ private void LfWsToLcmWs(Dictionary lfWsList, // case, and one no field claims is analysis. Matching on the tag as LF spells it, // both sides being LF's own strings, so a non-canonical spelling still matches // itself. - bool isVernacular = vernacularTags.Contains(lfWs.Tag); + if (classification == null) + classification = ClassifyForNewWritingSystems(countUsage); + bool isVernacular = classification.Value.Vernacular.Contains(lfWs.Tag); if (isVernacular) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); - if (!isVernacular || analysisTags.Contains(lfWs.Tag)) + if (!isVernacular || classification.Value.Analysis.Contains(lfWs.Tag)) ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); } } } + /// + /// Which writing systems this project treats as vernacular and which as analysis, derived + /// from its own config and text rather than from languageCode alone; see + /// ClassifyVernacularWritingSystems. + /// + private (ISet Vernacular, ISet Analysis, ISet Unresolved) + ClassifyForNewWritingSystems(Func countUsage) + { + var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, + ProjectRecord.LanguageCode, countUsage?.Invoke()); + if (classification.Unresolved.Count > 0) + { + Logger.Notice("MongoToLcm: writing system(s) {0} appear only in config fields whose " + + "vernacular/analysis role could not be determined; treating them as both", + string.Join(", ", classification.Unresolved)); + } + return classification; + } + /// /// The best string for a single-string LCM field, and the writing system it is in. Null when /// LF holds no text in any writing system LCM has -- which is not the same as holding no text diff --git a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs index 65b8a631..62d213a6 100644 --- a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs +++ b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs @@ -70,9 +70,11 @@ public IEnumerable TagsWithText(string fieldPath) } /// - /// Counts the text in every built-in multitext field of every entry. Deleted entries are - /// skipped: they are tombstones Language Forge keeps so the deletion can be replayed, and - /// their text is not evidence of anything. + /// Counts the text in the fields classification consults: the vernacular and analysis + /// anchors and the pronunciation field. Counting every other field too would only lengthen + /// a pass over the whole lexicon for numbers nothing reads. Deleted entries are skipped: + /// they are tombstones Language Forge keeps so the deletion can be replayed, and their text + /// is not evidence of anything. /// public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon) { @@ -88,16 +90,7 @@ public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon) usage.Add(CitationForm, entry.CitationForm); usage.Add(Note, entry.Note); usage.Add(LiteralMeaning, entry.LiteralMeaning); - usage.Add("cvPattern", entry.CvPattern); - usage.Add("entryBibliography", entry.EntryBibliography); - usage.Add("entryRestrictions", entry.EntryRestrictions); - usage.Add("etymology", entry.Etymology); - usage.Add("etymologyComment", entry.EtymologyComment); - usage.Add("etymologyGloss", entry.EtymologyGloss); - usage.Add("etymologySource", entry.EtymologySource); usage.Add(Pronunciation, entry.Pronunciation); - usage.Add("summaryDefinition", entry.SummaryDefinition); - usage.Add("tone", entry.Tone); if (entry.Senses == null) continue; foreach (LfSense sense in entry.Senses) @@ -106,37 +99,14 @@ public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon) continue; usage.Add(Definition, sense.Definition); usage.Add(Gloss, sense.Gloss); - usage.Add("senses.fields.anthropologyNote", sense.AnthropologyNote); - usage.Add("senses.fields.discourseNote", sense.DiscourseNote); - usage.Add("senses.fields.encyclopedicNote", sense.EncyclopedicNote); - usage.Add("senses.fields.generalNote", sense.GeneralNote); - usage.Add("senses.fields.grammarNote", sense.GrammarNote); - usage.Add("senses.fields.phonologyNote", sense.PhonologyNote); - usage.Add("senses.fields.scientificName", sense.ScientificName); - usage.Add("senses.fields.semanticsNote", sense.SemanticsNote); - usage.Add("senses.fields.senseBibliography", sense.SenseBibliography); - usage.Add("senses.fields.senseImportResidue", sense.SenseImportResidue); - usage.Add("senses.fields.senseRestrictions", sense.SenseRestrictions); - usage.Add("senses.fields.sociolinguisticsNote", sense.SociolinguisticsNote); - usage.Add("senses.fields.source", sense.Source); - if (sense.Examples != null) - { - foreach (LfExample example in sense.Examples) - { - if (example == null) - continue; - usage.Add(ExampleSentence, example.Sentence); - usage.Add(ExampleTranslation, example.Translation); - usage.Add("senses.fields.examples.fields.reference", example.Reference); - } - } - if (sense.Pictures != null) + if (sense.Examples == null) + continue; + foreach (LfExample example in sense.Examples) { - foreach (LfPicture picture in sense.Pictures) - { - if (picture != null) - usage.Add("senses.fields.pictures", picture.Caption); - } + if (example == null) + continue; + usage.Add(ExampleSentence, example.Sentence); + usage.Add(ExampleTranslation, example.Translation); } } } From f0da10eb6cba06cf2ea0e40842d318c5b1cb331d Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 12:02:10 +0700 Subject: [PATCH 28/51] Classify config fields without regard to their order The field loop placed writing systems field by field, in the order the config lists its fields, and a field with no evidence of its role put in doubt only the writing systems not placed yet. So a writing system offered in such a field and in one that does have a role was placed when the field with a role came first, but put in doubt when the field of no role came first. Once in doubt, the companions and language-affinity passes could strip one of the roles the other field had given it: en-fonipa, offered in a vernacular field and an analysis field, ended up analysis only if a field of no role came before them, and both otherwise. Every field's evidence is now gathered first, and a field of no role puts in doubt only the writing systems that neither the anchors nor any field with a role places, wherever the fields come in the config. Measured with a simulation of this classifier on the 2026-10-01 corpus, under the config's own order, its reverse and 20 random orders of every project's fields: no project's answer depended on the order before this, and none does now, and in the config's own order nothing changes. So this is a fix for a config the corpus does not yet hold, and changes no project's result today. The new test fails without this when the field of no role is listed first. AWritingSystemInDoubtIsSettledByTheCompanyItKeeps still passes; its comment described the old field-by-field placing and is updated. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 33 ++++++++++++-- .../ConvertMongoToLcmLexicon.cs | 43 +++++++++++-------- 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index fbee577e..6f6f0764 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -547,10 +547,9 @@ public void PronunciationTextDoesNotTakeAwayAnAnalysisRole() } /// - /// The field loop takes its evidence as it stood before the loop began, so a field it visits - /// BEFORE the one that settles a writing system sees no evidence and puts that writing system - /// in doubt. The company the writing system keeps settles it again. The doubtful field is - /// listed first on purpose: fields are visited in config order. + /// A writing system offered both in a field that says nothing about its role and in one + /// whose company settles it is settled by the latter. The field of no role is listed first + /// on purpose: placing as each field came, it used to put the writing system in doubt. /// [TestCase("kal", true)] [TestCase("en", false)] @@ -570,6 +569,32 @@ public void AWritingSystemInDoubtIsSettledByTheCompanyItKeeps(string companion, Assert.That(result.Analysis.Contains("abc"), Is.EqualTo(!isVernacular)); } + /// + /// The answer must not depend on the order the config lists its fields in. Here en-fonipa is + /// offered in a vernacular field and an analysis field, so it plays both roles. Placing as + /// each field came, a field of no role listed before them put it in doubt, and language + /// affinity with "en" then stripped its vernacular role; listed after them, it did not. + /// + [TestCase(true, TestName = "TheFieldOrderDoesNotChangeTheAnswerWithTheNoRoleFieldFirst")] + [TestCase(false, TestName = "TheFieldOrderDoesNotChangeTheAnswerWithTheNoRoleFieldLast")] + public void TheFieldOrderDoesNotChangeTheAnswer(bool noRoleFieldFirst) + { + var noRole = ("customField_entry_NoRole", (LfConfigFieldBase)Field("en-fonipa")); + var fields = new List<(string, LfConfigFieldBase)> { + ("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Vernacular", Field("kal", "en-fonipa")), + ("customField_entry_Analysis", Field("en", "en-fonipa")), + }; + fields.Insert(noRoleFieldFirst ? 2 : fields.Count, noRole); + var config = Config(fields.ToArray()); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + + Assert.That(result.Vernacular, Contains.Item("en-fonipa")); + Assert.That(result.Analysis, Contains.Item("en-fonipa")); + Assert.That(result.Unresolved, Does.Not.Contain("en-fonipa")); + } + [Test] public void WritingSystemsWhoseOnlyCompanyIsEachOtherStayUnresolved() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 415c11ba..f1c3bfff 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -300,7 +300,15 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); analysisOnly.ExceptWith(vernacular); + // Every field's evidence is gathered before any writing system is placed by it, so that + // the answer does not depend on the order the config lists its fields in. Placing as + // each field came used to let a field with no evidence, listed before the one that + // placed a writing system, put that writing system in doubt, and the passes below could + // then strip one of the roles that field gave it. var anchors = new HashSet(VernacularAnchorFields.Concat(AnalysisAnchorFields)); + var inVernacularField = new HashSet(StringComparer.OrdinalIgnoreCase); + var inAnalysisField = new HashSet(StringComparer.OrdinalIgnoreCase); + var inFieldOfNoRole = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var field in byPath) { if (anchors.Contains(field.Key) || field.Value.Count == 0) @@ -308,36 +316,35 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre // A writing system its own text has spoken for is still evidence of what this field // is for, but nothing here may change the answer it was given. var undecided = field.Value.Where(tag => !decidedByText.Contains(tag)).ToList(); - if (undecided.Count == 0) - continue; bool withVernacular = field.Value.Any(vernacularOnly.Contains); bool withAnalysis = field.Value.Any(analysisOnly.Contains); if (withVernacular && !withAnalysis) - { - vernacular.UnionWith(undecided); - } + inVernacularField.UnionWith(undecided); else if (withAnalysis && !withVernacular) - { - analysis.UnionWith(undecided); - } + inAnalysisField.UnionWith(undecided); else { // Overlaps neither role, or both: nothing says which role this field plays. A field // offering a vernacular-only and an analysis-only writing system side by side is // no more evidence for one role than the other, just as only unanimous company // counts in ResolveByCompanions. - // - // That puts in doubt only the writing systems nothing else has placed. One already - // placed keeps its place: "en" in both the lexeme and the gloss is not made - // doubtful by also turning up in a note, and if it were, the passes below could - // strip one of its roles on the strength of the company it keeps there. - var unplaced = undecided - .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag)).ToList(); - vernacular.UnionWith(unplaced); - analysis.UnionWith(unplaced); - unresolved.UnionWith(unplaced); + inFieldOfNoRole.UnionWith(undecided); } } + // A field of no role puts in doubt only the writing systems nothing else places, neither + // the anchors nor a field that has a role. One already placed keeps its place: "en" in + // both the lexeme and the gloss is not made doubtful by also turning up in a note, and if + // it were, the passes below could strip one of its roles on the strength of the company + // it keeps there. + var doubtful = inFieldOfNoRole + .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag) + && !inVernacularField.Contains(tag) && !inAnalysisField.Contains(tag)) + .ToList(); + vernacular.UnionWith(inVernacularField); + analysis.UnionWith(inAnalysisField); + vernacular.UnionWith(doubtful); + analysis.UnionWith(doubtful); + unresolved.UnionWith(doubtful); ResolveByCompanions(byPath, vernacular, analysis, unresolved); ResolveByLanguageAffinity(vernacular, analysis, unresolved); From ef2c7fc688e6e8cbe9195e2c6db166186cd904d7 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 12:10:36 +0700 Subject: [PATCH 29/51] Test a writing system and its LanguageTag namesake round trip 9491fac's test checks the handles of a writing system whose Id is not its LanguageTag and of one whose Id is that LanguageTag, but nothing carried text in both through a sync: the xkk-flex-2022 layout, where re-registering the first evicted the second and took its handle. The new test gives each a gloss of its own and takes them through an export to LF, an LF edit of both, an import and a second export. Each must keep its handle and its own text in LCM and in LF. It uses ca-Latn-AD and ca-AD, whose LanguageTag is the same under both SLDR datasets; xkk-Khmr's is not, and the tests run on the older one. With the Replace call 9491fac removed put back, the test fails: the second export throws a NullReferenceException in LcmWsToLfWs. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 5ddb74e7..620450ec 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -296,6 +296,71 @@ public void LfWsToLcmWs_ExistingWritingSystemWhoseIdIsNotItsLanguageTag_KeepsIts } } + /// + /// The xkk-flex-2022 layout, carried all the way round: one writing system whose Id is not + /// its LanguageTag, and a second whose Id IS that LanguageTag, each with text of its own. + /// LfMerge used to look the first up by LanguageTag when re-registering it, evict the second + /// and take its handle, so the two writing systems' text could end up under one of them. + /// Each must keep its own text, in LCM and in LF, through an export, an LF edit and an + /// import. The pair is one whose LanguageTag is the same under both SLDR datasets; xkk's own + /// is not, and the tests run on the older one. + /// + [Test] + public void RoundTrip_WritingSystemAndItsLanguageTagNamesake_EachKeepsItsOwnText() + { + // Setup: both writing systems in the analysis list, each with a gloss + const string id = "ca-Latn-AD", namesakeId = "ca-AD"; + WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager; + CoreWritingSystemDefinition real = AddWritingSystemWithId(id, namesakeId); + CoreWritingSystemDefinition namesake = wsManager.Set(namesakeId); + int realHandle = real.Handle, namesakeHandle = namesake.Handle; + Assert.That(namesakeHandle, Is.Not.EqualTo(realHandle)); + Guid entryGuid = Guid.Parse(TestEntryGuidStr); + var lcmEntry = (ILexEntry)_cache.ServiceLocator.GetObject(entryGuid); + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + { + _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(real); + _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(namesake); + lcmEntry.SensesOS[0].Gloss.set_String(realHandle, "glossa andorrana"); + lcmEntry.SensesOS[0].Gloss.set_String(namesakeHandle, "glossa de l'homonima"); + }); + + // Exercise: export, edit both in LF, import, export again + SutLcmToMongo.Run(_lfProj); + LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid); + Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo("glossa andorrana")); + Assert.That(lfEntry.Senses[0].Gloss[namesakeId].Value, Is.EqualTo("glossa de l'homonima")); + lfEntry.Senses[0].Gloss[id] = LfStringField.FromString("glossa andorrana, corregida a LF"); + lfEntry.Senses[0].Gloss[namesakeId] = LfStringField.FromString("glossa de l'homonima, corregida a LF"); + lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(lfEntry); + var restores = new[] { AddLfInputSystem(id), AddLfInputSystem(namesakeId) }; + try + { + SutMongoToLcm.Run(_lfProj); + SutLcmToMongo.Run(_lfProj); + + // Verify: neither writing system took the other's place or its text + Assert.That(real.Handle, Is.EqualTo(realHandle)); + Assert.That(namesake.Handle, Is.EqualTo(namesakeHandle)); + Assert.That(wsManager.GetWsFromStr(id), Is.EqualTo(realHandle)); + Assert.That(wsManager.GetWsFromStr(namesakeId), Is.EqualTo(namesakeHandle)); + Assert.That(lcmEntry.SensesOS[0].Gloss.get_String(realHandle).Text, + Is.EqualTo("glossa andorrana, corregida a LF")); + Assert.That(lcmEntry.SensesOS[0].Gloss.get_String(namesakeHandle).Text, + Is.EqualTo("glossa de l'homonima, corregida a LF")); + lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid); + Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo("glossa andorrana, corregida a LF")); + Assert.That(lfEntry.Senses[0].Gloss[namesakeId].Value, Is.EqualTo("glossa de l'homonima, corregida a LF")); + Assert.That(_conn.GetInputSystems(_lfProj).Keys, Is.SupersetOf(new[] { id, namesakeId })); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } + /// /// LF's editor matches a field's input systems against an entry's keys exactly, so the two /// must be spelled alike. The keys are spelled by Id; the input systems were spelled by From 5392d8281c2b208b775ab106824085351a576db3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 12:17:01 +0700 Subject: [PATCH 30/51] Write the config's spelling when two keys name one writing system An LF multitext can hold two keys that resolve to the same LCM writing system. It happens when LF's config spells a writing system otherwise than LfMerge's export does: the export keys every value by the writing system's Id, an LF user edits under the config's spelling, and the export's value stays in the entry, hidden from LF's editor, until the entry is exported again -- which, once the edit is written to LCM and the dates match, may not be for a long time. In spt-flex, whose config says "hi-IN" for the Id "hi-Deva-IN", any entry edited in LF would hold both. WriteToLcm wrote the keys in document order, so the later one silently replaced the earlier. The keys are now resolved before anything is written, one is chosen for each writing system, and the other is reported: the built-in fields and the MultiUnicode custom fields both log a warning naming the two keys and the start of each one's text. The key chosen is the one spelled as the project's config spells the writing system, since that is what LF's editor shows and so the only one a user can have edited. Before the config is re-spelled by Id, that is the user's edit and not the hidden export; after, it is an edit made in place under the Id and not a key left over from before. Where both spellings or neither are in the config, it is the later key, as it always was: LF adds a new key after the existing ones. Preferring the key spelled as the Id would drop the user's edit in every project whose config has not been re-spelled. Preferring the value that differs from LCM would write the user's edit once, and then, on the entry's next sync, write the stale hidden value back over it. In the 2026-10-01 corpus only one multitext holds such a pair, a huz-flex lexeme with the same text under "huz-Cyrl" and "huz". (cmo-khm-flex's "km-Khmr-kh" and "km-KH" resolve to two writing systems, and xkk-flex-2022's "xkk" and "xkk-Khmr" are each a writing system's exact Id.) The new tests fail with either the Id spelling or the later key always preferred: three cases and two respectively. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 44 ++++++++++ .../ConvertMongoToLcmCustomField.cs | 13 ++- .../ConvertMongoToLcmLexicon.cs | 24 +++++- .../LanguageForge/Model/LfMultiText.cs | 82 +++++++++++++++++-- 4 files changed, 150 insertions(+), 13 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 620450ec..6d7025cf 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -296,6 +296,50 @@ public void LfWsToLcmWs_ExistingWritingSystemWhoseIdIsNotItsLanguageTag_KeepsIts } } + /// + /// Two LF keys naming one writing system: the export's, spelled as its Id, and one spelled + /// as its LanguageTag. Only one can be written, and it is the one under the spelling LF's + /// config uses, wherever it comes, since that is the one LF's editor shows; with neither + /// spelling configured, the later one. The other is reported. + /// + /// The export's key comes first in an entry, since an LF user's new key is added after it. + /// Before the config is re-spelled by Id, the user's edit is under the LanguageTag and the + /// export's value is hidden and stale; afterwards, an Id-spelled edit is made in place, and a + /// LanguageTag key left from before is what is stale. + /// + // A pair each, since writing systems a test adds stay in the fixture's project + [TestCase("languageTag", true, "sv-Latn-FI", "sv-FI", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelledByLanguageTag_WritesTheEdit")] + [TestCase("id", true, "da-Latn-DE", "da-DE", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelledById_WritesTheEditMadeInPlace")] + [TestCase("languageTag", false, "nb-Latn-NO", "nb-NO", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelling_WinsWhereverItComes")] + [TestCase("neither", true, "fi-Latn-SE", "fi-SE", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_NeitherConfigured_WritesTheLaterKey")] + public void WriteToLcm_TwoKeysForOneWritingSystem(string configured, bool idFirst, string id, string languageTag) + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, languageTag); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, languageTag), Is.EqualTo(ws.Handle), + "precondition: the LanguageTag spelling should resolve to the same writing system"); + var multiText = new LfMultiText(); + if (idFirst) + multiText.Add(id, LfStringField.FromString("under the Id")); + multiText.Add(languageTag, LfStringField.FromString("under the LanguageTag")); + if (!idFirst) + multiText.Add(id, LfStringField.FromString("under the Id")); + var configuredTags = new HashSet(StringComparer.Ordinal) { "en", "fr" }; + if (configured == "id") + configuredTags.Add(id); + else if (configured == "languageTag") + configuredTags.Add(languageTag); + string expected = configured == "id" ? id : configured == "languageTag" ? languageTag : (idFirst ? languageTag : id); + var gloss = ((ILexEntry)_cache.ServiceLocator.GetObject(Guid.Parse(TestEntryGuidStr))).SensesOS[0].Gloss; + var reported = new List<(string NotWritten, string Written)>(); + + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + multiText.WriteToLcmMultiString(gloss, _cache.WritingSystemFactory, configuredTags, + (notWritten, written) => reported.Add((notWritten, written)))); + + Assert.That(gloss.get_String(ws.Handle).Text, Is.EqualTo(expected == id ? "under the Id" : "under the LanguageTag")); + Assert.That(reported, Is.EqualTo(new[] { (expected == id ? languageTag : id, expected) })); + } + /// /// The xkk-flex-2022 layout, carried all the way round: one writing system whose Id is not /// its LanguageTag, and a second whose Id IS that LanguageTag, each with text of its own. diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index aa4c9ce0..022a4afe 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -28,14 +28,18 @@ public class ConvertMongoToLcmCustomField // Fields already reported as not writable, so each is reported once per sync rather than // once for every entry, sense or example that has it private readonly HashSet unwritableFieldsReported = new HashSet(); + // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm + private readonly ISet configuredTags; - public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn) + public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn, + ISet configuredTags = null) { this.cache = cache; this.servLoc = serviceLocator; this.lcmMetaData = (IFwMetaDataCacheManaged)cache.MetaDataCacheAccessor; this.logger = logger; this.wsEn = wsEn; + this.configuredTags = configuredTags; } public Guid ParseGuidOrDefault(string input) @@ -383,8 +387,13 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui wsId => data.get_MultiStringAlt(hvo, flid, wsId), (wsId, tss) => data.SetMultiStringAlt(hvo, flid, wsId, tss), servLoc.WritingSystemFactory, + configuredTags, tag => logger.Warning("Custom field {0}: skipping unidentified writing system {1}", - fieldName, tag)); + fieldName, tag), + (notWritten, written) => logger.Warning( + "Custom field {0}: keys \"{1}\" and \"{2}\" name the same writing system; wrote \"{2}\" ({3}), not \"{1}\" ({4})", + fieldName, notWritten, written, valueAsMultiText.Excerpt(written), + valueAsMultiText.Excerpt(notWritten))); } case CellarPropertyType.String: diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index f1c3bfff..c1f7d9ce 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -40,6 +40,8 @@ public class ConvertMongoToLcmLexicon private int _wsEn; private ConvertMongoToLcmCustomField _convertCustomField; + // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm + private ISet _configuredTags; // Entries where LF's copy differs from FieldWorks but no LF user has touched it since the // last sync, so FieldWorks holds the newer version (see EditedInLfSinceLastSync) @@ -149,7 +151,8 @@ public ConversionError RunConversion() // Set English ws handle again in case it changed _wsEn = ServiceLocator.WritingSystemFactory.GetWsFromStr("en"); - _convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn); + _configuredTags = ConfiguredTags(ProjectRecord.Config); + _convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn, _configuredTags); IEnumerable lexicon = GetLexicon(LfProject); _changedOnlyInFieldWorks = 0; @@ -562,6 +565,20 @@ private static ISet Union(IDictionary> byPath, IEnu return result; } + /// + /// Every writing-system spelling the config's fields use, exactly as spelled: LF's editor + /// matches them against an entry's keys exactly. + /// + private static ISet ConfiguredTags(LfProjectConfig config) + { + var byPath = new Dictionary>(); + CollectInputSystems((config == null) ? null : config.Entry, "", byPath); + var tags = new HashSet(StringComparer.Ordinal); + foreach (ISet fieldTags in byPath.Values) + tags.UnionWith(fieldTags); + return tags; + } + /// /// Record every config field's input systems, keyed by its dotted path as Mongo spells it, so /// nesting goes through "fields" ("senses.fields.examples.fields.sentence"). @@ -1365,7 +1382,10 @@ private void SetMultiStringFrom(IMultiStringAccessor dest, LfMultiText source) if (source == null) ClearMultiString(dest); else - source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager); + source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager, _configuredTags, + (notWritten, written) => Logger.Warning( + "MongoToLcm: keys \"{0}\" and \"{1}\" name the same writing system; wrote \"{1}\" ({2}), not \"{0}\" ({3})", + notWritten, written, source.Excerpt(written), source.Excerpt(notWritten))); } /// diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 2da775df..6bf56a7e 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -149,11 +149,13 @@ public KeyValuePair FirstNonEmptyKeyValue() new KeyValuePair(result.Key, result.Value.Value); } - public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager) + public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager, + ISet configuredTags = null, Action onKeyNotWritten = null) { if (dest == null) return false; - return WriteToLcm(dest.AvailableWritingSystemIds, dest.get_String, dest.set_String, wsManager); + return WriteToLcm(dest.AvailableWritingSystemIds, dest.get_String, dest.set_String, wsManager, + configuredTags: configuredTags, onKeyNotWritten: onKeyNotWritten); } /// @@ -171,29 +173,68 @@ public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFacto /// /// An alternative whose value has not changed is left alone, so it stays out of the .fwdata /// XML and out of the Mercurial commit. + /// + /// Two keys can name the same writing system, spelled two ways that both resolve to it, and + /// only one can be written. That happens when LF's config spells a writing system otherwise + /// than LfMerge's export: the export keys every value by the writing system's Id, an LF user + /// edits under the config's spelling, and the export's value stays in the entry, hidden + /// from the editor, until the entry is next exported -- which, the edit once written to LCM, + /// may be a long time. In spt-flex, whose config says "hi-IN" for the Id "hi-Deva-IN", an + /// edited entry holds both. The value to write is the one under the spelling the config + /// uses, since that is what LF's editor shows and so the one a user can have edited; the + /// other is the hidden export, or, after the config has been re-spelled, an edit made under + /// the old spelling that has already been written. Where both spellings or neither are in + /// the config, it is the later key, as it always was: LF adds a new key after the existing + /// ones. /// /// The writing systems LCM currently holds alternatives for. /// Reads LCM's alternative for a writing system. /// Writes LCM's alternative for a writing system. /// Resolves LF's keys to LCM writing systems. + /// + /// Every writing-system spelling the project's config uses, to choose between two keys + /// naming one writing system; null to go on key order alone. + /// /// Told each key that resolves to no writing system. + /// + /// Told each key left unwritten because another key names the same writing system, and the + /// key written instead. + /// /// Whether anything in LCM changed. public bool WriteToLcm(IEnumerable existingWsIds, Func getAlternative, Action setAlternative, ILgWritingSystemFactory wsManager, - Action onUnidentifiedTag = null) + ISet configuredTags = null, Action onUnidentifiedTag = null, + Action onKeyNotWritten = null) { - var wsIdsToClear = new HashSet(existingWsIds); - bool changed = false; - foreach (KeyValuePair kv in this) + // Each writing system's key, chosen before anything is written + var keyFor = new Dictionary(); + foreach (string key in Keys) { - int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); + int wsId = LanguageTags.WsIdFromLfTag(wsManager, key); if (wsId == 0) { - onUnidentifiedTag?.Invoke(kv.Key); + onUnidentifiedTag?.Invoke(key); continue; } + string other; + if (keyFor.TryGetValue(wsId, out other)) + { + string kept = PreferredKey(other, key, configuredTags); + onKeyNotWritten?.Invoke(kept == key ? other : key, kept); + keyFor[wsId] = kept; + } + else + keyFor[wsId] = key; + } + + var wsIdsToClear = new HashSet(existingWsIds); + bool changed = false; + foreach (KeyValuePair chosen in keyFor) + { + int wsId = chosen.Key; + LfStringField field = this[chosen.Value]; wsIdsToClear.Remove(wsId); - string text = (kv.Value == null) ? string.Empty : (kv.Value.Value ?? string.Empty); + string text = (field == null) ? string.Empty : (field.Value ?? string.Empty); ITsString newValue = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(text, wsId, wsManager); ITsString oldValue = getAlternative(wsId); // GetDiffsInTsStrings() returns null when there are no changes @@ -213,6 +254,29 @@ public bool WriteToLcm(IEnumerable existingWsIds, Func getA return changed; } + /// The start of one key's text, enough to find the entry it is in. + public string Excerpt(string key) + { + LfStringField field; + string text = (TryGetValue(key, out field) && field != null) ? (field.Value ?? string.Empty) : string.Empty; + return text.Length <= 40 ? text : text.Substring(0, 40) + "..."; + } + + /// + /// Of two keys naming one writing system, the one to write: the one spelled as the config + /// spells it, or else the later one. LF's editor matches spellings exactly, so this does too. + /// + private static string PreferredKey(string earlier, string later, ISet configuredTags) + { + if (configuredTags != null) + { + bool earlierConfigured = configuredTags.Contains(earlier); + if (earlierConfigured != configuredTags.Contains(later)) + return earlierConfigured ? earlier : later; + } + return later; + } + } } From 9b914f5887b60598a68715f5c880c0704f739a61 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 12:24:25 +0700 Subject: [PATCH 31/51] Look a new writing system's roles up under any spelling of its tag LfWsToLcmWs gives a writing system new to LCM the roles the classification found for it, looking its input system's tag up in the classified sets. Those sets spell each writing system as the config and the lexicon do, which need not be how the input system spells it. An LF user who adds the input system "qaa-x-qaa-new" and configures the lexeme field with "qaa-x-new" got a writing system that was not found among the vernacular ones and so was made analysis. The lookup now matches any tag with the same canonical form. That cannot merge two writing systems: this one is new to LCM, and had LCM held any writing system with that canonical form, WsIdFromLfTag would have found it. This is half of what was proposed for this finding. The other half -- counting text under every spelling of a writing system together -- was measured with a simulation of this classifier on the 2026-10-01 corpus and left out: against the roles FLEx gives the 564 writing systems LfMerge's export records, it made agreement worse, 516 to 515. ket-flex's "ket" took the ten analysis-field strings of "ket-Cyrl", which FLEx has as vernacular, and became analysis only where the config alone had made it both. The lookup on its own changes no writing system there. The new test fails without this: the writing system is made analysis only. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 35 +++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 26 +++++++++++--- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 6d7025cf..b7c4abc9 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -540,6 +540,41 @@ public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis() } } + /// + /// An input system LF spells non-canonically, while the config's fields spell it + /// canonically: the lexeme field says "qaa-x-fresh" for the input system "qaa-x-qaa-fresh". The + /// writing system is created under the canonical form, and must still get the vernacular + /// role the lexeme field gives it; looked up under the input system's spelling only, it + /// was not found, and was made analysis. + /// + [Test] + public void LfWsToLcmWs_NewInputSystemSpelledOtherwiseThanInTheConfig_GetsTheConfigsRole() + { + const string inputSystemTag = "qaa-x-qaa-fresh", configTag = "qaa-x-fresh"; + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(configTag), "testlangproj should not contain {0}", configTag); + var restores = new[] { + AddLfInputSystem(inputSystemTag), + AddToConfigField(configTag, "lexeme"), + }; + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + ILangProject langProj = _cache.LanguageProject; + Assert.That(LcmWritingSystemIds(), Contains.Item(configTag)); + Assert.That(langProj.CurrentVernacularWritingSystems.Select(ws => ws.Id), Contains.Item(configTag)); + Assert.That(langProj.CurrentAnalysisWritingSystems.Select(ws => ws.Id), Does.Not.Contain(configTag)); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } + /// /// Classifying a writing system counts the text in the whole lexicon, a second pass over /// Mongo. Only a writing system new to LCM is classified, so with none the lexicon is read diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index c1f7d9ce..898aef3f 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -703,20 +703,36 @@ private void LfWsToLcmWs(Dictionary lfWsList, // LF doesn't distinguish between vernacular/analysis WS, so the roles are worked // out per project from its config -- see ClassifyVernacularWritingSystems. A // writing system can play both, "en" in lexeme and gloss alike being the usual - // case, and one no field claims is analysis. Matching on the tag as LF spells it, - // both sides being LF's own strings, so a non-canonical spelling still matches - // itself. + // case, and one no field claims is analysis. The input system, the config and + // the lexicon need not spell the writing system alike, so the roles are looked + // up under any spelling of it; see HasRole. if (classification == null) classification = ClassifyForNewWritingSystems(countUsage); - bool isVernacular = classification.Value.Vernacular.Contains(lfWs.Tag); + bool isVernacular = HasRole(classification.Value.Vernacular, lfWs.Tag); if (isVernacular) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); - if (!isVernacular || classification.Value.Analysis.Contains(lfWs.Tag)) + if (!isVernacular || HasRole(classification.Value.Analysis, lfWs.Tag)) ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); } } } + /// + /// Whether a writing system new to LCM plays this role, under any spelling of its tag. The + /// classification spells each writing system as the config and the lexicon do, and the + /// input system can spell it otherwise: an LF user who adds "qaa-x-qaa-new" may configure + /// fields with "qaa-x-new". Spellings sharing a canonical form can be taken for one writing + /// system here, because this one is new: had LCM held any writing system with that canonical + /// form, WsIdFromLfTag would have found it. + /// + private static bool HasRole(ISet role, string tag) + { + if (role.Contains(tag)) + return true; + string canonical = LanguageTags.Canonical(tag); + return role.Any(other => string.Equals(LanguageTags.Canonical(other), canonical, StringComparison.OrdinalIgnoreCase)); + } + /// /// Which writing systems this project treats as vernacular and which as analysis, derived /// from its own config and text rather than from languageCode alone; see From d4fc7ac7f229884070e78001ce2f484fb90f988c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 12:25:47 +0700 Subject: [PATCH 32/51] Derive the writing systems of one role only in one place The field loop, ResolveByCompanions and ResolveByLanguageAffinity each worked out which writing systems are vernacular and not analysis, and analysis and not vernacular -- the first two with the same four lines, the third with two loops -- since each pass weighs evidence after the passes before it have changed the sets. ExclusiveRoles now does it for all three, so a change to how it is done, such as to the case-insensitivity, cannot reach one copy and miss the others. No behaviour changes; the tests are unchanged and pass. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmLexicon.cs | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 898aef3f..9856c6fd 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -298,10 +298,7 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre analysis.UnionWith(analysisAnchor.Where(tag => !decidedByText.Contains(tag))); // Only a writing system exclusive to one role is evidence of a field's role. - var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase); - vernacularOnly.ExceptWith(analysis); - var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); - analysisOnly.ExceptWith(vernacular); + var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis); // Every field's evidence is gathered before any writing system is placed by it, so that // the answer does not depend on the order the config lists its fields in. Placing as @@ -360,6 +357,21 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre return (vernacular, analysis, unresolved); } + /// + /// The writing systems that play one role only: those vernacular and not analysis, and those + /// analysis and not vernacular. Each pass that weighs evidence recomputes them, since the + /// passes before it change the two sets. + /// + private static (HashSet VernacularOnly, HashSet AnalysisOnly) ExclusiveRoles( + ISet vernacular, ISet analysis) + { + var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase); + vernacularOnly.ExceptWith(analysis); + var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); + analysisOnly.ExceptWith(vernacular); + return (vernacularOnly, analysisOnly); + } + /// /// Settles what is left by the company a writing system keeps. A writing system nothing has /// settled still shares its fields with others, and those may all have been settled: a @@ -378,10 +390,7 @@ private static void ResolveByCompanions(IDictionary> byPath // Unresolved writing systems sit in both sets, so they are in neither of these and // cannot vote for each other. - var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase); - vernacularOnly.ExceptWith(analysis); - var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); - analysisOnly.ExceptWith(vernacular); + var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis); foreach (string tag in unresolved.ToList()) { @@ -417,18 +426,9 @@ private static void ResolveByLanguageAffinity(ISet vernacular, ISet(StringComparer.OrdinalIgnoreCase); - var analysisLanguages = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (string tag in vernacular) - { - if (!analysis.Contains(tag)) - vernacularLanguages.Add(LanguageOf(tag)); - } - foreach (string tag in analysis) - { - if (!vernacular.Contains(tag)) - analysisLanguages.Add(LanguageOf(tag)); - } + var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis); + var vernacularLanguages = new HashSet(vernacularOnly.Select(LanguageOf), StringComparer.OrdinalIgnoreCase); + var analysisLanguages = new HashSet(analysisOnly.Select(LanguageOf), StringComparer.OrdinalIgnoreCase); foreach (string tag in unresolved.ToList()) { From ce389f3093b44e3ecdbbadab78b2600917fb5279 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 13:15:18 +0700 Subject: [PATCH 33/51] Warn when a built-in field's text names no writing system LCM has SetMultiStringFrom, which writes every built-in multitext field, gave WriteToLcm no callback for a key that resolves to no LCM writing system, so the text under it was skipped without a word. It never reaches FieldWorks, and it leaves LF as well once FieldWorks next changes the entry and the export rewrites the field from LCM. The custom fields and the single-string fields already logged this case; the commonest path did not. It now logs a warning naming the key and the start of its text, so the entry can be found. WriteToLcmMultiString takes the callback, as WriteToLcm does. In the 2026-10-01 corpus this is rare: of the keys outside each project's inputSystems record, all but 6 were written by LfMerge's own export and so name writing systems LCM has, just outside its vernacular and analysis lists. The 6 hold 13 values, all in entries LF users edited; one, in spa-par-flex, would be written on its next sync. The new test fails without this: nothing is logged. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 23 ++++++++++++++++++- .../ConvertMongoToLcmLexicon.cs | 5 ++++ .../LanguageForge/Model/LfMultiText.cs | 5 ++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index b7c4abc9..8eb0edcc 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -136,6 +136,27 @@ public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWrit "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId); } + /// + /// Text in a built-in multitext field under a key that names no writing system LCM has has + /// nowhere to go. It is skipped, as it always was, but no longer silently: the custom fields + /// and the single-string fields already said so, and this, the commonest path, did not. + /// + [Test] + public void SetMultiStringFrom_KeyNamingNoWritingSystem_IsSkippedWithAWarning() + { + const string tag = "zzz-x-nonesuch"; + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, tag), Is.EqualTo(0)); + var data = new SampleData(); + data.bsonTestData["senses"][0]["definition"][tag] = new BsonDocument { { "value", "a definition with nowhere to go" } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + SutMongoToLcm.Run(_lfProj); + + Assert.That(_env.Logger.GetMessages(), + Does.Contain("MongoToLcm: skipping text under \"zzz-x-nonesuch\" (a definition with nowhere to go)")); + } + [Test] public void SingleStringField_TextInNoKnownWritingSystem_KeepsTheLcmValueAndSyncsTheRest() { @@ -334,7 +355,7 @@ public void WriteToLcm_TwoKeysForOneWritingSystem(string configured, bool idFirs UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => multiText.WriteToLcmMultiString(gloss, _cache.WritingSystemFactory, configuredTags, - (notWritten, written) => reported.Add((notWritten, written)))); + onKeyNotWritten: (notWritten, written) => reported.Add((notWritten, written)))); Assert.That(gloss.get_String(ws.Handle).Text, Is.EqualTo(expected == id ? "under the Id" : "under the LanguageTag")); Assert.That(reported, Is.EqualTo(new[] { (expected == id ? languageTag : id, expected) })); diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 9856c6fd..114f79be 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -1399,6 +1399,11 @@ private void SetMultiStringFrom(IMultiStringAccessor dest, LfMultiText source) ClearMultiString(dest); else source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager, _configuredTags, + // Text under a key that names no writing system LCM has never reaches FieldWorks, + // and leaves LF too once FieldWorks next changes the entry and it is exported again + tag => Logger.Warning( + "MongoToLcm: skipping text under \"{0}\" ({1}), which names no writing system LCM has", + tag, source.Excerpt(tag)), (notWritten, written) => Logger.Warning( "MongoToLcm: keys \"{0}\" and \"{1}\" name the same writing system; wrote \"{1}\" ({2}), not \"{0}\" ({3})", notWritten, written, source.Excerpt(written), source.Excerpt(notWritten))); diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 6bf56a7e..e024008d 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -150,12 +150,13 @@ public KeyValuePair FirstNonEmptyKeyValue() } public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager, - ISet configuredTags = null, Action onKeyNotWritten = null) + ISet configuredTags = null, Action onUnidentifiedTag = null, + Action onKeyNotWritten = null) { if (dest == null) return false; return WriteToLcm(dest.AvailableWritingSystemIds, dest.get_String, dest.set_String, wsManager, - configuredTags: configuredTags, onKeyNotWritten: onKeyNotWritten); + configuredTags, onUnidentifiedTag, onKeyNotWritten); } /// From 7dde0d8d6bd5134095db802c609180f14b3f57d7 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 15:15:32 +0700 Subject: [PATCH 34/51] Say what the pronunciation rule does to a custom-field writing system ClassifyByText's comment said the rule that makes a writing system with only pronunciation text vernacular only added that role, so that the config could still add analysis, and that a stray pronunciation string should not take an analysis role away. That holds where the config offers the writing system in an anchor field. It does not where the config offers it only in custom fields: there, once placed, it is not put in doubt by a field of no role, and a custom field it alone is offered in is taken for a vernacular one, so it is vernacular only. A code review found the mismatch. The behaviour is kept and the comment now says what it is. In the 2026-10-01 corpus all 57 writing systems with pronunciation text and no anchor text are pronunciation writing systems -- phonetic, IPA and audio ones, and xin-flex's "xin" -- and FLEx has as vernacular each of the 11 whose role LfMerge's export records. The only stray strings, Swedish in dtlat-flex and English in two LF-only test projects, are in writing systems the config offers for an anchor field, so they are both. Applying the rule after the other passes instead, as the comment implied, was simulated: it agreed with FLEx no better, and turned 11 phonetic writing systems from vernacular only into both. The new test records that a writing system offered only in a custom field, with pronunciation text and nothing else, is vernacular only. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 23 +++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 16 +++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 6f6f0764..babd2b05 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -527,6 +527,29 @@ public void PronunciationTextMakesAWritingSystemVernacular() Assert.That(result.Analysis, Does.Not.Contain("xyz")); } + /// + /// A writing system offered only in a custom field, with pronunciation text and nothing + /// else, is vernacular only, by design. Without the pronunciation text the custom field + /// would say nothing about it and it would be both; with it, it counts as placed, so that + /// field cannot put it in doubt. In the 2026-10-01 corpus every writing system in that + /// position is a pronunciation writing system -- a phonetic, IPA or audio one -- for which + /// vernacular only is the right answer. + /// + [Test] + public void PronunciationTextAloneMakesAWritingSystemOfferedOnlyInACustomFieldVernacularOnly() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_senses_Phonetic", Field("kal-fonipa"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "kal-fonipa", 1)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Contains.Item("kal-fonipa")); + Assert.That(result.Analysis, Does.Not.Contain("kal-fonipa")); + Assert.That(result.Unresolved, Does.Not.Contain("kal-fonipa")); + } + /// /// Pronunciation text only adds the vernacular role; it does not take away an analysis role /// the config gives. In dtlat-flex a single pronunciation string in Swedish would otherwise diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 114f79be..ce44e448 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -504,11 +504,17 @@ private static string LanguageOf(string tag) /// /// A writing system whose only text is in the pronunciation field is made vernacular, /// because FieldWorks draws its pronunciation writing systems from the vernacular ones - /// (InitializePronunciationWritingSystems considers no others). That is not an answer, only - /// a role it must have, so the config can still add analysis: the field is not an anchor - /// (see VernacularAnchorFields), and a stray pronunciation string should not take a - /// writing system's analysis role away. In the 2026-10-01 corpus this gives xin-flex's - /// "xin", which holds 226 pronunciations and nothing else, the vernacular role it needs. + /// (InitializePronunciationWritingSystems considers no others). It is not returned as + /// answered for, so the anchor fields can still add analysis: Swedish offered for the + /// gloss, with a stray pronunciation string, is both. But from here on it counts as + /// vernacular, as any placed writing system does: a field of no role does not put it in + /// doubt, and a custom field it alone is offered in is taken for a vernacular field. So + /// a writing system offered only in custom fields, with pronunciation text and nothing + /// else, is vernacular only. In the 2026-10-01 corpus all 57 writing systems with + /// pronunciation text and no anchor text are used that way -- phonetic, IPA and audio + /// writing systems, and xin-flex's "xin", which holds 226 pronunciations and nothing else + /// -- and FLEx has as vernacular every one of them whose role LfMerge's export records. + /// The pronunciation field is still not an anchor; see VernacularAnchorFields. /// private static ISet ClassifyByText(LfWritingSystemUsage usage, IDictionary> byPath, ISet vernacular, ISet analysis) From 969b7ffcd3a6b43f71a72dee479e2801edacfb14 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 15:20:05 +0700 Subject: [PATCH 35/51] Bring comments the code review found out of date back in line None of these change behaviour. - LanguageTags.WsIdFromLfTag, LfWsToLcmWs and a writing-system test said LfMerge exports input systems by LanguageTag; 2377f89 made it spell them by Id, so they now say it used to. - WsIdFromLfTag said that of several writing systems sharing a LanguageTag the one created first wins. It is the one with the lowest handle, and LCM numbers writing systems in the order it loads them. - ClassifyVernacularWritingSystems' summary named the old anchors, listed the example sentence among the fields assigned by overlap, and said nothing of the text, the pronunciation rule, the order-independent field loop, the companions or language affinity. It now describes all of them. Its returns comment cited grc-vie-flex and "11 projects"; in the 2026-10-01 corpus it is 7, and grc-vie-flex is no longer one of them, so it now cites kam-flex. - LfWsToLcmWs said roles are worked out from the config; they are worked out from the config and the text. - HasRole now says that a role found under one spelling counts for all of them, even where the text decided otherwise under another -- which can only add a role. - The doc comment for LcmWsToLfWs had been left above LfTagOf, giving it two summaries; it is back on LcmWsToLfWs. That method's FW 8 comment no longer claims the code stays compatible with FW 8. - LfTagsOf said it keeps a missing Id from ending the transfer. That holds for the pronunciation list; one in the vernacular or analysis lists still ends it, in LcmWsToLfWs. - TheExampleSentenceIsClassifiedLikeAnyOtherField is now TheNestedExampleAnchorsAreFound, the example sentence being an anchor. - A test comment said LfMultiText resolves keys with GetWsFromStr; it used to. - The writing-system tests' restore helpers said the mock project record outlives a test. Each test gets a fresh one; what outlives a test is a writing system added to LCM. - LexiconReads said each read is a pass over Mongo. Filtered requests count too. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 17 +++--- .../VernacularClassificationTests.cs | 6 +- src/LfMerge.Core.Tests/TestDoubles.cs | 6 +- .../ConvertLcmToMongoLexicon.cs | 15 +++-- .../ConvertMongoToLcmLexicon.cs | 56 +++++++++++-------- src/LfMerge.Core/LanguageTags.cs | 8 ++- 6 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 8eb0edcc..4da48971 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -39,9 +39,10 @@ public class ConvertMongoToLcmWritingSystemsTests : LcmTestBase private const string MixedCaseId = "qaa-Zxxx-x-kal-audio"; /// - /// Adds one input system to the memoized mock project record, and returns an action that puts - /// the record back as it was. The double memoizes per project code, so without the restore the - /// addition would leak into every later test in this fixture. + /// Adds one input system to the mock project record, and returns an action that puts the + /// record back as it was. Each test gets a fresh record, so the restore is only tidiness; + /// what does outlive a test is a writing system it adds to LCM, which is why the tests that + /// add one each use tags of their own. /// private Action AddLfInputSystem(string tag) { @@ -62,7 +63,7 @@ private Action AddLfInputSystem(string tag) /// /// Adds a tag to one config field's input systems, found by its path through the entry's /// field lists, and returns an action that takes it out again. Like the input systems, the - /// config is memoized with the record, so the change must not outlive the test. + /// config comes fresh with each test's record, so this too is only tidiness. /// private Action AddToConfigField(string tag, params string[] path) { @@ -111,9 +112,9 @@ public void LfWsToLcmWs_NonCanonicalTag_ReusesTheCanonicalWritingSystem(string l [Test] public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWritingSystem() { - // Setup: a definition keyed by the non-canonical tag. LfMultiText resolves each key with - // GetWsFromStr and skips anything that returns 0, so an unresolved key silently drops the - // value -- and the clearing pass then blanks whatever LCM had for that field. + // Setup: a definition keyed by the non-canonical tag. LfMultiText used to resolve each key + // with GetWsFromStr and skip anything that returned 0, so an unresolved key dropped the + // value -- and the clearing pass then blanked whatever LCM had for that field. const string newDefinition = "Definition stored under a non-canonical writing system tag"; var data = new SampleData(); data.bsonTestData["senses"][0]["definition"] = new BsonDocument { @@ -252,7 +253,7 @@ public void WsIdFromLfTag_CanonicalTagFindsAWritingSystemWhoseIdIsSpelledOtherwi public void LfWsToLcmWs_InputSystemSpelledAsTheLanguageTag_DoesNotDuplicateTheWritingSystem() { // The spt-flex layout: LCM holds "hi-Deva-IN", whose LanguageTag is "hi-IN", and LF's - // input systems -- which LfMerge exports by LanguageTag -- call it "hi-IN". GetOrSet looks + // input systems -- which LfMerge used to export by LanguageTag -- call it "hi-IN". GetOrSet looks // only at Ids, so it created a second writing system "hi-IN" and put it in the lists. CoreWritingSystemDefinition real = AddWritingSystemWithId("pt-Latn-AO", "pt-AO"); Action restore = AddLfInputSystem("pt-AO"); diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index babd2b05..a8d8f587 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -300,11 +300,11 @@ public void AWritingSystemInBothAnchorsIsBothVernacularAndAnalysis() } /// - /// The example sentence is nested, so this covers the recursion into a field list as well as - /// the classification. Its sibling translation field is analysis and must stay that way. + /// The example sentence and translation are anchors nested in a field list, so this covers + /// the recursion into one: the sentence is vernacular and the translation analysis. /// [Test] - public void TheExampleSentenceIsClassifiedLikeAnyOtherField() + public void TheNestedExampleAnchorsAreFound() { var config = Config(("lexeme", Field("kal")), ("senses.fields.definition", Field("fr")), ("senses.fields.gloss", Field("en")), diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index b6d73622..7a752db3 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -241,7 +241,11 @@ public IEnumerable GetRecords(ILfProject project, string c return GetRecords(project, collectionName).Where(filter.Compile()); } - /// How many times the lexicon has been read, each read being a pass over Mongo. + /// + /// How many times the lexicon has been asked for, each request for the whole of it being a + /// pass over Mongo. A filtered request counts too, so a test counting passes should have + /// nothing else asking: the comment converter asks once per comment. + /// public int LexiconReads { get; private set; } public IEnumerable GetRecords(ILfProject project, string collectionName) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index 97720c71..d2da3dc3 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -580,10 +580,6 @@ private static string LcmPictureFilenameToLfPictureFilename(string LcmInternalFi return Regex.Replace(LcmInternalFilename.Normalize(System.Text.NormalizationForm.FormC), @"^Pictures[/\\]", ""); } - /// - /// Converts Lcm writing systems to LF input systems - /// - /// The list of LF input systems. /// /// How Language Forge spells an LCM writing system: exactly as every multitext key LfMerge /// writes is spelled, since those come from GetStrFromWs too (see LfMultiText). That is the @@ -602,7 +598,10 @@ private string LfTagOf(CoreWritingSystemDefinition ws) /// /// The LF tags of one of the project's current writing-system lists, leaving out any Id the - /// list names that LCM has no writing system for, rather than ending the transfer. + /// list names that LCM has no writing system for, rather than ending the transfer. That + /// protects the pronunciation list. A missing Id in a current vernacular or analysis list is + /// almost always in the full list too, and LcmWsToLfWs, which reads the full lists through + /// AllWritingSystems, still fails on it first -- as FieldWorks itself would. /// /// The list is read with foreach on purpose. liblcm's enumerator yields null for such an /// Id, but its indexer and CopyTo throw KeyNotFoundException, and LINQ's Select and ToList @@ -625,9 +624,13 @@ private List LfTagsOf(IEnumerable wss, stri return tags; } + /// + /// Converts Lcm writing systems to LF input systems + /// + /// The list of LF input systems. private Dictionary LcmWsToLfWs() { - // Using var here so that we'll stay compatible with both FW 8 and 9 (the type of these two lists changed between 8 and 9). + // The types of these two lists changed between FW 8 and 9, which is why they are var. var vernacularWSList = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems; var analysisWSList = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems; diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index ce44e448..1e64f0bd 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -237,23 +237,29 @@ private IEnumerable GetLexicon(ILfProject project) /// /// Work out which writing systems this project treats as vernacular and which as analysis, - /// from its own config. + /// from its own config and the text in its lexicon. /// - /// The lexeme and citation form fields are vernacular by definition, the sense definition and - /// gloss are analysis by definition. Every other field carrying input systems (etymology, the - /// example sentence, custom fields) is assigned by overlap: a writing system that appears in - /// the vernacular anchors and NOT in the analysis anchors is evidence the field is vernacular, - /// and vice versa. A writing system present in both anchors -- "en" very often is -- is no - /// evidence either way and is ignored for that purpose; it is both vernacular and analysis. + /// The anchor fields have a role FieldWorks fixes: the lexeme, citation form and example + /// sentence are vernacular; the definition, gloss, example translation, note and literal + /// meaning are analysis (see VernacularAnchorFields). /// - /// Evidence decides the role of the field, not of every writing system in it, and it has to - /// point one way: a field offering writing systems exclusive to each role, such as an - /// etymology configured as [seh, pt], is evidence of neither. So it does not make Portuguese - /// vernacular, nor anything else it offers that the anchors have not placed. + /// Where the lexicon has text in the anchors for a writing system, that text decides: one + /// holding nearly all of it in the vernacular anchors is vernacular whatever the config + /// offers, and vice versa, and one whose text is split is both. See + /// . A writing system whose only text is pronunciation text is + /// vernacular; see . /// - /// Where the lexicon has text to show for a writing system, that text decides instead: a - /// writing system holding nearly all of its text in the vernacular anchors is vernacular - /// whatever the config offers, and vice versa. See . + /// The rest go by the config. A writing system the anchors offer is placed by them, and one + /// in both anchor groups -- "en" very often is -- is both. Every other field carrying input + /// systems (etymology, custom fields) is assigned by overlap: a writing system exclusive to + /// the vernacular role is evidence the field is vernacular, and vice versa. Evidence decides + /// the role of the field, not of every writing system in it, and it has to point one way: a + /// field offering writing systems exclusive to each role, such as an etymology configured + /// as [seh, pt], is evidence of neither, so it does not make Portuguese vernacular, nor + /// anything else it offers that the anchors have not placed. All the fields' evidence is + /// weighed before any writing system is placed by it, so the order of the config's fields + /// does not matter. What is left is settled, where it can be, by the company a writing + /// system keeps and then by its language. /// /// This replaces comparing each tag against ProjectRecord.LanguageCode, which named exactly one /// vernacular writing system and got it wrong whenever languageCode disagreed with the lexeme @@ -265,17 +271,18 @@ private IEnumerable GetLexicon(ILfProject project) /// How much text each writing system holds in each field, or null to go on the config alone. /// /// - /// The vernacular tags; the analysis tags; and the tags that could not be resolved. The first two sets overlap wherever a writing system plays both roles. A tag in - /// neither set -- one no config field uses -- is analysis, as it always has been. + /// The vernacular tags; the analysis tags; and the tags that could not be resolved. The first + /// two sets overlap wherever a writing system plays both roles. A tag in neither set -- one + /// no config field uses -- is analysis, as it always has been. /// /// Unresolved writing systems are treated as BOTH vernacular and analysis, since nothing says /// which they are and a writing system missing from the list a field draws on leaves that /// field's data with nowhere to go. The FieldWorks fields such a writing system most often /// feeds (etymology form, example sentence) are vernacular-typed, which is why being only /// analysis will not do; but a custom field can be analysis-typed, as a French-only notes - /// field would be, which is why being only vernacular will not do either. 11 projects in the - /// corpus need this, e.g. grc-vie-flex, whose etymologies are in Hebrew and Aramaic -- - /// neither its vernacular (Greek) nor its analysis (English, Vietnamese). + /// field would be, which is why being only vernacular will not do either. 7 projects in the + /// 2026-10-01 corpus need this, e.g. kam-flex, whose Spanish and French are offered only in + /// a wordlist field that offers its vernacular and its analysis language as well. /// public static (ISet Vernacular, ISet Analysis, ISet Unresolved) ClassifyVernacularWritingSystems(LfProjectConfig config, string languageCode, @@ -668,8 +675,8 @@ private void LfWsToLcmWs(Dictionary lfWsList, */ // Find the writing system the way every other LF tag is found before creating one. - // LfMerge exports input systems by LanguageTag, while LCM knows a writing system by - // its Id, and in older projects the two differ: spt-flex's input system "hi-IN" is + // LfMerge used to export input systems by LanguageTag, while LCM knows a writing + // system by its Id, and in older projects the two differ: spt-flex's input system "hi-IN" is // the writing system LCM holds as "hi-Deva-IN". GetOrSet looks a tag up only as an // Id, so it did not find it, created a duplicate "hi-IN" and put it in the current // lists; WsIdFromLfTag, which every multitext key goes through, then matched the @@ -707,7 +714,7 @@ private void LfWsToLcmWs(Dictionary lfWsList, if (!wsAlreadyExisted) { // LF doesn't distinguish between vernacular/analysis WS, so the roles are worked - // out per project from its config -- see ClassifyVernacularWritingSystems. A + // out per project from its config and text -- see ClassifyVernacularWritingSystems. A // writing system can play both, "en" in lexeme and gloss alike being the usual // case, and one no field claims is analysis. The input system, the config and // the lexicon need not spell the writing system alike, so the roles are looked @@ -729,7 +736,10 @@ private void LfWsToLcmWs(Dictionary lfWsList, /// input system can spell it otherwise: an LF user who adds "qaa-x-qaa-new" may configure /// fields with "qaa-x-new". Spellings sharing a canonical form can be taken for one writing /// system here, because this one is new: had LCM held any writing system with that canonical - /// form, WsIdFromLfTag would have found it. + /// form, WsIdFromLfTag would have found it. A role found under one spelling therefore counts + /// for all of them, even where the text decided otherwise under another: a writing system + /// whose text makes it vernacular only, offered by the config in the gloss under another + /// spelling, is both. That can only add a role, never take one away. /// private static bool HasRole(ISet role, string tag) { diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs index 2963dc83..620c47b0 100644 --- a/src/LfMerge.Core/LanguageTags.cs +++ b/src/LfMerge.Core/LanguageTags.cs @@ -50,8 +50,9 @@ public static string Canonical(string tag) /// LCM knows a writing system by its Id, which is whatever its .ldml file is named, and also /// has its LanguageTag, which is always canonical. The two differ in older projects: /// brb-flex-2022 holds "km-Khmr-KH.ldml", whose identity is km plus KH, so its Id is - /// "km-Khmr-KH" and its LanguageTag "km-KH". LfMerge writes LF's multitext keys by Id and the - /// project's input systems by LanguageTag, so LF can hand back either spelling. Tried in turn: + /// "km-Khmr-KH" and its LanguageTag "km-KH". LfMerge writes LF's multitext keys by Id, but it + /// used to write the project's input systems by LanguageTag, and LF users type under the + /// spelling the config gives, so LF can hand back either spelling. Tried in turn: /// /// 1. The tag as LF spells it, as an Id. This must come first. brb-flex-2022 also holds a /// second, unused writing system whose Id is "km-KH"; trying the canonical form first @@ -60,7 +61,8 @@ public static string Canonical(string tag) /// 2. Its canonical form, as an Id: the Id LCM gives any writing system it creates. /// 3. Its canonical form as a LanguageTag, for an Id spelled some other way: /// spt-flex's "hi-IN" is the writing system LCM holds as "hi-Deva-IN". Should several - /// writing systems share the LanguageTag, the one created first wins. + /// writing systems share the LanguageTag, the one with the lowest handle wins: LCM + /// numbers writing systems in the order it loads them, not the order they were made. /// public static int WsIdFromLfTag(ILgWritingSystemFactory wsf, string tag) { From 3540fd973f533ace5ee7748a2dbce3e0b6268cef Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 16:03:41 +0700 Subject: [PATCH 36/51] Make three tests test what they say they test A code review found tests that would still pass with the code they cover broken: - AWritingSystemInDoubtIsSettledByTheCompanyItKeeps passed with ResolveByCompanions deleted. Since f0da10e its "abc" is placed by the field loop, and the writing system the companions pass does settle, "xyz", was never asserted. It now is. - LcmToMongo_PronunciationListNamingAWritingSystemLcmDoesNotHave_LeavesItOut only showed the export did not throw: its assertion was about the inputSystems record, which the pronunciation list never feeds. The Mongo double now keeps the vernacular, analysis and pronunciation lists it is given, and the test checks the pronunciation list holds the writing system LCM has and not the one it lacks. - LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEverywhere checked only the record and a gloss key. It now checks the analysis list and the custom-field config are spelled by Id too. Each now fails with its code broken: with ResolveByCompanions skipped, with LfTagsOf returning nothing, and with custom-field input systems spelled by LanguageTag. The double's LexiconReads also counted filtered requests, such as the comment converter's one per comment, as passes over the lexicon, so MongoToLcm_NoInputSystemNewToLcm_ReadsTheLexiconOnce held only while the sample data had no comments. Filtered requests are no longer counted. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 15 +++++++---- .../VernacularClassificationTests.cs | 5 ++++ src/LfMerge.Core.Tests/TestDoubles.cs | 26 +++++++++++++++---- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 4da48971..89a7e2f3 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -460,6 +460,12 @@ public void LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEver Assert.That(lfEntry.Senses[0].Gloss.Keys, Contains.Item(id), "the multitext key and the input system should be spelled alike"); Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo(gloss)); + Assert.That(_conn.LastAnalysisWss, Contains.Item(id)); + Assert.That(_conn.LastAnalysisWss, Does.Not.Contain("nl-BE")); + var customFieldTags = _conn.GetCustomFieldConfig(_lfProj).Values.OfType() + .SelectMany(field => field.InputSystems).ToList(); + Assert.That(customFieldTags, Contains.Item(id), "a custom field offering every analysis writing system"); + Assert.That(customFieldTags, Does.Not.Contain("nl-BE")); } [Test] @@ -467,16 +473,15 @@ public void LcmToMongo_PronunciationListNamingAWritingSystemLcmDoesNotHave_Leave { // Setup: a pronunciation list naming an Id LCM has no writing system for. (The same in // the vernacular list would still fail, inside liblcm's own GetWritingSystemList.) + string vernacularId = _cache.LanguageProject.DefaultVernacularWritingSystem.Id; UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => - _cache.LanguageProject.CurPronunWss = "zzz-x-nonesuch"); + _cache.LanguageProject.CurPronunWss = vernacularId + " zzz-x-nonesuch"); // Exercise SutLcmToMongo.Run(_lfProj); - // Verify - Dictionary inputSystems = _conn.GetInputSystems(_lfProj); - Assert.That(inputSystems.Keys, Contains.Item(_cache.LanguageProject.DefaultVernacularWritingSystem.Id)); - Assert.That(inputSystems.Keys, Does.Not.Contain("zzz-x-nonesuch")); + // Verify: the list LF is given keeps the writing system LCM has and leaves out the other + Assert.That(_conn.LastPronunciationWss, Is.EqualTo(new[] { vernacularId })); } [Test] diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index a8d8f587..cf1b8d62 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -590,6 +590,11 @@ public void AWritingSystemInDoubtIsSettledByTheCompanyItKeeps(string companion, "abc shares a field with {0}, which is settled, so abc is settled too", companion); Assert.That(result.Vernacular.Contains("abc"), Is.EqualTo(isVernacular)); Assert.That(result.Analysis.Contains("abc"), Is.EqualTo(!isVernacular)); + // xyz is offered only in the field of no role, so the field loop puts it in doubt; only + // the company it keeps there, abc, settles it + Assert.That(result.Unresolved, Does.Not.Contain("xyz"), "xyz is settled by the company it keeps"); + Assert.That(result.Vernacular.Contains("xyz"), Is.EqualTo(isVernacular)); + Assert.That(result.Analysis.Contains("xyz"), Is.EqualTo(!isVernacular)); } /// diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index 7a752db3..024a7a54 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -154,11 +154,19 @@ public Dictionary GetInputSystems(ILfProject projec return _storedInputSystems; } + /// The current vernacular, analysis and pronunciation lists the last export gave. + public List LastVernacularWss { get; private set; } + public List LastAnalysisWss { get; private set; } + public List LastPronunciationWss { get; private set; } + public bool SetInputSystems(ILfProject project, Dictionary inputSystems, List vernacularWss, List analysisWss, List pronunciationWss) { foreach (var ws in inputSystems.Keys) _storedInputSystems[ws] = inputSystems[ws]; + LastVernacularWss = vernacularWss; + LastAnalysisWss = analysisWss; + LastPronunciationWss = pronunciationWss; if (project.IsInitialClone) { @@ -238,22 +246,30 @@ public IEnumerable GetLfOptionLists() public IEnumerable GetRecords(ILfProject project, string collectionName, Expression> filter) { - return GetRecords(project, collectionName).Where(filter.Compile()); + // A filtered request finds a few records, not a pass over the collection, so it is not + // counted in LexiconReads + return Records(collectionName).Where(filter.Compile()); } /// - /// How many times the lexicon has been asked for, each request for the whole of it being a - /// pass over Mongo. A filtered request counts too, so a test counting passes should have - /// nothing else asking: the comment converter asks once per comment. + /// How many times the whole lexicon has been asked for, each request being a pass over + /// Mongo. Filtered requests, such as the comment converter's one per comment, are not + /// counted. /// public int LexiconReads { get; private set; } public IEnumerable GetRecords(ILfProject project, string collectionName) + { + if (collectionName == MagicStrings.LfCollectionNameForLexicon) + LexiconReads++; + return Records(collectionName); + } + + private IEnumerable Records(string collectionName) { switch (collectionName) { case MagicStrings.LfCollectionNameForLexicon: - LexiconReads++; return (IEnumerable)GetLfLexEntries(); case MagicStrings.LfCollectionNameForOptionLists: return (IEnumerable)GetLfOptionLists(); From 88d426a94e33dff894f8d7e1698412690d1cd7f5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 16:05:13 +0700 Subject: [PATCH 37/51] Ignore a null or empty tag in a config field's input systems CollectInputSystems copied each config field's input systems as they were, so a null tag reached ClassifyByText, where counting its text called Dictionary.TryGetValue(null) and threw. The exception escaped LfWsToLcmWs and ended the whole transfer. An empty tag did no harm but named no writing system either. Both are now left out when the config is read, which ConfiguredTags also goes through. The 2026-10-01 corpus has no such tag in any project's config, so this is only a guard. The new test fails without this, with an ArgumentNullException. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 17 +++++++++++++++++ .../DataConverters/ConvertMongoToLcmLexicon.cs | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index cf1b8d62..42222d39 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -88,6 +88,23 @@ private static LfWritingSystemUsage Usage(params (string Path, string Tag, int C return LfWritingSystemUsage.FromLexicon(lexicon); } + /// + /// A null or empty tag in a config field's input systems names no writing system. The + /// 2026-10-01 corpus has none, but a null one made counting the text throw, which would + /// have ended the whole transfer. + /// + [Test] + public void ANullOrEmptyTagInTheConfigIsIgnored() + { + var config = Config(("lexeme", Field("kal", null)), ("senses.fields.gloss", Field("en", ""))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + [Test] public void LexemeAndCitationFormAreAlwaysVernacular() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 1e64f0bd..7b91fabf 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -605,8 +605,11 @@ private static void CollectInputSystems(LfConfigFieldList fieldList, string path { string childPath = string.IsNullOrEmpty(path) ? child.Key : path + "." + child.Key; var multiText = child.Value as LfConfigMultiText; + // A null or empty tag names no writing system, and a null one would make the text + // counts throw, ending the whole transfer if (multiText != null && multiText.InputSystems != null) - byPath[childPath] = new HashSet(multiText.InputSystems, StringComparer.OrdinalIgnoreCase); + byPath[childPath] = new HashSet(multiText.InputSystems.Where(tag => !string.IsNullOrEmpty(tag)), + StringComparer.OrdinalIgnoreCase); var nested = child.Value as LfConfigFieldList; if (nested != null) CollectInputSystems(nested, childPath + ".fields", byPath); From fa97a1626f25ea6f22a0a9ec72721501d37a797f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 16:07:53 +0700 Subject: [PATCH 38/51] Choose between two spellings in single-string fields as multitexts do BestStringAndWsId, which the single-string lexicon fields, picture captions and the String custom field go through, took the first non-empty value in document order when two keys named one writing system. WriteToLcm, which writes the multitext fields, takes the key spelled as the project's config spells the writing system, since that is the one LF's editor shows. The two could pick different spellings' text for the same keys, and the single-string fields could pick the export's hidden, stale value over an LF user's edit. BestStringAndWsId now chooses each writing system's key with the same rule, from the same configured spellings, before it leaves out empty values, so a value the user has cleared under the config's spelling is not replaced by the hidden one either. The 2026-10-01 corpus holds no two keys naming one writing system in any single-string field, so this changes nothing there today. The two new tests fail with the old choice. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 40 +++++++++++++++++++ .../ConvertMongoToLcmCustomField.cs | 2 +- .../ConvertMongoToLcmLexicon.cs | 4 +- .../LanguageForge/Model/LfMultiText.cs | 38 ++++++++++++++---- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 89a7e2f3..04fc47bc 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -524,6 +524,46 @@ public void BestStringAndWsId_FallsBackToTheFirstValueInAKnownWritingSystem() Assert.That(best.Value, Is.EqualTo("English")); } + /// + /// A single-string field chooses between two keys naming one writing system as a multitext + /// field does: the one spelled as the config spells it. Here the export's hidden value is + /// under the Id and the LF user's under the LanguageTag, which the config uses. + /// + [Test] + public void BestStringAndWsId_TwoKeysForOneWritingSystem_TakesTheConfigsSpelling() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("pl-Latn-PL", "pl-PL"); + var multiText = new LfMultiText { + { "pl-Latn-PL", LfStringField.FromString("the export's value") }, + { "pl-PL", LfStringField.FromString("the user's edit") }, + }; + var configuredTags = new HashSet(StringComparer.Ordinal) { "pl-PL" }; + + KeyValuePair best = multiText.BestStringAndWsId(new[] { ws.Handle }, _cache.WritingSystemFactory, configuredTags); + + Assert.That(best.Key, Is.EqualTo(ws.Handle)); + Assert.That(best.Value, Is.EqualTo("the user's edit")); + } + + /// + /// The key is chosen before empty values are left out: a value the LF user has cleared, + /// under the config's spelling, is not replaced by the export's hidden one. + /// + [Test] + public void BestStringAndWsId_ValueClearedUnderTheConfigsSpelling_IsNotReplacedByTheHiddenOne() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("cs-Latn-CZ", "cs-CZ"); + var multiText = new LfMultiText { + { "cs-Latn-CZ", LfStringField.FromString("the export's value") }, + { "cs-CZ", LfStringField.FromString("") }, + }; + var configuredTags = new HashSet(StringComparer.Ordinal) { "cs-CZ" }; + + KeyValuePair best = multiText.BestStringAndWsId(new[] { ws.Handle }, _cache.WritingSystemFactory, configuredTags); + + Assert.That(best.Value, Is.Null, "nothing should be placed for the cleared writing system"); + } + [Test] public void SpanStrToTsString_NonCanonicalSpanLang_UsesTheCanonicalWritingSystem() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index 022a4afe..7947dba6 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -401,7 +401,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); int wsIdForField = lcmMetaData.GetFieldWs(flid); KeyValuePair kv = valueAsMultiText.BestStringAndWsId( - new[] { wsIdForField }, servLoc.WritingSystemFactory); + new[] { wsIdForField }, servLoc.WritingSystemFactory, configuredTags); int foundWsId = kv.Key; string foundData = kv.Value ?? string.Empty; if (foundWsId == 0) diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 7b91fabf..07b2a7a8 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -796,7 +796,7 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool // By handle, not by comparing LF's keys with ws.Id, which would miss a key LF spells // differently from LCM. Falls back to the first value in any writing system LCM knows. KeyValuePair best = input.BestStringAndWsId( - wsesToSearch.Select(ws => ws.Handle), ServiceLocator.WritingSystemFactory); + wsesToSearch.Select(ws => ws.Handle), ServiceLocator.WritingSystemFactory, _configuredTags); if (best.Value == null) return null; return new Tuple(best.Value, best.Key); @@ -1230,7 +1230,7 @@ private void LfPictureToLcmPicture(LfPicture lfPicture, ILexSense owner) // SetMultiStringFrom below fills in every alternative that can be placed. KeyValuePair kv = lfPicture.Caption.BestStringAndWsId( ServiceLocator.LanguageProject.AnalysisWritingSystems.Select(ws => ws.Handle), - ServiceLocator.WritingSystemFactory); + ServiceLocator.WritingSystemFactory, _configuredTags); if (kv.Value != null) { captionWs = kv.Key; diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index e024008d..002d46d3 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -117,18 +117,42 @@ public string FirstNonEmptyString() /// back to the first non-empty value in any writing system LCM knows, together with that /// writing system's handle. Keys are resolved to handles before they are compared, so a key /// LF spells differently from LCM's id for the writing system still matches it. + /// + /// Where two keys name one writing system, the value is the one WriteToLcm would write: the + /// key spelled as the config spells it, or else the later one. That is chosen before empty + /// values are left out, so a value an LF user has cleared under the config's spelling is + /// not replaced by the export's hidden, stale one. /// + /// + /// Every writing-system spelling the project's config uses; see WriteToLcm. + /// /// The handle and the value, or (0, null) when no non-empty value resolves. - public KeyValuePair BestStringAndWsId(IEnumerable wsSearchOrder, ILgWritingSystemFactory wsManager) + public KeyValuePair BestStringAndWsId(IEnumerable wsSearchOrder, ILgWritingSystemFactory wsManager, + ISet configuredTags = null) { - var resolved = new List>(); - foreach (KeyValuePair kv in this) + // Each writing system's key, in the order the writing systems first appear + var keyFor = new Dictionary(); + var order = new List(); + foreach (string key in Keys) { - if (kv.Value == null || kv.Value.IsEmpty) + int wsId = LanguageTags.WsIdFromLfTag(wsManager, key); + if (wsId == 0) continue; - int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); - if (wsId != 0) - resolved.Add(new KeyValuePair(wsId, kv.Value.Value)); + string other; + if (keyFor.TryGetValue(wsId, out other)) + keyFor[wsId] = PreferredKey(other, key, configuredTags); + else + { + keyFor[wsId] = key; + order.Add(wsId); + } + } + var resolved = new List>(); + foreach (int wsId in order) + { + LfStringField field = this[keyFor[wsId]]; + if (field != null && !field.IsEmpty) + resolved.Add(new KeyValuePair(wsId, field.Value)); } foreach (int wsId in wsSearchOrder) { From f77493c357aca411a8620db7907afcc09279244a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 16:10:03 +0700 Subject: [PATCH 39/51] Keep a span's text when its lang names no writing system LCM has SpanStrToTsString gave a run whose span lang resolves to no LCM writing system writing system 0, and building the string then threw ArgumentOutOfRangeException ("The specified writing system code is invalid"). Inside WriteToLcm that came after earlier alternatives had been written and before the clearing pass, so the entry's per-entry catch left it half-written -- the failure c66141f fixed for single-string fields whose text could not be placed. 8900058 resolved span langs through WsIdFromLfTag, which fixed non-canonical spellings of writing systems LCM has, but not this. Such a run's text is now kept in the string's own writing system, losing only its language mark, and a warning names the lang and the text. In the 2026-10-01 corpus 29 of the 83,743 span langs in Send/Receive projects name no writing system those projects have, in five projects, and none is in an entry edited since its project last synced. The new test fails without this, with the ArgumentOutOfRangeException. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 17 +++++++++++++++++ .../ConvertMongoToLcmTsStrings.cs | 9 +++++++++ 2 files changed, 26 insertions(+) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 04fc47bc..96e16e27 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -576,6 +576,23 @@ public void SpanStrToTsString_NonCanonicalSpanLang_UsesTheCanonicalWritingSystem Assert.That(tss.get_WritingSystem(1), Is.EqualTo(wsFr)); } + /// + /// A span whose lang names no writing system LCM has used to make the string build throw, + /// abandoning the entry half-written. Its text is now kept in the string's own writing + /// system, and the lang reported. + /// + [Test] + public void SpanStrToTsString_SpanLangNamingNoWritingSystem_KeepsTheTextInTheMainWritingSystem() + { + var tss = ConvertMongoToLcmTsStrings.SpanStrToTsString( + "English unplaceable", _wsEn, _cache.WritingSystemFactory); + + Assert.That(tss.Text, Is.EqualTo("English unplaceable")); + for (int run = 0; run < tss.RunCount; run++) + Assert.That(tss.get_WritingSystem(run), Is.EqualTo(_wsEn)); + Assert.That(_env.Logger.GetMessages(), Does.Contain("span lang \"zzz-x-nonesuch\" names no writing system LCM has")); + } + [Test] public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis() { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs index 21d01d33..56d8242f 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs @@ -90,6 +90,15 @@ public static ITsString SpanStrToTsString(string source, int mainWs, ILgWritingS // To remove a string property, you set it to null, so we can just use StyleName directly whether or not it's null. builder.SetStrPropValue((int)FwTextPropType.ktptNamedStyle, run.StyleName); int runWs = (run.Lang == null) ? mainWs : LanguageTags.WsIdFromLfTag(wsf, run.Lang); + if (runWs == 0) + { + // A lang naming no writing system LCM has cannot be built into the string, and the + // exception that follows abandons the whole entry half-written. The run's text is + // kept in the string's own writing system; only its language mark is lost. + MainClass.Logger?.Warning("MongoToLcm: span lang \"{0}\" names no writing system LCM has; " + + "keeping \"{1}\" in the field's writing system", run.Lang, run.Content); + runWs = mainWs; + } builder.SetIntPropValues((int)FwTextPropType.ktptWs, (int)FwTextPropVar.ktpvDefault, runWs); // We don't care about Guids in this function, so run.Guid is ignored // But we do need to set any other int or string properties that were in the original From 28e2cc52975b25f96e903500134b11f4842e215b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 16:12:40 +0700 Subject: [PATCH 40/51] Flag an input system as in a list whatever case the list spells it in LcmWsToLfWs set each input system's VernacularWS and AnalysisWS flags with the current lists' own Contains, which compares Ids exactly. LCM resolves the lists' Ids ignoring case, and LfTagsOf reads them that way too, so a list spelling an Id in another case put the writing system in the list LF is given but flagged its input system as not in it. testlangproj's vernacular list names "qaa-Zxxx-x-kal-AUDIO" for "qaa-Zxxx-x-kal-audio"; a TODO in TransferLcmToMongoActionTests asked why that one was not flagged, and worked around it. The flags now come from the handles of the writing systems the lists yield, read the same way as LfTagsOf reads them. The TODO and its workaround are gone, and that test now checks every vernacular writing system is flagged. Language Forge does not read the flags, so nothing it shows changes. The new test fails without this: the flag is false. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 16 +++++++++++ .../Lcm/TransferLcmToMongoActionTests.cs | 12 +++----- .../ConvertLcmToMongoLexicon.cs | 28 +++++++++++++++---- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 96e16e27..52a0dbdf 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -468,6 +468,22 @@ public void LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEver Assert.That(customFieldTags, Does.Not.Contain("nl-BE")); } + /// + /// testlangproj's current vernacular list names "qaa-Zxxx-x-kal-AUDIO", and LCM holds the + /// writing system as "qaa-Zxxx-x-kal-audio". It is in the vernacular list LF is given, so its + /// input system must be flagged vernacular too; the list's own Contains compares Ids exactly. + /// + [Test] + public void LcmToMongo_ListSpellingAnIdInAnotherCase_FlagsTheInputSystemAsInTheList() + { + Assert.That(_cache.LanguageProject.CurVernWss, Does.Contain(MixedCaseTag), "precondition"); + + SutLcmToMongo.Run(_lfProj); + + Assert.That(_conn.LastVernacularWss, Contains.Item(MixedCaseId)); + Assert.That(_conn.GetInputSystems(_lfProj)[MixedCaseId].VernacularWS, Is.True); + } + [Test] public void LcmToMongo_PronunciationListNamingAWritingSystemLcmDoesNotHave_LeavesItOut() { diff --git a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs index 3a7d8b55..1f425769 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016 SIL International +// Copyright (c) 2016 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; @@ -58,18 +58,14 @@ public void Action_IsInitialClone_ShouldPopulateMongoInputSystems() // Verify const int expectedNumVernacularWS = 3; const int expectedNumAnalysisWS = 2; - // TODO: Investigate why qaa-Zxxx-x-kal-audio is in CurrentVernacularWritingSystems, but somehow in - // UpdateMongoDbFromLcm.LcmWsToLfWs() is not contained in _cache.LangProject.CurrentVernacularWritingSystems - const string notVernacularWs = "qaa-Zxxx-x-kal-audio"; lfWsList = _conn.GetInputSystems(lfProject); var languageProj = lfProject.FieldWorksProject.Cache.LangProject; + // Every one, qaa-Zxxx-x-kal-audio included: the list spells it qaa-Zxxx-x-kal-AUDIO, + // which the list's own Contains, comparing Ids exactly, did not match foreach (var lcmVernacularWs in languageProj.CurrentVernacularWritingSystems) - { - if (lcmVernacularWs.Id != notVernacularWs) - Assert.That(lfWsList[lcmVernacularWs.Id].VernacularWS); - } + Assert.That(lfWsList[lcmVernacularWs.Id].VernacularWS, "{0} should be flagged vernacular", lcmVernacularWs.Id); Assert.That(languageProj.CurrentVernacularWritingSystems.Count, Is.EqualTo(expectedNumVernacularWS)); foreach (var lcmAnalysisWs in languageProj.CurrentAnalysisWritingSystems) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index d2da3dc3..0a74f7be 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -624,15 +624,33 @@ private List LfTagsOf(IEnumerable wss, stri return tags; } + /// + /// The handles of the writing systems in one of the project's current lists, leaving out any + /// Id it names that LCM has no writing system for; see LfTagsOf. + /// + private static HashSet Handles(IEnumerable wss) + { + var handles = new HashSet(); + foreach (CoreWritingSystemDefinition ws in wss) + { + if (ws != null) + handles.Add(ws.Handle); + } + return handles; + } + /// /// Converts Lcm writing systems to LF input systems /// /// The list of LF input systems. private Dictionary LcmWsToLfWs() { - // The types of these two lists changed between FW 8 and 9, which is why they are var. - var vernacularWSList = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems; - var analysisWSList = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems; + // By handle, from the same enumerator LfTagsOf reads the lists with. The lists' own + // Contains compares Ids exactly, while LCM resolves them ignoring case, so a list + // naming "qaa-Zxxx-x-kal-AUDIO" for the writing system "qaa-Zxxx-x-kal-audio" put it in + // the vernacular list LF is given but flagged it as not vernacular. + var vernacularHandles = Handles(ServiceLocator.LanguageProject.CurrentVernacularWritingSystems); + var analysisHandles = Handles(ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems); var lfWsList = new Dictionary(); foreach (var LcmWs in ServiceLocator.LanguageProject.AllWritingSystems) @@ -645,8 +663,8 @@ private Dictionary LcmWsToLfWs() IsRightToLeft = LcmWs.RightToLeftScript, LanguageName = LcmWs.LanguageName, Tag = LfTagOf(LcmWs), - VernacularWS = vernacularWSList.Contains(LcmWs), - AnalysisWS = analysisWSList.Contains(LcmWs) + VernacularWS = vernacularHandles.Contains(LcmWs.Handle), + AnalysisWS = analysisHandles.Contains(LcmWs.Handle) }; lfWsList[lfWs.Tag] = lfWs; From f7f7f1611d77b7352910497cdad536fef842274e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 5 Oct 2026 17:07:26 +0700 Subject: [PATCH 41/51] Bring two more comments the code review found out of date in line ResolveByCompanions' summary gave as its example a writing system the field loop has already placed: one sharing a custom field with the vernacular alone. What the pass settles is a writing system offered only in fields that said nothing of their role, whose company there another field has since placed. It also makes one pass, so a writing system whose only company is another one in doubt keeps both roles; the summary now says so. The header of ConvertMongoToLcmWritingSystemsTests said LCM's lookups match an Id literally and that the tests add nothing to the fixture. The lookups ignore case, as the file itself says further down, and many tests now add writing systems of their own, each under a tag no other test uses. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 11 +++++++---- .../DataConverters/ConvertMongoToLcmLexicon.cs | 14 +++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 52a0dbdf..30aa02f0 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -17,12 +17,15 @@ namespace LfMerge.Core.Tests.Lcm.DataConverters { /// /// Language Forge stores a writing system tag exactly as it was typed, but LCM canonicalizes a - /// tag when it creates a writing system and thereafter knows that writing system only by the - /// canonical id. Every LCM lookup -- WritingSystemManager.TryGet, GetWsFromStr -- matches that id - /// literally, so a tag that is not already canonical is never found. + /// tag when it creates a writing system and thereafter knows that writing system by its Id. LCM's + /// own lookups -- WritingSystemManager.TryGet, GetWsFromStr -- match an Id regardless of case + /// but otherwise literally, so a tag differing from the Id in structure, a subtag absorbed or + /// dropped, is not found by them; LanguageTags.WsIdFromLfTag finds it. /// /// testlangproj already contains "qaa-x-kal" and "qaa-Zxxx-x-kal-audio", so a non-canonical LF - /// spelling of either exercises the mismatch without adding anything to the shared fixture project. + /// spelling of either exercises the mismatch without adding to the shared fixture project. The + /// tests that need a writing system of their own add it under a tag no other test uses, since + /// what is added to the fixture's LCM stays for the rest of the run. /// public class ConvertMongoToLcmWritingSystemsTests : LcmTestBase { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 07b2a7a8..d0ca0f70 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -380,14 +380,18 @@ private static (HashSet VernacularOnly, HashSet AnalysisOnly) Ex } /// - /// Settles what is left by the company a writing system keeps. A writing system nothing has - /// settled still shares its fields with others, and those may all have been settled: a - /// custom field holding the vernacular and one unplaced writing system is being used for - /// vernacular content, so the unplaced one is vernacular too. + /// Settles what is left by the company a writing system keeps. What is left is offered only + /// in fields that said nothing of their role -- a field with a writing system of one role + /// alone has already placed the rest -- but the company it keeps there may since have been + /// placed by another field: in a custom field offering "abc" and "xyz" and nothing else of + /// known role, "xyz" is vernacular if another custom field, offering "abc" beside the + /// vernacular, has made "abc" vernacular. /// /// Only unanimous company counts. A field holding writing systems of both roles says /// nothing, which is the same reason a writing system in both anchors is no evidence of a - /// field's role. + /// field's role. One pass is made, not one until nothing changes: a writing system whose + /// only company is another left in doubt stays in doubt, and keeps both roles, even if that + /// other is settled here. /// private static void ResolveByCompanions(IDictionary> byPath, ISet vernacular, ISet analysis, ISet unresolved) From be87e86d8ac7f5053efc75113f85a9d10961895e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 06:11:03 +0700 Subject: [PATCH 42/51] Say why a single-string field's text was not written A single-string field keeps its value when none of LF's text can be placed, and logged that the text "has nowhere to go, since LCM has none" of its writing systems. That is one reason. The other is that the text is all under keys passed over for an empty key naming the same writing system: the config's spelling, cleared by an LF user or filled in empty by LF's editor beside the export's value under the Id. LCM has that writing system, so the log named the wrong cause. Each key with text left out is now reported with its own reason, and the field still keeps its value, since nothing here tells a clear from the editor's empty value. The String custom field, which said nothing at all, reports the same way. The choice of one key per writing system, which WriteToLcm and BestStringAndWsId each made, is now made in one place, ChooseKeys, and KeysPassedOver gives the keys it leaves out. The Mongo test double kept its entries by round-tripping them through LfMerge's serializer, which drops a multitext holding an empty value, so no test could hand the converter LF's empty values. It now keeps entries as documents, as Mongo does; LfMerge's own writes still go through its serializer, as they do against Mongo. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 67 +++++++++++++ src/LfMerge.Core.Tests/TestDoubles.cs | 47 +++++---- .../ConvertMongoToLcmCustomField.cs | 8 +- .../ConvertMongoToLcmLexicon.cs | 31 +++++- .../LanguageForge/Model/LfMultiText.cs | 99 +++++++++++-------- 5 files changed, 189 insertions(+), 63 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 30aa02f0..159d3a08 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -196,6 +196,53 @@ public void SingleStringField_TextInNoKnownWritingSystem_KeepsTheLcmValueAndSync "text that cannot be placed must not replace or clear what LCM holds"); Assert.That(lcmSense.SocioLinguisticsNote.get_String(_wsEn).Text, Is.EqualTo(laterNote), "the rest of the entry should still have been converted"); + Assert.That(_env.Logger.GetMessages(), Does.Contain( + "MongoToLcm: skipping text under \"zzz-x-nonesuch\" (Unplaceable), which names no writing system LCM has; " + + "leaving the field as it was")); + } + + /// + /// A single-string field whose text is all under a key passed over for an empty one naming + /// the same writing system -- the config's spelling, which LF's editor fills in empty -- + /// keeps what LCM holds, and the log says so for that reason, not that LCM lacks the + /// writing system. + /// + [Test] + public void SingleStringField_TextOnlyUnderAKeyPassedOverForAnEmptyOne_KeepsTheLcmValueAndSaysWhy() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("ro-Latn-MD", "ro-MD"); + const string original = "Homo sapiens"; + var data = new SampleData(); + BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument; + sense["scientificName"] = new BsonDocument { { "en", new BsonDocument { { "value", original } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + SutMongoToLcm.Run(_lfProj); + + Action restore = AddToConfigField("ro-MD", "senses", "scientificName"); + try + { + sense["scientificName"] = new BsonDocument { + { "ro-Latn-MD", new BsonDocument { { "value", "the export's value" } } }, + { "ro-MD", new BsonDocument { { "value", "" } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow.AddMinutes(1); + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + var entry = _cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)) as ILexEntry; + Assert.That(entry.SensesOS[0].ScientificName.Text, Is.EqualTo(original)); + Assert.That(_env.Logger.GetMessages(), Does.Contain( + "MongoToLcm: keys \"ro-Latn-MD\" and \"ro-MD\" name the same writing system, and \"ro-MD\", the one to " + + "write, is empty; leaving the field as it was rather than write \"ro-Latn-MD\" (the export's value)")); + Assert.That(_env.Logger.GetMessages(), Does.Not.Contain("names no writing system LCM has")); + } + finally + { + restore(); + } } [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)] @@ -583,6 +630,26 @@ public void BestStringAndWsId_ValueClearedUnderTheConfigsSpelling_IsNotReplacedB Assert.That(best.Value, Is.Null, "nothing should be placed for the cleared writing system"); } + [Test] + public void KeysPassedOver_NamesEachKeyWithTextLeftOutAndWhy() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("nl-Latn-SR", "nl-SR"); + var multiText = new LfMultiText { + { "nl-Latn-SR", LfStringField.FromString("the export's value") }, + { "nl-SR", LfStringField.FromString("") }, + { "zzz-x-nonesuch", LfStringField.FromString("unplaceable") }, + { "en", LfStringField.FromString("placed") }, + { "fr", LfStringField.FromString("") }, + }; + var configuredTags = new HashSet(StringComparer.Ordinal) { "nl-SR", "en", "fr" }; + + var passedOver = multiText.KeysPassedOver(_cache.WritingSystemFactory, configuredTags); + + Assert.That(passedOver, Is.EquivalentTo(new[] { + new KeyValuePair("nl-Latn-SR", "nl-SR"), + new KeyValuePair("zzz-x-nonesuch", null) })); + } + [Test] public void SpanStrToTsString_NonCanonicalSpanLang_UsesTheCanonicalWritingSystem() { diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index 024a7a54..b551294e 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -116,7 +116,9 @@ public static void Initialize() } private readonly Dictionary _storedInputSystems = new Dictionary(); - private readonly Dictionary _storedLfLexEntries = new Dictionary(); + // Held as documents, as Mongo holds them: an entry LF wrote keeps what LfMerge's own + // serializer would leave out, such as a multitext holding an empty value + private readonly Dictionary _storedLfLexEntries = new Dictionary(); private readonly Dictionary _storedLfOptionLists = new Dictionary(); private Dictionary _storedCustomFieldConfig = new Dictionary(); private Dictionary _storedLastSyncDate = new Dictionary(); @@ -190,30 +192,36 @@ public Dictionary GetCustomFieldConfig(ILfProject pro return _storedCustomFieldConfig; } - public void UpdateMockLfLexEntry(BsonDocument mockData) - { - LfLexEntry data = BsonSerializer.Deserialize(mockData); - UpdateMockLfLexEntry(data); - } - // The user SampleData's entries were last modified by private static readonly ObjectId LfUserRef = ObjectId.Parse("561b666c0f87096a35c3cf2d"); /// /// Store an entry the way an LF user's edit does: LF's MapperModel::write stamps DateModified, /// and LexEntryCommands::updateEntry records who made the edit. LfMerge's own writes go - /// through UpdateRecord, which does neither. + /// through UpdateRecord, which does neither. The document is stored as given, empty values + /// and all, since LF writes what its editor sends. /// - public void UpdateMockLfLexEntry(LfLexEntry mockData) + public void UpdateMockLfLexEntry(BsonDocument mockData) { - Guid guid = mockData.Guid ?? Guid.Empty; - var stored = DeepCopy(mockData); - stored.DateModified = DateTime.UtcNow; - if (stored.AuthorInfo != null && stored.AuthorInfo.ModifiedByUserRef == null) - stored.AuthorInfo.ModifiedByUserRef = LfUserRef; + var stored = (BsonDocument)mockData.DeepClone(); + Guid guid = BsonSerializer.Deserialize(stored).Guid ?? Guid.Empty; + stored["dateModified"] = new BsonDateTime(DateTime.UtcNow); + BsonValue authorInfo; + if (stored.TryGetValue("authorInfo", out authorInfo) && authorInfo.IsBsonDocument) + { + BsonValue userRef; + if (!authorInfo.AsBsonDocument.TryGetValue("modifiedByUserRef", out userRef) || userRef.IsBsonNull) + authorInfo.AsBsonDocument["modifiedByUserRef"] = LfUserRef; + } _storedLfLexEntries[guid] = stored; } + /// As UpdateMockLfLexEntry(BsonDocument), for an entry built as an object. + public void UpdateMockLfLexEntry(LfLexEntry mockData) + { + UpdateMockLfLexEntry(mockData.ToBsonDocument()); + } + public void UpdateMockOptionList(BsonDocument mockData) { LfOptionList data = BsonSerializer.Deserialize(mockData); @@ -228,14 +236,14 @@ public void UpdateMockOptionList(LfOptionList mockData) public IEnumerable GetLfLexEntries() { - return new List(_storedLfLexEntries.Values.Select(entry => DeepCopy(entry))); + return new List(_storedLfLexEntries.Values.Select(entry => BsonSerializer.Deserialize(entry))); } public LfLexEntry GetLfLexEntryByGuid(ILfProject _project, Guid key) { - LfLexEntry result; + BsonDocument result; if (_storedLfLexEntries.TryGetValue(key, out result)) - return result; + return BsonSerializer.Deserialize(result); return null; } @@ -281,7 +289,8 @@ private IEnumerable Records(string collectionName) public Dictionary GetAllModifiedDatesForEntries(ILfProject project) { - return _storedLfLexEntries.ToDictionary(kv => kv.Key, kv => kv.Value.AuthorInfo.ModifiedDate); + return _storedLfLexEntries.ToDictionary(kv => kv.Key, + kv => BsonSerializer.Deserialize(kv.Value).AuthorInfo.ModifiedDate); } public LfOptionList GetLfOptionListByCode(ILfProject project, string listCode) @@ -308,7 +317,7 @@ public IMongoDatabase GetMainDatabase() public bool UpdateRecord(ILfProject project, LfLexEntry data) { - _storedLfLexEntries[data.Guid ?? Guid.Empty] = DeepCopy(data); + _storedLfLexEntries[data.Guid ?? Guid.Empty] = data.ToBsonDocument(); return true; } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index 7947dba6..b301b2b3 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -405,7 +405,13 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui int foundWsId = kv.Key; string foundData = kv.Value ?? string.Empty; if (foundWsId == 0) - return false; // Skip any unidentified writing systems + { + // Nothing placeable, so the field is left as it was; if LF holds text, say why + if (!valueAsMultiText.IsEmpty) + ConvertMongoToLcmLexicon.LogTextNotPlaced(valueAsMultiText, "Custom field " + fieldName, + logger, servLoc.WritingSystemFactory, configuredTags); + return false; + } ITsString oldValue = data.get_StringProp(hvo, flid); ITsString newValue = ConvertMongoToLcmTsStrings.SpanStrToTsString(foundData, foundWsId, servLoc.WritingSystemFactory); if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) // GetDiffsInTsStrings() returns null when there are no changes diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index d0ca0f70..cfc03282 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -815,6 +815,12 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool /// field keeps its current value. Neither alternative will do: building the string in /// writing system 0 throws, which abandons the rest of the entry half-written, and clearing /// the field would delete what LCM holds on account of text that could not be placed. + /// + /// The field also keeps its value when LF's text is all under keys passed over for an empty + /// key naming the same writing system. That empty key may be a clear made under the config's + /// spelling, or the empty value LF's editor fills in for every configured input system an + /// entry lacks, beside the export's value under the Id; nothing here tells the two apart, and + /// keeping what LCM holds loses nothing that LF can show. /// private ITsString BestTsStringFromMultiText(LfMultiText input, ITsString current, bool isAnalysisField = true) { @@ -835,18 +841,35 @@ private string BestStringFromMultiText(LfMultiText input, string current, bool i /// /// Whether LF holds no text at all here. When it does hold text that nevertheless could not - /// be placed, says so in the log, since that text is not reaching FieldWorks. + /// be placed, says why in the log, since that text is not reaching FieldWorks. /// private bool NothingToPlace(LfMultiText input) { if (input == null || input.IsEmpty) return true; - Logger.Warning("MongoToLcm: text in writing system(s) {0} has nowhere to go, since LCM has none " + - "of them; leaving the field as it was", - string.Join(", ", input.Where(kv => kv.Value != null && !kv.Value.IsEmpty).Select(kv => kv.Key))); + LogTextNotPlaced(input, "MongoToLcm", Logger, ServiceLocator.WritingSystemFactory, _configuredTags); return false; } + /// + /// Logs why each key holding text in a single-string field's multitext was not written + /// there, when none was. + /// + internal static void LogTextNotPlaced(LfMultiText input, string context, ILogger logger, + ILgWritingSystemFactory wsManager, ISet configuredTags) + { + foreach (KeyValuePair passedOver in input.KeysPassedOver(wsManager, configuredTags)) + { + if (passedOver.Value == null) + logger.Warning("{0}: skipping text under \"{1}\" ({2}), which names no writing system LCM has; " + + "leaving the field as it was", context, passedOver.Key, input.Excerpt(passedOver.Key)); + else + logger.Warning("{0}: keys \"{1}\" and \"{2}\" name the same writing system, and \"{2}\", the one " + + "to write, is empty; leaving the field as it was rather than write \"{1}\" ({3})", + context, passedOver.Key, passedOver.Value, input.Excerpt(passedOver.Key)); + } + } + // This GetOrCreate() function takes an extra out parameter so we can correctly update // the entry counts in LfLexEntryToLcmLexEntry(). We don't update the counts here // because we don't yet know if the LF entry was deleted (in which case we wouldn't diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 002d46d3..02e6b5d9 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -130,29 +130,12 @@ public string FirstNonEmptyString() public KeyValuePair BestStringAndWsId(IEnumerable wsSearchOrder, ILgWritingSystemFactory wsManager, ISet configuredTags = null) { - // Each writing system's key, in the order the writing systems first appear - var keyFor = new Dictionary(); - var order = new List(); - foreach (string key in Keys) - { - int wsId = LanguageTags.WsIdFromLfTag(wsManager, key); - if (wsId == 0) - continue; - string other; - if (keyFor.TryGetValue(wsId, out other)) - keyFor[wsId] = PreferredKey(other, key, configuredTags); - else - { - keyFor[wsId] = key; - order.Add(wsId); - } - } var resolved = new List>(); - foreach (int wsId in order) + foreach (KeyValuePair chosen in ChooseKeys(wsManager, configuredTags)) { - LfStringField field = this[keyFor[wsId]]; + LfStringField field = this[chosen.Value]; if (field != null && !field.IsEmpty) - resolved.Add(new KeyValuePair(wsId, field.Value)); + resolved.Add(new KeyValuePair(chosen.Key, field.Value)); } foreach (int wsId in wsSearchOrder) { @@ -232,25 +215,7 @@ public bool WriteToLcm(IEnumerable existingWsIds, Func getA Action onKeyNotWritten = null) { // Each writing system's key, chosen before anything is written - var keyFor = new Dictionary(); - foreach (string key in Keys) - { - int wsId = LanguageTags.WsIdFromLfTag(wsManager, key); - if (wsId == 0) - { - onUnidentifiedTag?.Invoke(key); - continue; - } - string other; - if (keyFor.TryGetValue(wsId, out other)) - { - string kept = PreferredKey(other, key, configuredTags); - onKeyNotWritten?.Invoke(kept == key ? other : key, kept); - keyFor[wsId] = kept; - } - else - keyFor[wsId] = key; - } + List> keyFor = ChooseKeys(wsManager, configuredTags, onUnidentifiedTag, onKeyNotWritten); var wsIdsToClear = new HashSet(existingWsIds); bool changed = false; @@ -287,6 +252,62 @@ public string Excerpt(string key) return text.Length <= 40 ? text : text.Substring(0, 40) + "..."; } + /// + /// The keys holding text that passes over, each with the + /// reason: null where the key names no writing system LCM has, or else the key naming the + /// same writing system that is chosen over it. + /// + public List> KeysPassedOver(ILgWritingSystemFactory wsManager, + ISet configuredTags = null) + { + var chosenFor = ChooseKeys(wsManager, configuredTags).ToDictionary(kv => kv.Key, kv => kv.Value); + var passedOver = new List>(); + foreach (KeyValuePair kv in this) + { + if (kv.Value == null || kv.Value.IsEmpty) + continue; + int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); + if (wsId == 0) + passedOver.Add(new KeyValuePair(kv.Key, null)); + else if (chosenFor[wsId] != kv.Key) + passedOver.Add(new KeyValuePair(kv.Key, chosenFor[wsId])); + } + return passedOver; + } + + /// + /// Each writing system's key, in the order the writing systems first appear: the only key + /// naming it, or of two, the one picks. + /// + private List> ChooseKeys(ILgWritingSystemFactory wsManager, ISet configuredTags, + Action onUnidentifiedTag = null, Action onKeyNotWritten = null) + { + var keyFor = new Dictionary(); + var order = new List(); + foreach (string key in Keys) + { + int wsId = LanguageTags.WsIdFromLfTag(wsManager, key); + if (wsId == 0) + { + onUnidentifiedTag?.Invoke(key); + continue; + } + string other; + if (keyFor.TryGetValue(wsId, out other)) + { + string kept = PreferredKey(other, key, configuredTags); + onKeyNotWritten?.Invoke(kept == key ? other : key, kept); + keyFor[wsId] = kept; + } + else + { + keyFor[wsId] = key; + order.Add(wsId); + } + } + return order.Select(wsId => new KeyValuePair(wsId, keyFor[wsId])).ToList(); + } + /// /// Of two keys naming one writing system, the one to write: the one spelled as the config /// spells it, or else the later one. LF's editor matches spellings exactly, so this does too. From 26c2bb345575351954702f7b9a9a9a86d6d4803b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 06:14:07 +0700 Subject: [PATCH 43/51] Test that the config's spellings reach every writer, through a sync Every test of choosing between two keys for one writing system handed LfMultiText a set of configured spellings built by hand. Had ConfiguredTags returned nothing, or a writer stopped being given the set, those tests would still have passed while a sync quietly went back to writing the later key. This test runs a whole sync with the config's spelling added to the field's input systems and put first in the entry, so that going on key order alone writes the export's value instead. It covers the built-in multitext fields (picture captions share their writer), the single-string fields, and the multitext and string custom fields. With ConfiguredTags made to return an empty set, all four cases fail. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 159d3a08..273e3c31 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -10,6 +10,7 @@ using NUnit.Framework; using SIL.LCModel; using SIL.LCModel.Infrastructure; +using SIL.LCModel.Core.KernelInterfaces; using SIL.LCModel.Core.Text; using SIL.LCModel.Core.WritingSystems; @@ -412,6 +413,93 @@ public void WriteToLcm_TwoKeysForOneWritingSystem(string configured, bool idFirs Assert.That(reported, Is.EqualTo(new[] { (expected == id ? languageTag : id, expected) })); } + /// + /// The config's spellings reach every writer that chooses between two keys naming one + /// writing system, through a whole sync: the built-in multitext fields, and picture captions + /// with them; the single-string fields; and the multitext and string custom fields. The + /// config's spelling comes first here, so a writer going on key order alone would write the + /// export's value instead. + /// + // A pair each, since writing systems a test adds stay in the fixture's project + [TestCase("gloss", "hu-Latn-SK", "hu-SK")] + [TestCase("scientificName", "sk-Latn-HU", "sk-HU")] + [TestCase("customMultiText", "sl-Latn-IT", "sl-IT")] + [TestCase("customString", "et-Latn-FI", "et-FI")] + public void MongoToLcm_TwoKeysForOneWritingSystem_WritesTheConfigsSpellingInEveryKindOfField( + string field, string id, string languageTag) + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, languageTag); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, languageTag), Is.EqualTo(ws.Handle), + "precondition: the LanguageTag spelling should resolve to the same writing system"); + var twoKeys = new BsonDocument { + { languageTag, new BsonDocument { { "value", "the user's edit" } } }, + { id, new BsonDocument { { "value", "the export's value" } } } }; + var data = new SampleData(); + BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument; + BsonDocument customFields = data.bsonTestData["customFields"].AsBsonDocument; + string[] configPath; + switch (field) + { + case "gloss": + sense["gloss"] = twoKeys; + configPath = new[] { "senses", "gloss" }; + break; + case "scientificName": + sense["scientificName"] = twoKeys; + configPath = new[] { "senses", "scientificName" }; + break; + case "customMultiText": + customFields["customField_entry_Cust_Single_Line_All"] = twoKeys; + configPath = new[] { "customField_entry_Cust_Single_Line_All" }; + break; + default: + customFields["customField_entry_Cust_Single_Line"] = twoKeys; + configPath = new[] { "customField_entry_Cust_Single_Line" }; + break; + } + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + Action restore = AddToConfigField(languageTag, configPath); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + var entry = (ILexEntry)_cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)); + ISilDataAccess sda = _cache.DomainDataByFlid; + string written; + switch (field) + { + case "gloss": + written = entry.SensesOS[0].Gloss.get_String(ws.Handle).Text; + break; + case "scientificName": + written = entry.SensesOS[0].ScientificName.Text; + break; + case "customMultiText": + written = sda.get_MultiStringAlt(entry.Hvo, EntryCustomFieldId("Cust Single Line All"), ws.Handle).Text; + break; + default: + written = sda.get_StringProp(entry.Hvo, EntryCustomFieldId("Cust Single Line")).Text; + break; + } + Assert.That(written, Is.EqualTo("the user's edit")); + } + finally + { + restore(); + } + } + + private int EntryCustomFieldId(string name) + { + int flid = _cache.MetaDataCacheAccessor.GetFieldId2(LexEntryTags.kClassId, name, false); + Assert.That(flid, Is.Not.EqualTo(0), "testlangproj should have the custom field {0}", name); + return flid; + } + /// /// The xkk-flex-2022 layout, carried all the way round: one writing system whose Id is not /// its LanguageTag, and a second whose Id IS that LanguageTag, each with text of its own. From ac41521489f595b86067b4e906a15a92057a1bda Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 06:14:54 +0700 Subject: [PATCH 44/51] Make the pronunciation custom-field test fail without the rule it tests The test's writing system, "kal-fonipa", is a phonetic spelling of the vernacular, so language affinity made it vernacular only whether or not it had pronunciation text; with the pronunciation rule removed from ClassifyByText, the test still passed. It now uses "xyz-fonipa", a language the project has nowhere else, and runs both with and without the pronunciation text: vernacular only with it, both and unresolved without. With the rule removed, the case with pronunciation text fails. Co-Authored-By: Claude Opus 5.5 --- .../VernacularClassificationTests.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 42222d39..7da5efcd 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -551,20 +551,29 @@ public void PronunciationTextMakesAWritingSystemVernacular() /// field cannot put it in doubt. In the 2026-10-01 corpus every writing system in that /// position is a pronunciation writing system -- a phonetic, IPA or audio one -- for which /// vernacular only is the right answer. + /// + /// The writing system's language is one the project has nowhere else: a phonetic spelling of + /// the vernacular, "kal-fonipa", would be made vernacular only by its language whether or not + /// it had pronunciation text, and the test would not show the rule at work. /// - [Test] - public void PronunciationTextAloneMakesAWritingSystemOfferedOnlyInACustomFieldVernacularOnly() + [TestCase(true)] + [TestCase(false)] + public void PronunciationTextAloneMakesAWritingSystemOfferedOnlyInACustomFieldVernacularOnly(bool hasPronunciationText) { var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), - ("customField_senses_Phonetic", Field("kal-fonipa"))); - var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), - (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "kal-fonipa", 1)); + ("customField_senses_Phonetic", Field("xyz-fonipa"))); + var counts = new List<(string, string, int)> { + (LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10) }; + if (hasPronunciationText) + counts.Add((LfWritingSystemUsage.Pronunciation, "xyz-fonipa", 1)); + var usage = Usage(counts.ToArray()); var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); - Assert.That(result.Vernacular, Contains.Item("kal-fonipa")); - Assert.That(result.Analysis, Does.Not.Contain("kal-fonipa")); - Assert.That(result.Unresolved, Does.Not.Contain("kal-fonipa")); + Assert.That(result.Vernacular, Contains.Item("xyz-fonipa")); + Assert.That(result.Analysis.Contains("xyz-fonipa"), Is.EqualTo(!hasPronunciationText), + "with pronunciation text, vernacular only; without, both"); + Assert.That(result.Unresolved.Contains("xyz-fonipa"), Is.EqualTo(!hasPronunciationText)); } /// From 54ef8165feefa1bf1bc276d81edc51b86d4e9f94 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 06:16:31 +0700 Subject: [PATCH 45/51] Test that the lexicon is counted once however many writing systems are new MongoToLcm_InputSystemNewToLcm_CountsTheLexiconToo added a single new input system, so it expected two reads of the lexicon whether or not the classification was kept after the first new writing system. Dropping that, and counting the whole lexicon again for every new writing system, would have gone unnoticed. The test now adds two and still expects two reads; without the cache it sees three. Co-Authored-By: Claude Opus 5.5 --- .../ConvertMongoToLcmWritingSystemsTests.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs index 273e3c31..40cd4c3a 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -848,23 +848,33 @@ public void MongoToLcm_NoInputSystemNewToLcm_ReadsTheLexiconOnce() Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(1)); } + /// + /// With writing systems new to LCM, the lexicon is counted once more, and only once however + /// many there are: the classification is worked out for the first and kept for the rest. + /// [Test] - public void MongoToLcm_InputSystemNewToLcm_CountsTheLexiconToo() + public void MongoToLcm_InputSystemsNewToLcm_CountTheLexiconOnceMore() { - const string tag = "qaa-x-new"; - Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag); - Action restore = AddLfInputSystem(tag); + var tags = new[] { "qaa-x-new", "qaa-x-newtoo" }; + var restores = new List(); + foreach (string tag in tags) + { + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag); + restores.Add(AddLfInputSystem(tag)); + } try { int readsBefore = _conn.LexiconReads; SutMongoToLcm.Run(_lfProj); + Assert.That(LcmWritingSystemIds(), Is.SupersetOf(tags), "precondition: both should have been created"); Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(2)); } finally { - restore(); + foreach (Action restore in restores) + restore(); } } } From 6760be9d7edbc13aa1bb938383fa4c1c185f70b7 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 07:34:28 +0700 Subject: [PATCH 46/51] Bring the last comments the code review found out of date in line The field loop's comment, and the docstring of the test covering it, said "en" is not made doubtful by turning up in "a note". The entry's note is an analysis anchor, which the loop skips; the field meant, and the one the test uses, is a sense's general note. LogTextNotPlaced's messages are right only when nothing was written, which both its callers ensure but its doc did not say. SetLastSyncedDate's comment still compared it with DeepCopy, which the double no longer uses for entries. Co-Authored-By: Claude Opus 5.5 --- .../DataConverters/VernacularClassificationTests.cs | 4 ++-- src/LfMerge.Core.Tests/TestDoubles.cs | 2 +- .../DataConverters/ConvertMongoToLcmLexicon.cs | 11 +++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs index 7da5efcd..3494b1a0 100644 --- a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -663,8 +663,8 @@ public void WritingSystemsWhoseOnlyCompanyIsEachOtherStayUnresolved() } /// - /// "en" is in both anchors, so it plays both roles. Turning up alone in a note, a field - /// that gives no evidence, must not put it in doubt: if it did, the company it keeps in the + /// "en" is in both anchors, so it plays both roles. Turning up alone in a sense's general + /// note, a field that gives no evidence, must not put it in doubt: if it did, the company it keeps in the /// lexeme field (the vernacular) would then strip it of its analysis role, and a new "en" /// writing system would be created vernacular only. /// diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index b551294e..9584c5e2 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -335,7 +335,7 @@ public bool RemoveRecord(ILfProject project, Guid guid) public bool SetLastSyncedDate(ILfProject project, DateTime? newSyncedDate) { - // Mongo keeps dates to the millisecond, as DeepCopy does for the entries' dates + // Mongo keeps dates to the millisecond, as the entries' dates are kept here in BSON if (newSyncedDate != null) newSyncedDate = newSyncedDate.Value.AddTicks(-(newSyncedDate.Value.Ticks % TimeSpan.TicksPerMillisecond)); _storedLastSyncDate[project.ProjectCode] = newSyncedDate; diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index cfc03282..1e19f170 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -340,9 +340,10 @@ public static (ISet Vernacular, ISet Analysis, ISet Unre } // A field of no role puts in doubt only the writing systems nothing else places, neither // the anchors nor a field that has a role. One already placed keeps its place: "en" in - // both the lexeme and the gloss is not made doubtful by also turning up in a note, and if - // it were, the passes below could strip one of its roles on the strength of the company - // it keeps there. + // both the lexeme and the gloss is not made doubtful by also turning up in a sense's + // general note, and if it were, the passes below could strip one of its roles on the + // strength of the company it keeps there. (The entry's note is an anchor, so never comes + // this far.) var doubtful = inFieldOfNoRole .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag) && !inVernacularField.Contains(tag) && !inAnalysisField.Contains(tag)) @@ -853,7 +854,9 @@ private bool NothingToPlace(LfMultiText input) /// /// Logs why each key holding text in a single-string field's multitext was not written - /// there, when none was. + /// there. Only for when nothing was: every key with text then either names no writing system + /// LCM has or was passed over for an empty key, and the messages say so. Called after a + /// value has been placed, it would call the key written empty. /// internal static void LogTextNotPlaced(LfMultiText input, string context, ILogger logger, ILgWritingSystemFactory wsManager, ISet configuredTags) From 96f584d89d6d320c44492a9d13c7b5d8e25bbc5f Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 07:35:43 +0700 Subject: [PATCH 47/51] Delete the dead FW8_COMPAT blocks Nothing defines FW8_COMPAT, and LfMerge has not built against FieldWorks 8 for years, so the #if branches naming IWritingSystemManager and IWritingSystem were never compiled. Only the FieldWorks 9 types are left, and with them goes the comment explaining why the #if blocks were there. Co-Authored-By: Claude Opus 5.5 --- .../ConvertLcmToMongoCustomField.cs | 4 ---- .../ConvertMongoToLcmLexicon.cs | 21 ------------------- 2 files changed, 25 deletions(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs index 2aed584f..f560084f 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs @@ -156,11 +156,7 @@ Dictionary lfCustomFieldTypes // Single line or MultiText fields need writing systems int fieldWs = LcmMetaData.GetFieldWs(flid); // That's a "magic" ws, which we need to expand into a (list of) real writing system(s). -#if FW8_COMPAT - var wsesForThisField = new List(); -#else var wsesForThisField = new List(); -#endif // GetWritingSystemList() in FW 8.3 is buggy and doesn't properly handle the kwsAnal and kwsVern cases, so we handle them here instead. switch (fieldWs) { case WritingSystemServices.kwsAnal: diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 1e19f170..359b3111 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -632,24 +632,7 @@ private static void CollectInputSystems(LfConfigFieldList fieldList, string path private void LfWsToLcmWs(Dictionary lfWsList, Func countUsage = null) { - // Between FW 8.2 and 9, a few classes and interfaces were renamed. The ones most relevant here are - // IWritingSystemManager (interface was removed and replaced with the WritingSystemManager concrete class), - // and PalasoWritingSystem which was replaced with CoreWritingSystemDefinition. Since their internals - // didn't change much (and the only changes were in areas we don't access), we can use a simple compiler - // define to choose the type of the wsm variable here (and the ws variable later) and we're fine. - // HOWEVER, if the code inside #if...#endif blocks starts to grow, this is not an ideal solution. A better - // solution if the code grows complex will be to write several classes to the same interface, each of which - // can deal with one particular version of FW or Lcm. Register them all with Autofac with a way to choose among them - // (http://docs.autofac.org/en/stable/register/registration.html#selection-of-an-implementation-by-parameter-value) - // and then, at runtime, we can instantiate the particular class that's needed for dealing with *this* FW project. - // - // But for now, these #if...#endif blocks are enough. - 2016-03 RM -#if FW8_COMPAT - // Note that we can't use ILgWritingSystemFactory here, because it doesn't have some methods we need later on. - IWritingSystemManager wsManager = ServiceLocator.WritingSystemManager; -#else WritingSystemManager wsManager = ServiceLocator.WritingSystemManager; -#endif if (wsManager == null) { Logger.Error("Failed to find the writing system manager"); @@ -664,11 +647,7 @@ private void LfWsToLcmWs(Dictionary lfWsList, // TODO: Split the inside of this foreach() out into its own function foreach (var lfWs in lfWsList.Values) { -#if FW8_COMPAT - IWritingSystem ws; -#else CoreWritingSystemDefinition ws; -#endif // It would be nice to call this to add to both analysis and vernacular WS. // But we need the flexibility of bringing in LF WS properties. From 499b2ea1bd0bce705b8509b748fc410626702e18 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 09:38:53 +0700 Subject: [PATCH 48/51] Add a mongosh script to re-spell LF input systems as LCM Ids LfMerge now exports every writing system by its LCM Id. It rewrites the inputSystems record and custom-field config on every sync, but the built-in fields' config only at initial clone, so projects cloned while LfMerge spelled input systems by LanguageTag keep a config LF's editor cannot match against the lexicon's Id-spelled keys. migrate-input-systems-to-lcm-ids.js re-spells, for the fifteen projects the 2026-07-06 and 2026-10-01 mongodumps show are affected, every config inputSystems array, the inputSystems record and languageCode, each mapping checked against the project's FieldWorks repository. In che-flex, whose lexicon is still keyed by the Id its Chechen writing system had before FieldWorks moved the text to "ce-Latn", it also renames the lexicon's keys in place, leaving entries' dates alone. It is a dry run unless DRY_RUN=false, leaves an inputSystems entry alone where the new spelling is already there, and changes nothing when run a second time. Run it before the new LfMerge build first syncs these projects, and keep them on HOLD from before it runs until that build is deployed: the old build's Replace call would evict or re-register writing systems keyed by Id. Co-Authored-By: Claude Opus 5.5 --- migrate-input-systems-to-lcm-ids.js | 362 ++++++++++++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 migrate-input-systems-to-lcm-ids.js diff --git a/migrate-input-systems-to-lcm-ids.js b/migrate-input-systems-to-lcm-ids.js new file mode 100644 index 00000000..0827c019 --- /dev/null +++ b/migrate-input-systems-to-lcm-ids.js @@ -0,0 +1,362 @@ +// Re-spell Language Forge input systems as the LCM writing system Ids LfMerge now exports. +// +// LfMerge wrote each project's input systems -- the project's inputSystems record, its languageCode, +// and, at initial clone, every config field's inputSystems list -- by the writing system's +// LanguageTag, but wrote every multitext key by its Id. Where the two differ (brb-flex-2022's +// Khmer is Id "km-Khmr-KH", LanguageTag "km-KH") LF's editor, which matches a field's input systems +// to the entry's keys exactly and case-sensitively, shows that text as empty. LfMerge now spells +// everything by Id. It rewrites the inputSystems record and custom-field config on every sync, but +// built-in field config only at initial clone, so projects cloned under the old spelling need +// their config re-spelled once. This script does that for the twelve projects the 2026-07-06 +// mongodump shows are affected, and re-keys the inputSystems record of three more, found in the +// 2026-10-01 mongodump, whose config already spells the writing system by Id. +// +// Each mapping was found in a dump -- the old spelling is the one LF's config or inputSystems +// record uses, the new one the Id LfMerge wrote into the lexicon's multitext keys -- and then +// checked against the project's FieldWorks repository at its tip, since the dump shows LCM only as +// it was at LF's last sync. +// The lexicon's keys are already spelled by Id and are not touched, except in che-flex; see +// LEXICON_REKEYS below. +// +// Run this BEFORE the new LfMerge build first syncs these projects. After that sync, the +// inputSystems record is spelled by Id, and in kxpw-flex the old spelling "acp" is by then the Id +// of a different writing system, added in FieldWorks after LF's clone. The script leaves an +// inputSystems entry alone rather than overwrite one already spelled the new way. +// +// And do not let the OLD LfMerge build sync these projects once this script has run: keep every +// project listed here on HOLD from before the script runs until the new build is deployed. The old +// build looks each inputSystems key up as an Id and then calls WritingSystemManager.Replace, which +// looks the writing system up again by LanguageTag. Keyed by the old spelling, as now, both find +// the same writing system. Keyed by Id, Replace finds a different one wherever another writing +// system has the LanguageTag as its Id -- brb-flex-2022's unused "km-KH", tkr-flex's unused +// "tkr", kxpw-flex's "acp" -- and evicts it, giving the renamed writing system its handle; every +// other renamed writing system is re-registered under a new handle partway through the sync. That +// is the corruption that broke xkk-flex-2022. The old build's export would also key the +// inputSystems record by LanguageTag again, undoing the renames. +// +// What is changed, for each project: +// - every array named "inputSystems" anywhere under config (entry fields, custom fields, role +// views and user views), replacing the old spelling with the new one, without duplicating it; +// - the project's inputSystems record: the old key is renamed to the new one and its tag updated +// (LfMerge would rewrite this on its next sync anyway; doing it here avoids a window in which +// LF shows bare tags instead of abbreviations); +// - languageCode, where it is the old spelling; +// - in che-flex alone, the lexicon's multitext keys (see LEXICON_REKEYS). +// Only exact matches of the old spellings are replaced. Running it twice changes nothing the +// second time. +// +// Usage (dry run by default -- it reports what it would change and changes nothing): +// mongosh "mongodb://HOST:PORT" migrate-input-systems-to-lcm-ids.js +// DRY_RUN=false mongosh "mongodb://HOST:PORT" migrate-input-systems-to-lcm-ids.js +// LF_DB selects the main database (default "scriptureforge"). + +const DRY_RUN = (process.env.DRY_RUN || 'true').toLowerCase() !== 'false'; +const DB_NAME = process.env.LF_DB || 'scriptureforge'; + +// projectCode -> { old spelling in LF's config : LCM writing system Id } +// awa-flex_lf is deliberately absent: its repository was begun afresh in March 2021 and renamed +// hi-Deva-IN to hi-IN a week later, so its config's "hi-IN" already matches LCM. +// huz-flex is absent too, though its languageCode is "huz-Cyrl" while its config says "huz": its +// repository holds both, as two writing systems, so that is not a spelling to correct. +const MIGRATIONS = { + 'ary-mo-flex': { 'ary-MO-fonipa-x-emic': 'ary-Arab-MO-fonipa-x-emic' }, + 'brb-flex-2022': { 'brb-KH': 'brb-Khmr-KH', 'km-KH': 'km-Khmr-KH' }, + // Not a respelling but a rename in FieldWorks: LF was cloned when this writing system was "ce-la" + // (LA being Laos), and in November 2020 its text moved to "ce-Latn", which now holds all of it. + 'che-flex': { 'ce-LA': 'ce-Latn' }, + 'jaijairai-flex2': { 'my-MM': 'my-Mymr-MM' }, + 'ket-flex': { 'ket': 'ket-Cyrl' }, + // Lower-case "k" is right: it is the name of the .ldml file in the WritingSystemStore LCM loads, + // and so the Id LfMerge exported all 14,493 multitext keys under. FieldWorks on Windows spells + // it "mve-Arab-PK" (its CachedSettings copy), but LF must match what LfMerge exports. + 'mve-n-flex': { 'mve-PK': 'mve-Arab-Pk' }, + 'kxpw-flex': { 'acp': 'acp-Latn', 'kxp-PK': 'kxp-Arab-PK' }, + 'odk-flex': { 'odk-PK': 'odk-Arab-PK' }, + 'oyb-flex': { 'lo-LA-fonipa-x-emic': 'lo-Laoo-LA-fonipa-x-emic' }, + 'spt-flex': { 'hi-IN': 'hi-Deva-IN', 'spt-x-tib': 'spt-Tibt-x-tib' }, + 'tmy-2020-flex': { 'tmy': 'tmy-Latn' }, + 'yor-mw-flex': { 'yo': 'yo-Latn' }, + // The next three need only their inputSystems record re-keyed: the config already spells the + // writing system by Id, but the record was last written by an LfMerge that keyed it by + // LanguageTag. Without this, the config's spelling has no input system record in LF, and the new + // LfMerge finds the writing system only by its LanguageTag, which depends on SLDR data. + // The 2026-09-29 sync wrote "rhg"; LCM's Hanifi Rohingya writing system is "rhg-Rohg". + 'rhg-flex': { 'rhg': 'rhg-Rohg' }, + // LCM holds two: "tkr-Latn", with all 3,427 strings and in the vernacular lists, and an unused + // "tkr". Keyed "tkr", the record names the unused one. + 'tkr-flex': { 'tkr': 'tkr-Latn' }, + // "rif-Latn" holds 249 strings and is in the analysis list; the config does not offer it. + 'sjs-flex': { 'rif': 'rif-Latn' }, +}; + +// projectCode -> { multitext key in LF's lexicon : LCM writing system Id } +// che-flex's lexicon is still keyed "ce-la", the Id LfMerge exported under before the text moved +// to "ce-Latn" in FieldWorks; LCM's "ce-la" now holds no text. LfMerge re-exports only the entries +// FieldWorks has changed since, 1,983 of 18,434 at the 2026-10-01 dump. The rest would stay keyed +// "ce-la", out of sight of a config spelled "ce-Latn", and an LF edit to one of them would write +// its old text to "ce-la" and clear its "ce-Latn" text in FieldWorks, since importing an entry +// clears every alternative LF lacks. In each of those entries every "ce-la" value is text the repo +// holds in "ce-Latn", so renaming the key gives LF what FieldWorks has. Keys are renamed in place, +// and entries' dates are left alone, so that LfMerge does not take them for LF edits. +const LEXICON_REKEYS = { + 'che-flex': { 'ce-la': 'ce-Latn' }, +}; + +// A subdocument, as opposed to an array, a date or a BSON value (ObjectId, Long and the like, which +// all carry _bsontype). +function isPlainObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value) && + !(value instanceof Date) && value._bsontype === undefined; +} + +// Collect a $set for every "inputSystems" array under node that holds an old spelling. +function collectInputSystemArrays(node, path, oldToNew, sets, warnings) { + if (Array.isArray(node)) { + node.forEach((value, i) => collectInputSystemArrays(value, path + '.' + i, oldToNew, sets, warnings)); + return; + } + if (!isPlainObject(node)) { + return; + } + for (const [key, value] of Object.entries(node)) { + const childPath = path ? path + '.' + key : key; + if (key === 'inputSystems' && Array.isArray(value)) { + if (!value.some(tag => Object.prototype.hasOwnProperty.call(oldToNew, tag))) { + continue; + } + if (childPath.split('.').some(segment => segment.includes('$'))) { + warnings.push('cannot address ' + childPath + ' in an update; left as it is'); + continue; + } + const respelled = []; + for (const tag of value) { + const newTag = Object.prototype.hasOwnProperty.call(oldToNew, tag) ? oldToNew[tag] : tag; + if (!respelled.includes(newTag)) { + respelled.push(newTag); + } + } + sets[childPath] = respelled; + } else { + collectInputSystemArrays(value, childPath, oldToNew, sets, warnings); + } + } +} + +// Every place in the project record an old spelling still appears, for the check afterwards. +function remainingOldSpellings(project, oldToNew) { + const found = []; + const olds = new Set(Object.keys(oldToNew)); + (function walk(node, path) { + if (Array.isArray(node)) { + node.forEach((value, i) => walk(value, path + '.' + i)); + } else if (isPlainObject(node)) { + for (const [key, value] of Object.entries(node)) { + if (olds.has(key)) found.push(path + '.' + key + ' (key)'); + // An input system's abbreviation and language name are labels, not spellings: sjs-flex's + // "rif-Latn" is abbreviated "rif" + if (key === 'abbreviation' || key === 'languageName') continue; + walk(value, path + '.' + key); + } + } else if (typeof node === 'string' && olds.has(node)) { + found.push(path); + } + })({ config: project.config, inputSystems: project.inputSystems, languageCode: project.languageCode }, ''); + return found; +} + +function migrateProject(projects, projectCode, oldToNew) { + const project = projects.findOne({ projectCode: projectCode }); + if (!project) { + print(' ' + projectCode + ': NOT FOUND, skipped'); + return { found: false }; + } + + const sets = {}; + const unsets = {}; + const warnings = []; + + collectInputSystemArrays(project.config, 'config', oldToNew, sets, warnings); + const arrays = Object.keys(sets).length; + + let renamed = 0; + const inputSystems = project.inputSystems || {}; + for (const [oldTag, newTag] of Object.entries(oldToNew)) { + if (!Object.prototype.hasOwnProperty.call(inputSystems, oldTag)) { + continue; + } + if (Object.prototype.hasOwnProperty.call(inputSystems, newTag)) { + // Already spelled the new way, most likely by the new LfMerge build. The entry under the old + // spelling may then be a different writing system (see kxpw-flex above), so leave both alone. + warnings.push('inputSystems already has ' + newTag + '; left ' + oldTag + ' as it is'); + continue; + } + sets['inputSystems.' + newTag] = Object.assign({}, inputSystems[oldTag], { tag: newTag }); + unsets['inputSystems.' + oldTag] = ''; + renamed++; + } + + let languageCode = null; + if (Object.prototype.hasOwnProperty.call(oldToNew, project.languageCode)) { + languageCode = { from: project.languageCode, to: oldToNew[project.languageCode] }; + sets['languageCode'] = languageCode.to; + } + + const mapping = Object.entries(oldToNew).map(([o, n]) => o + ' -> ' + n).join(', '); + print(' ' + projectCode + ': ' + mapping); + print(' config inputSystems arrays re-spelled: ' + arrays); + print(' inputSystems record entries renamed: ' + renamed); + print(' languageCode: ' + + (languageCode ? languageCode.from + ' -> ' + languageCode.to : 'unchanged')); + warnings.forEach(w => print(' WARNING: ' + w)); + + if (arrays === 0 && renamed === 0 && !languageCode) { + print(' nothing to do (already migrated?)'); + return { found: true, changed: false }; + } + if (DRY_RUN) { + return { found: true, changed: true }; + } + + const update = {}; + if (Object.keys(sets).length > 0) update.$set = sets; + if (Object.keys(unsets).length > 0) update.$unset = unsets; + const result = projects.updateOne({ _id: project._id }, update); + print(' updated: matched ' + result.matchedCount + ', modified ' + result.modifiedCount); + + const left = remainingOldSpellings(projects.findOne({ _id: project._id }), oldToNew); + if (left.length > 0) { + print(' WARNING: old spellings still present at:'); + left.forEach(p => print(' ' + p)); + } else { + print(' verified: no old spelling remains in config, inputSystems or languageCode'); + } + return { found: true, changed: true }; +} + +// A multitext: a subdocument every one of whose values is a { value: ... } string field. +function isMultiText(node) { + if (!isPlainObject(node)) return false; + const values = Object.values(node); + return values.length > 0 && values.every(v => isPlainObject(v) && Object.prototype.hasOwnProperty.call(v, 'value')); +} + +// Collect a $set for every multitext under node that has an old key, renaming the key where it +// stands so that the keys keep their order. A multitext that already has the new key as well is +// left alone and reported: which of the two to keep is not this script's call. +function collectLexiconRekeys(node, path, oldToNew, sets, warnings) { + if (Array.isArray(node)) { + node.forEach((value, i) => collectLexiconRekeys(value, path + '.' + i, oldToNew, sets, warnings)); + return; + } + if (!isPlainObject(node)) { + return; + } + if (isMultiText(node)) { + const olds = Object.keys(node).filter(key => Object.prototype.hasOwnProperty.call(oldToNew, key)); + if (olds.length === 0) { + return; + } + if (olds.some(key => Object.prototype.hasOwnProperty.call(node, oldToNew[key]))) { + warnings.push(path + ' has both spellings; left as it is'); + return; + } + if (path.split('.').some(segment => segment.includes('$'))) { + warnings.push('cannot address ' + path + ' in an update; left as it is'); + return; + } + const renamed = {}; + for (const [key, value] of Object.entries(node)) { + renamed[Object.prototype.hasOwnProperty.call(oldToNew, key) ? oldToNew[key] : key] = value; + } + sets[path] = renamed; + return; + } + for (const [key, value] of Object.entries(node)) { + // A key with a dot in it cannot be addressed by a dotted path + if (key.includes('.')) { + warnings.push('cannot address ' + path + ' key "' + key + '" in an update; left as it is'); + continue; + } + collectLexiconRekeys(value, path ? path + '.' + key : key, oldToNew, sets, warnings); + } +} + +// How many multitexts in the lexicon still have an old key. +function countOldKeys(lexicon, oldToNew) { + let count = 0; + lexicon.find({}).forEach(entry => { + (function walk(node) { + if (Array.isArray(node)) { + node.forEach(walk); + } else if (isPlainObject(node)) { + if (isMultiText(node) && Object.keys(node).some(key => Object.prototype.hasOwnProperty.call(oldToNew, key))) { + count++; + } + Object.values(node).forEach(walk); + } + })(entry); + }); + return count; +} + +function rekeyLexicon(projectCode, oldToNew) { + const mapping = Object.entries(oldToNew).map(([o, n]) => o + ' -> ' + n).join(', '); + print(' ' + projectCode + ' lexicon: ' + mapping); + const lexicon = db.getSiblingDB('sf_' + projectCode).lexicon; + let entries = 0; + let multiTexts = 0; + const warnings = []; + lexicon.find({}).forEach(entry => { + const sets = {}; + const entryWarnings = []; + for (const [key, value] of Object.entries(entry)) { + if (key === '_id') continue; + collectLexiconRekeys(value, key, oldToNew, sets, entryWarnings); + } + entryWarnings.forEach(w => warnings.push('entry ' + entry.guid + ': ' + w)); + const count = Object.keys(sets).length; + if (count === 0) { + return; + } + entries++; + multiTexts += count; + if (!DRY_RUN) { + lexicon.updateOne({ _id: entry._id }, { $set: sets }); + } + }); + print(' entries re-keyed: ' + entries); + print(' multitexts re-keyed: ' + multiTexts); + warnings.forEach(w => print(' WARNING: ' + w)); + if (!DRY_RUN && entries > 0) { + const left = countOldKeys(lexicon, oldToNew); + print(left === 0 ? ' verified: no old key remains in the lexicon' + : ' WARNING: ' + left + ' multitexts still have an old key'); + } + return entries > 0; +} + +const projects = db.getSiblingDB(DB_NAME).projects; +print((DRY_RUN ? 'DRY RUN -- nothing will be changed. ' : '') + 'Database: ' + DB_NAME); +let changed = 0; +let missing = 0; +for (const [projectCode, oldToNew] of Object.entries(MIGRATIONS)) { + const outcome = migrateProject(projects, projectCode, oldToNew); + if (!outcome.found) missing++; + else if (outcome.changed) changed++; +} +let lexiconsChanged = 0; +for (const [projectCode, oldToNew] of Object.entries(LEXICON_REKEYS)) { + if (!projects.findOne({ projectCode: projectCode })) { + print(' ' + projectCode + ' lexicon: project NOT FOUND, skipped'); + continue; + } + if (rekeyLexicon(projectCode, oldToNew)) lexiconsChanged++; +} +print(''); +print((DRY_RUN ? 'Would change ' : 'Changed ') + changed + ' of ' + Object.keys(MIGRATIONS).length + + ' projects' + (missing ? '; ' + missing + ' not found' : '') + '.'); +print((DRY_RUN ? 'Would re-key ' : 'Re-keyed ') + lexiconsChanged + ' of ' + Object.keys(LEXICON_REKEYS).length + + ' lexicons.'); +if (DRY_RUN) { + print('Run again with DRY_RUN=false to apply.'); +} From b91108ec57019e0f9e2c796a374164d67d14bb6b Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 6 Oct 2026 12:21:36 +0700 Subject: [PATCH 49/51] Remove the migration script from the repository migrate-input-systems-to-lcm-ids.js is run once against Language Forge's Mongo before this build first syncs the projects it lists, and is then of no further use, so it does not belong in LfMerge's long-lived code. Co-Authored-By: Claude Opus 5.5 --- migrate-input-systems-to-lcm-ids.js | 362 ---------------------------- 1 file changed, 362 deletions(-) delete mode 100644 migrate-input-systems-to-lcm-ids.js diff --git a/migrate-input-systems-to-lcm-ids.js b/migrate-input-systems-to-lcm-ids.js deleted file mode 100644 index 0827c019..00000000 --- a/migrate-input-systems-to-lcm-ids.js +++ /dev/null @@ -1,362 +0,0 @@ -// Re-spell Language Forge input systems as the LCM writing system Ids LfMerge now exports. -// -// LfMerge wrote each project's input systems -- the project's inputSystems record, its languageCode, -// and, at initial clone, every config field's inputSystems list -- by the writing system's -// LanguageTag, but wrote every multitext key by its Id. Where the two differ (brb-flex-2022's -// Khmer is Id "km-Khmr-KH", LanguageTag "km-KH") LF's editor, which matches a field's input systems -// to the entry's keys exactly and case-sensitively, shows that text as empty. LfMerge now spells -// everything by Id. It rewrites the inputSystems record and custom-field config on every sync, but -// built-in field config only at initial clone, so projects cloned under the old spelling need -// their config re-spelled once. This script does that for the twelve projects the 2026-07-06 -// mongodump shows are affected, and re-keys the inputSystems record of three more, found in the -// 2026-10-01 mongodump, whose config already spells the writing system by Id. -// -// Each mapping was found in a dump -- the old spelling is the one LF's config or inputSystems -// record uses, the new one the Id LfMerge wrote into the lexicon's multitext keys -- and then -// checked against the project's FieldWorks repository at its tip, since the dump shows LCM only as -// it was at LF's last sync. -// The lexicon's keys are already spelled by Id and are not touched, except in che-flex; see -// LEXICON_REKEYS below. -// -// Run this BEFORE the new LfMerge build first syncs these projects. After that sync, the -// inputSystems record is spelled by Id, and in kxpw-flex the old spelling "acp" is by then the Id -// of a different writing system, added in FieldWorks after LF's clone. The script leaves an -// inputSystems entry alone rather than overwrite one already spelled the new way. -// -// And do not let the OLD LfMerge build sync these projects once this script has run: keep every -// project listed here on HOLD from before the script runs until the new build is deployed. The old -// build looks each inputSystems key up as an Id and then calls WritingSystemManager.Replace, which -// looks the writing system up again by LanguageTag. Keyed by the old spelling, as now, both find -// the same writing system. Keyed by Id, Replace finds a different one wherever another writing -// system has the LanguageTag as its Id -- brb-flex-2022's unused "km-KH", tkr-flex's unused -// "tkr", kxpw-flex's "acp" -- and evicts it, giving the renamed writing system its handle; every -// other renamed writing system is re-registered under a new handle partway through the sync. That -// is the corruption that broke xkk-flex-2022. The old build's export would also key the -// inputSystems record by LanguageTag again, undoing the renames. -// -// What is changed, for each project: -// - every array named "inputSystems" anywhere under config (entry fields, custom fields, role -// views and user views), replacing the old spelling with the new one, without duplicating it; -// - the project's inputSystems record: the old key is renamed to the new one and its tag updated -// (LfMerge would rewrite this on its next sync anyway; doing it here avoids a window in which -// LF shows bare tags instead of abbreviations); -// - languageCode, where it is the old spelling; -// - in che-flex alone, the lexicon's multitext keys (see LEXICON_REKEYS). -// Only exact matches of the old spellings are replaced. Running it twice changes nothing the -// second time. -// -// Usage (dry run by default -- it reports what it would change and changes nothing): -// mongosh "mongodb://HOST:PORT" migrate-input-systems-to-lcm-ids.js -// DRY_RUN=false mongosh "mongodb://HOST:PORT" migrate-input-systems-to-lcm-ids.js -// LF_DB selects the main database (default "scriptureforge"). - -const DRY_RUN = (process.env.DRY_RUN || 'true').toLowerCase() !== 'false'; -const DB_NAME = process.env.LF_DB || 'scriptureforge'; - -// projectCode -> { old spelling in LF's config : LCM writing system Id } -// awa-flex_lf is deliberately absent: its repository was begun afresh in March 2021 and renamed -// hi-Deva-IN to hi-IN a week later, so its config's "hi-IN" already matches LCM. -// huz-flex is absent too, though its languageCode is "huz-Cyrl" while its config says "huz": its -// repository holds both, as two writing systems, so that is not a spelling to correct. -const MIGRATIONS = { - 'ary-mo-flex': { 'ary-MO-fonipa-x-emic': 'ary-Arab-MO-fonipa-x-emic' }, - 'brb-flex-2022': { 'brb-KH': 'brb-Khmr-KH', 'km-KH': 'km-Khmr-KH' }, - // Not a respelling but a rename in FieldWorks: LF was cloned when this writing system was "ce-la" - // (LA being Laos), and in November 2020 its text moved to "ce-Latn", which now holds all of it. - 'che-flex': { 'ce-LA': 'ce-Latn' }, - 'jaijairai-flex2': { 'my-MM': 'my-Mymr-MM' }, - 'ket-flex': { 'ket': 'ket-Cyrl' }, - // Lower-case "k" is right: it is the name of the .ldml file in the WritingSystemStore LCM loads, - // and so the Id LfMerge exported all 14,493 multitext keys under. FieldWorks on Windows spells - // it "mve-Arab-PK" (its CachedSettings copy), but LF must match what LfMerge exports. - 'mve-n-flex': { 'mve-PK': 'mve-Arab-Pk' }, - 'kxpw-flex': { 'acp': 'acp-Latn', 'kxp-PK': 'kxp-Arab-PK' }, - 'odk-flex': { 'odk-PK': 'odk-Arab-PK' }, - 'oyb-flex': { 'lo-LA-fonipa-x-emic': 'lo-Laoo-LA-fonipa-x-emic' }, - 'spt-flex': { 'hi-IN': 'hi-Deva-IN', 'spt-x-tib': 'spt-Tibt-x-tib' }, - 'tmy-2020-flex': { 'tmy': 'tmy-Latn' }, - 'yor-mw-flex': { 'yo': 'yo-Latn' }, - // The next three need only their inputSystems record re-keyed: the config already spells the - // writing system by Id, but the record was last written by an LfMerge that keyed it by - // LanguageTag. Without this, the config's spelling has no input system record in LF, and the new - // LfMerge finds the writing system only by its LanguageTag, which depends on SLDR data. - // The 2026-09-29 sync wrote "rhg"; LCM's Hanifi Rohingya writing system is "rhg-Rohg". - 'rhg-flex': { 'rhg': 'rhg-Rohg' }, - // LCM holds two: "tkr-Latn", with all 3,427 strings and in the vernacular lists, and an unused - // "tkr". Keyed "tkr", the record names the unused one. - 'tkr-flex': { 'tkr': 'tkr-Latn' }, - // "rif-Latn" holds 249 strings and is in the analysis list; the config does not offer it. - 'sjs-flex': { 'rif': 'rif-Latn' }, -}; - -// projectCode -> { multitext key in LF's lexicon : LCM writing system Id } -// che-flex's lexicon is still keyed "ce-la", the Id LfMerge exported under before the text moved -// to "ce-Latn" in FieldWorks; LCM's "ce-la" now holds no text. LfMerge re-exports only the entries -// FieldWorks has changed since, 1,983 of 18,434 at the 2026-10-01 dump. The rest would stay keyed -// "ce-la", out of sight of a config spelled "ce-Latn", and an LF edit to one of them would write -// its old text to "ce-la" and clear its "ce-Latn" text in FieldWorks, since importing an entry -// clears every alternative LF lacks. In each of those entries every "ce-la" value is text the repo -// holds in "ce-Latn", so renaming the key gives LF what FieldWorks has. Keys are renamed in place, -// and entries' dates are left alone, so that LfMerge does not take them for LF edits. -const LEXICON_REKEYS = { - 'che-flex': { 'ce-la': 'ce-Latn' }, -}; - -// A subdocument, as opposed to an array, a date or a BSON value (ObjectId, Long and the like, which -// all carry _bsontype). -function isPlainObject(value) { - return value !== null && typeof value === 'object' && !Array.isArray(value) && - !(value instanceof Date) && value._bsontype === undefined; -} - -// Collect a $set for every "inputSystems" array under node that holds an old spelling. -function collectInputSystemArrays(node, path, oldToNew, sets, warnings) { - if (Array.isArray(node)) { - node.forEach((value, i) => collectInputSystemArrays(value, path + '.' + i, oldToNew, sets, warnings)); - return; - } - if (!isPlainObject(node)) { - return; - } - for (const [key, value] of Object.entries(node)) { - const childPath = path ? path + '.' + key : key; - if (key === 'inputSystems' && Array.isArray(value)) { - if (!value.some(tag => Object.prototype.hasOwnProperty.call(oldToNew, tag))) { - continue; - } - if (childPath.split('.').some(segment => segment.includes('$'))) { - warnings.push('cannot address ' + childPath + ' in an update; left as it is'); - continue; - } - const respelled = []; - for (const tag of value) { - const newTag = Object.prototype.hasOwnProperty.call(oldToNew, tag) ? oldToNew[tag] : tag; - if (!respelled.includes(newTag)) { - respelled.push(newTag); - } - } - sets[childPath] = respelled; - } else { - collectInputSystemArrays(value, childPath, oldToNew, sets, warnings); - } - } -} - -// Every place in the project record an old spelling still appears, for the check afterwards. -function remainingOldSpellings(project, oldToNew) { - const found = []; - const olds = new Set(Object.keys(oldToNew)); - (function walk(node, path) { - if (Array.isArray(node)) { - node.forEach((value, i) => walk(value, path + '.' + i)); - } else if (isPlainObject(node)) { - for (const [key, value] of Object.entries(node)) { - if (olds.has(key)) found.push(path + '.' + key + ' (key)'); - // An input system's abbreviation and language name are labels, not spellings: sjs-flex's - // "rif-Latn" is abbreviated "rif" - if (key === 'abbreviation' || key === 'languageName') continue; - walk(value, path + '.' + key); - } - } else if (typeof node === 'string' && olds.has(node)) { - found.push(path); - } - })({ config: project.config, inputSystems: project.inputSystems, languageCode: project.languageCode }, ''); - return found; -} - -function migrateProject(projects, projectCode, oldToNew) { - const project = projects.findOne({ projectCode: projectCode }); - if (!project) { - print(' ' + projectCode + ': NOT FOUND, skipped'); - return { found: false }; - } - - const sets = {}; - const unsets = {}; - const warnings = []; - - collectInputSystemArrays(project.config, 'config', oldToNew, sets, warnings); - const arrays = Object.keys(sets).length; - - let renamed = 0; - const inputSystems = project.inputSystems || {}; - for (const [oldTag, newTag] of Object.entries(oldToNew)) { - if (!Object.prototype.hasOwnProperty.call(inputSystems, oldTag)) { - continue; - } - if (Object.prototype.hasOwnProperty.call(inputSystems, newTag)) { - // Already spelled the new way, most likely by the new LfMerge build. The entry under the old - // spelling may then be a different writing system (see kxpw-flex above), so leave both alone. - warnings.push('inputSystems already has ' + newTag + '; left ' + oldTag + ' as it is'); - continue; - } - sets['inputSystems.' + newTag] = Object.assign({}, inputSystems[oldTag], { tag: newTag }); - unsets['inputSystems.' + oldTag] = ''; - renamed++; - } - - let languageCode = null; - if (Object.prototype.hasOwnProperty.call(oldToNew, project.languageCode)) { - languageCode = { from: project.languageCode, to: oldToNew[project.languageCode] }; - sets['languageCode'] = languageCode.to; - } - - const mapping = Object.entries(oldToNew).map(([o, n]) => o + ' -> ' + n).join(', '); - print(' ' + projectCode + ': ' + mapping); - print(' config inputSystems arrays re-spelled: ' + arrays); - print(' inputSystems record entries renamed: ' + renamed); - print(' languageCode: ' + - (languageCode ? languageCode.from + ' -> ' + languageCode.to : 'unchanged')); - warnings.forEach(w => print(' WARNING: ' + w)); - - if (arrays === 0 && renamed === 0 && !languageCode) { - print(' nothing to do (already migrated?)'); - return { found: true, changed: false }; - } - if (DRY_RUN) { - return { found: true, changed: true }; - } - - const update = {}; - if (Object.keys(sets).length > 0) update.$set = sets; - if (Object.keys(unsets).length > 0) update.$unset = unsets; - const result = projects.updateOne({ _id: project._id }, update); - print(' updated: matched ' + result.matchedCount + ', modified ' + result.modifiedCount); - - const left = remainingOldSpellings(projects.findOne({ _id: project._id }), oldToNew); - if (left.length > 0) { - print(' WARNING: old spellings still present at:'); - left.forEach(p => print(' ' + p)); - } else { - print(' verified: no old spelling remains in config, inputSystems or languageCode'); - } - return { found: true, changed: true }; -} - -// A multitext: a subdocument every one of whose values is a { value: ... } string field. -function isMultiText(node) { - if (!isPlainObject(node)) return false; - const values = Object.values(node); - return values.length > 0 && values.every(v => isPlainObject(v) && Object.prototype.hasOwnProperty.call(v, 'value')); -} - -// Collect a $set for every multitext under node that has an old key, renaming the key where it -// stands so that the keys keep their order. A multitext that already has the new key as well is -// left alone and reported: which of the two to keep is not this script's call. -function collectLexiconRekeys(node, path, oldToNew, sets, warnings) { - if (Array.isArray(node)) { - node.forEach((value, i) => collectLexiconRekeys(value, path + '.' + i, oldToNew, sets, warnings)); - return; - } - if (!isPlainObject(node)) { - return; - } - if (isMultiText(node)) { - const olds = Object.keys(node).filter(key => Object.prototype.hasOwnProperty.call(oldToNew, key)); - if (olds.length === 0) { - return; - } - if (olds.some(key => Object.prototype.hasOwnProperty.call(node, oldToNew[key]))) { - warnings.push(path + ' has both spellings; left as it is'); - return; - } - if (path.split('.').some(segment => segment.includes('$'))) { - warnings.push('cannot address ' + path + ' in an update; left as it is'); - return; - } - const renamed = {}; - for (const [key, value] of Object.entries(node)) { - renamed[Object.prototype.hasOwnProperty.call(oldToNew, key) ? oldToNew[key] : key] = value; - } - sets[path] = renamed; - return; - } - for (const [key, value] of Object.entries(node)) { - // A key with a dot in it cannot be addressed by a dotted path - if (key.includes('.')) { - warnings.push('cannot address ' + path + ' key "' + key + '" in an update; left as it is'); - continue; - } - collectLexiconRekeys(value, path ? path + '.' + key : key, oldToNew, sets, warnings); - } -} - -// How many multitexts in the lexicon still have an old key. -function countOldKeys(lexicon, oldToNew) { - let count = 0; - lexicon.find({}).forEach(entry => { - (function walk(node) { - if (Array.isArray(node)) { - node.forEach(walk); - } else if (isPlainObject(node)) { - if (isMultiText(node) && Object.keys(node).some(key => Object.prototype.hasOwnProperty.call(oldToNew, key))) { - count++; - } - Object.values(node).forEach(walk); - } - })(entry); - }); - return count; -} - -function rekeyLexicon(projectCode, oldToNew) { - const mapping = Object.entries(oldToNew).map(([o, n]) => o + ' -> ' + n).join(', '); - print(' ' + projectCode + ' lexicon: ' + mapping); - const lexicon = db.getSiblingDB('sf_' + projectCode).lexicon; - let entries = 0; - let multiTexts = 0; - const warnings = []; - lexicon.find({}).forEach(entry => { - const sets = {}; - const entryWarnings = []; - for (const [key, value] of Object.entries(entry)) { - if (key === '_id') continue; - collectLexiconRekeys(value, key, oldToNew, sets, entryWarnings); - } - entryWarnings.forEach(w => warnings.push('entry ' + entry.guid + ': ' + w)); - const count = Object.keys(sets).length; - if (count === 0) { - return; - } - entries++; - multiTexts += count; - if (!DRY_RUN) { - lexicon.updateOne({ _id: entry._id }, { $set: sets }); - } - }); - print(' entries re-keyed: ' + entries); - print(' multitexts re-keyed: ' + multiTexts); - warnings.forEach(w => print(' WARNING: ' + w)); - if (!DRY_RUN && entries > 0) { - const left = countOldKeys(lexicon, oldToNew); - print(left === 0 ? ' verified: no old key remains in the lexicon' - : ' WARNING: ' + left + ' multitexts still have an old key'); - } - return entries > 0; -} - -const projects = db.getSiblingDB(DB_NAME).projects; -print((DRY_RUN ? 'DRY RUN -- nothing will be changed. ' : '') + 'Database: ' + DB_NAME); -let changed = 0; -let missing = 0; -for (const [projectCode, oldToNew] of Object.entries(MIGRATIONS)) { - const outcome = migrateProject(projects, projectCode, oldToNew); - if (!outcome.found) missing++; - else if (outcome.changed) changed++; -} -let lexiconsChanged = 0; -for (const [projectCode, oldToNew] of Object.entries(LEXICON_REKEYS)) { - if (!projects.findOne({ projectCode: projectCode })) { - print(' ' + projectCode + ' lexicon: project NOT FOUND, skipped'); - continue; - } - if (rekeyLexicon(projectCode, oldToNew)) lexiconsChanged++; -} -print(''); -print((DRY_RUN ? 'Would change ' : 'Changed ') + changed + ' of ' + Object.keys(MIGRATIONS).length + - ' projects' + (missing ? '; ' + missing + ' not found' : '') + '.'); -print((DRY_RUN ? 'Would re-key ' : 'Re-keyed ') + lexiconsChanged + ' of ' + Object.keys(LEXICON_REKEYS).length + - ' lexicons.'); -if (DRY_RUN) { - print('Run again with DRY_RUN=false to apply.'); -} From 1a174b5228110344b6ac6b66a65d4fe952dd5856 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 12 Nov 2026 12:06:32 +0700 Subject: [PATCH 50/51] Handle any hg versions in create-tarball script The current way of creating the installation tarball is brittle: any changes to the file list of Mercurial will result in the files not being copied. Instead, we will just copy everything from the Mercurial NuGet package, then ensure that the `hg` and `chg` scripts are world-executable so that the www-data user can run them. This makes it far less brittle against Mercurial version updates. --- docker/scripts/create-installation-tarball.sh | 52 ++----------------- 1 file changed, 5 insertions(+), 47 deletions(-) diff --git a/docker/scripts/create-installation-tarball.sh b/docker/scripts/create-installation-tarball.sh index b9f37e15..556d0995 100755 --- a/docker/scripts/create-installation-tarball.sh +++ b/docker/scripts/create-installation-tarball.sh @@ -52,53 +52,11 @@ install -m 755 output/${BUILD}/${FRAMEWORK}/LfMerge ${DBDESTDIR}/${LIB} 2>/dev/n install -m 755 output/${BUILD}/${FRAMEWORK}/LfMergeQueueManager ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/LfMergeQueueManager ${DBDESTDIR}/${LIB} install -m 755 output/${BUILD}/${FRAMEWORK}/chorusmerge ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/chorusmerge ${DBDESTDIR}/${LIB} chmod 755 ${DBDESTDIR}/${LIB}/FixFwData.exe -install -d ${DBDESTDIR}/${LIB}/Mercurial -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils -install -d ${DBDESTDIR}/${LIB}/MercurialExtensions -install -d ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8 -install -m 755 Mercurial/chg ${DBDESTDIR}/${LIB}/Mercurial -install -m 755 Mercurial/hg ${DBDESTDIR}/${LIB}/Mercurial -install -m 644 Mercurial/mercurial.ini ${DBDESTDIR}/${LIB}/Mercurial -install -m 644 Mercurial/hgdemandimport/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport -install -m 644 Mercurial/hgext/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext -install -m 644 Mercurial/hgext/convert/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert -install -m 644 Mercurial/hgext/highlight/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight -install -m 644 Mercurial/hgext/largefiles/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles -install -m 644 Mercurial/hgext/zeroconf/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf -install -m 644 Mercurial/mercurial/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial -install -m 644 Mercurial/mercurial/defaultrc/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc -install -m 644 Mercurial/mercurial/dirstateutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils -install -m 644 Mercurial/mercurial/hgweb/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb -install -m 644 Mercurial/mercurial/httpclient/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true -install -m 644 Mercurial/mercurial/interfaces/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces -install -m 644 Mercurial/mercurial/pure/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure -install -m 644 Mercurial/mercurial/revlogutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils -install -m 644 Mercurial/mercurial/stabletailgraph/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph -install -m 644 Mercurial/mercurial/testing/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing -install -m 644 Mercurial/mercurial/thirdparty/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty -install -m 644 Mercurial/mercurial/thirdparty/attr/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr -install -m 644 Mercurial/mercurial/utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils -install -m 644 Mercurial/mercurial/upgrade_utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils -install -m 644 MercurialExtensions/fixutf8/*.* ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8 +cp -a Mercurial ${DBDESTDIR}/${LIB}/ +cp -a MercurialExtensions ${DBDESTDIR}/${LIB}/ +# Ensure hg and chg are world-executable +chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/hg +chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/chg install -d ${DBDESTDIR}/${LIB}/runtimes install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64 install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64/lib From 2e748f81e9104022089cf7f78b3337449df7c32e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 12 Nov 2026 12:17:37 +0700 Subject: [PATCH 51/51] Send a never-synced LF project's entries FieldWorks lacks A never-synced project's entries count as edited in LF only if they carry a user reference (#362), because LfMerge clears those on every entry it writes. A LIFT import into LF leaves none either, so on an LF project's first Send/Receive -- as when one is migrated -- every imported entry was taken to be LfMerge's own and left out. The transfer back to Mongo then marked each of them deleted, since FieldWorks did not have it: 12,638 of a test project's 14,370 entries, and 47,827 entries across the projects due to be migrated. FieldWorks cannot hold a newer version of an entry it does not have, so in a never-synced project such an entry is now sent. Entries FieldWorks has are still left to FieldWorks, as #362 intended. The one case this changes besides migration is an entry LfMerge wrote on an initial clone and FLEx then deleted, before any sync was recorded: it now comes back instead of staying deleted. Co-Authored-By: Claude Opus 5.5 --- .../Lcm/TransferMongoToLcmActionTests.cs | 33 +++++++++++++++++++ .../ConvertMongoToLcmLexicon.cs | 30 +++++++++++++---- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs index 0013eca5..86dc7aa1 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs @@ -745,6 +745,39 @@ public void Action_NeverSyncedProjectWhoseEntriesLfMergeWrote_KeepsFieldWorksVer Assert.That(_counts.Modified, Is.EqualTo(0)); } + [Test] + public void Action_NeverSyncedProjectWithEntryImportedIntoLf_SendsItToFieldWorks() + { + // Setup: an LF project's first Send/Receive, as when one is migrated. A LIFT import into + // LF leaves no user references on its entries, the same as LfMerge's own writes do, so + // the entry is stored with UpdateRecord rather than as an LF user's edit. + var lfProj = _lfProj; + _conn.SetLastSyncedDate(lfProj, MagicValues.UnixEpoch); + FwProject fwProject = lfProj.FieldWorksProject; + string vernacularWS = fwProject.Cache.LanguageProject.DefaultVernacularWritingSystem.Id; + Guid importedGuid = Guid.NewGuid(); + var imported = new LfLexEntry(); + imported.Guid = importedGuid; + imported.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, "imported into LF"); + imported.AuthorInfo = new LfAuthorInfo(); + imported.AuthorInfo.CreatedDate = new DateTime(2016, 4, 10, 0, 0, 0, DateTimeKind.Utc); + imported.AuthorInfo.ModifiedDate = imported.AuthorInfo.CreatedDate; + imported.DateModified = new DateTime(2016, 7, 25, 0, 0, 0, DateTimeKind.Utc); + _conn.UpdateRecord(lfProj, imported); + + // Exercise + SutMongoToLcm.Run(lfProj); + + // Verify + ILexEntry sent; + Assert.That(fwProject.ServiceLocator.GetInstance().TryGetObject(importedGuid, out sent), Is.True, + "an entry only LF has must reach FieldWorks, or the transfer back to Mongo marks it deleted"); + Assert.That(sent.LexemeFormOA.Form.VernacularDefaultWritingSystem.Text, Is.EqualTo("imported into LF")); + Assert.That(_counts.Added, Is.EqualTo(1)); + Assert.That(_counts.Modified, Is.EqualTo(0)); + Assert.That(_counts.Deleted, Is.EqualTo(0)); + } + // TODO: Move custom field tests to their own test class, and move these helper functions with them public void AddCustomFieldToSense(LfSense sense, string fieldName, BsonValue fieldValue) { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 359b3111..c86a1065 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -43,8 +43,10 @@ public class ConvertMongoToLcmLexicon // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm private ISet _configuredTags; - // Entries where LF's copy differs from FieldWorks but no LF user has touched it since the - // last sync, so FieldWorks holds the newer version (see EditedInLfSinceLastSync) + // Entries where LF's copy differs from FieldWorks, or FieldWorks lacks it, but no LF user has + // touched it since the last sync, so FieldWorks holds the newer version (see + // EditedInLfSinceLastSync). An entry FieldWorks lacks in a project that has never synced is + // not counted: it is LF's own, and is sent (see LfLexEntryToLcmLexEntry). private int _changedOnlyInFieldWorks; private int _absentOnlyFromFieldWorks; @@ -1058,6 +1060,9 @@ private DateTime? LastSyncedDate /// LfMerge's: the initial clone's transfer to Mongo does not always get as far as recording a /// sync. LfMerge clears the user references on every entry it writes, and LF sets one on every /// edit and creation; a deletion sets none, but deleting what FieldWorks lacks is harmless. + /// A LIFT import into LF sets none either, so this says "not edited" of every imported entry + /// in a never-synced LF project; LfLexEntryToLcmLexEntry sends such an entry anyway when + /// FieldWorks does not have it. /// private bool EditedInLfSinceLastSync(LfLexEntry lfEntry) { @@ -1079,12 +1084,25 @@ private void LfLexEntryToLcmLexEntry(LfLexEntry lfEntry) if (lfEntry.IsDeleted) { // An LF deletion that an earlier sync has already sent + return; + } + if (GetInstance().TryGetObject(guid, out existing)) + { + if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified) + _changedOnlyInFieldWorks++; + return; } - else if (!GetInstance().TryGetObject(guid, out existing)) + if (LastSyncedDate != null) + { _absentOnlyFromFieldWorks++; - else if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified) - _changedOnlyInFieldWorks++; - return; + return; + } + // Never synced, and FieldWorks does not have it: an LF project's own entry, going to + // FieldWorks for the first time. A LIFT import into LF sets no user references either, + // so EditedInLfSinceLastSync cannot tell it from one LfMerge wrote; but FieldWorks + // cannot hold a newer version of an entry it does not have, so there is nothing to + // keep. Leaving it out would also lose it: the transfer back to Mongo marks every + // entry FieldWorks lacks as deleted. } bool createdEntry = false; bool wantCreation = !lfEntry.IsDeleted;