diff --git a/docker/scripts/create-installation-tarball.sh b/docker/scripts/create-installation-tarball.sh index b9f37e15..556d0995 100755 --- a/docker/scripts/create-installation-tarball.sh +++ b/docker/scripts/create-installation-tarball.sh @@ -52,53 +52,11 @@ install -m 755 output/${BUILD}/${FRAMEWORK}/LfMerge ${DBDESTDIR}/${LIB} 2>/dev/n install -m 755 output/${BUILD}/${FRAMEWORK}/LfMergeQueueManager ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/LfMergeQueueManager ${DBDESTDIR}/${LIB} install -m 755 output/${BUILD}/${FRAMEWORK}/chorusmerge ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/chorusmerge ${DBDESTDIR}/${LIB} chmod 755 ${DBDESTDIR}/${LIB}/FixFwData.exe -install -d ${DBDESTDIR}/${LIB}/Mercurial -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles -install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils -install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils -install -d ${DBDESTDIR}/${LIB}/MercurialExtensions -install -d ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8 -install -m 755 Mercurial/chg ${DBDESTDIR}/${LIB}/Mercurial -install -m 755 Mercurial/hg ${DBDESTDIR}/${LIB}/Mercurial -install -m 644 Mercurial/mercurial.ini ${DBDESTDIR}/${LIB}/Mercurial -install -m 644 Mercurial/hgdemandimport/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport -install -m 644 Mercurial/hgext/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext -install -m 644 Mercurial/hgext/convert/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert -install -m 644 Mercurial/hgext/highlight/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight -install -m 644 Mercurial/hgext/largefiles/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles -install -m 644 Mercurial/hgext/zeroconf/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf -install -m 644 Mercurial/mercurial/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial -install -m 644 Mercurial/mercurial/defaultrc/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc -install -m 644 Mercurial/mercurial/dirstateutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils -install -m 644 Mercurial/mercurial/hgweb/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb -install -m 644 Mercurial/mercurial/httpclient/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true -install -m 644 Mercurial/mercurial/interfaces/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces -install -m 644 Mercurial/mercurial/pure/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure -install -m 644 Mercurial/mercurial/revlogutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils -install -m 644 Mercurial/mercurial/stabletailgraph/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph -install -m 644 Mercurial/mercurial/testing/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing -install -m 644 Mercurial/mercurial/thirdparty/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty -install -m 644 Mercurial/mercurial/thirdparty/attr/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr -install -m 644 Mercurial/mercurial/utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils -install -m 644 Mercurial/mercurial/upgrade_utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils -install -m 644 MercurialExtensions/fixutf8/*.* ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8 +cp -a Mercurial ${DBDESTDIR}/${LIB}/ +cp -a MercurialExtensions ${DBDESTDIR}/${LIB}/ +# Ensure hg and chg are world-executable +chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/hg +chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/chg install -d ${DBDESTDIR}/${LIB}/runtimes install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64 install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64/lib diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs new file mode 100644 index 00000000..3773e8e2 --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs @@ -0,0 +1,184 @@ +// Copyright (c) 2016-2018 SIL International +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +using System; +using System.Linq; +using LfMerge.Core.DataConverters; +using MongoDB.Bson; +using NUnit.Framework; +using SIL.LCModel; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// Writing custom fields from LF into LCM. + /// + /// Cust_Single_Line_All in the test project is a MultiUnicode field with English and French + /// alternatives. That is the shape every text custom field an LF project owns takes: LF has + /// never offered a single-string custom field, so a field created there is always multilingual + /// even when only one writing system is filled in. + /// + public class ConvertMongoToLcmCustomFieldTests : LcmTestBase + { + private const string MultiTextField = "customField_entry_Cust_Single_Line_All"; + // A String field, whose writing system is the magic "first analysis" selector. + private const string SingleStringField = "customField_entry_Cust_Single_Line"; + + private static BsonDocument MultiText(params string[] wsThenValue) + { + var multiText = new BsonDocument(); + for (int i = 0; i < wsThenValue.Length; i += 2) + multiText.Add(wsThenValue[i], new BsonDocument("value", wsThenValue[i + 1])); + return multiText; + } + + private ILexEntry TestEntry() + { + var entry = _servLoc.GetInstance().GetObject(Guid.Parse(TestEntryGuidStr)); + Assert.That(entry, Is.Not.Null); + return entry; + } + + private ConvertMongoToLcmCustomField Converter() + { + return new ConvertMongoToLcmCustomField(_cache, _servLoc, + new TestLogger(TestContext.CurrentContext.Test.Name), _wsEn); + } + + /// Reads the entry's custom fields back out through the converter going the other way. + private BsonDocument ReadBack(ILexEntry entry) + { + var reader = new ConvertLcmToMongoCustomField(_cache, _servLoc, + new TestLogger(TestContext.CurrentContext.Test.Name)); + return reader.GetCustomFieldsForThisCmObject(entry, "entry", _listConverters)[0].AsBsonDocument; + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldWriteEveryMultiUnicodeAlternative() + { + // Setup + ILexEntry entry = TestEntry(); + var customFields = new BsonDocument(MultiTextField, + MultiText("en", "Updated English", "fr", "Updated French")); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", customFields, null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written["en"]["value"].AsString, Is.EqualTo("Updated English")); + Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Updated French")); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldWriteAFieldThatWasEmptyInLcm() + { + // The migration case: LF holds the only copy, and LCM has nothing in the field yet. + // Emptying it leaves the alternatives in place holding empty strings rather than removing + // them, so the precondition checks the contents rather than the presence of the key. + ILexEntry entry = TestEntry(); + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "", "fr", "")), null); + foreach (BsonElement alternative in ReadBack(entry)[MultiTextField].AsBsonDocument) + Assert.That(alternative.Value["value"].AsString, Is.Empty, + $"precondition: {alternative.Name} should have been emptied"); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "Brand new")), null); + + // Verify + Assert.That(ReadBack(entry)[MultiTextField]["en"]["value"].AsString, Is.EqualTo("Brand new")); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldClearAnAlternativeLfNoLongerHas() + { + // The reverse converter exports every alternative LCM holds, so one missing from LF is + // one the user deleted there -- not one LF never knew about. + ILexEntry entry = TestEntry(); + Assert.That(ReadBack(entry)[MultiTextField].AsBsonDocument.Contains("fr"), Is.True, + "precondition: the test project's field starts with a French alternative"); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "English only")), null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written["en"]["value"].AsString, Is.EqualTo("English only")); + Assert.That(written.Contains("fr"), Is.False, "French should have been cleared, not left stale"); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldSkipUnidentifiedWritingSystems() + { + // Setup + ILexEntry entry = TestEntry(); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, + MultiText("en", "Kept", "zzz-x-nonesuch", "Dropped")), null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written["en"]["value"].AsString, Is.EqualTo("Kept")); + Assert.That(written.Contains("zzz-x-nonesuch"), Is.False); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagToTheCanonicalWritingSystem() + { + // LF keeps a tag as typed, while LCM knows the writing system only by its canonical id: + // "fr-Latn" canonicalizes to "fr", French's suppress-script being dropped. Looked up + // literally the key resolves to 0, and French would then be cleared as missing from LF. + ILexEntry entry = TestEntry(); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(MultiTextField, MultiText("en", "English", "fr-Latn", "Français")), null); + + // Verify + BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument; + Assert.That(written.Contains("fr"), Is.True, "French should have been written, not cleared"); + Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Français")); + } + + [Test] + public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagInAStringField() + { + // The String case resolved the key it picked with a literal GetWsFromStr too, and gave + // up on the whole field when that returned 0, so the LF edit never reached LCM. + ILexEntry entry = TestEntry(); + + // Exercise + Converter().SetCustomFieldsForThisCmObject(entry, "entry", + new BsonDocument(SingleStringField, MultiText("fr-Latn", "Texte")), null); + + // Verify + BsonDocument written = ReadBack(entry)[SingleStringField].AsBsonDocument; + Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Texte")); + } + + [Test] + public void SetCustomFieldData_FieldOfATypeItCannotWrite_IsReportedOncePerSyncNotOncePerObject() + { + // Setup: an owning sequence, a type the LF to LCM direction does not implement, on + // three entries. The test project has no custom field of that type, so a built-in + // field stands in for one; the writer does not tell them apart. + var logger = new TestLogger(TestContext.CurrentContext.Test.Name); + var converter = new ConvertMongoToLcmCustomField(_cache, _servLoc, logger, _wsEn); + var entries = _servLoc.GetInstance().AllInstances().Take(3).ToList(); + Assert.That(entries, Has.Count.EqualTo(3)); + + // Exercise + foreach (ILexEntry entry in entries) + Assert.That(converter.SetCustomFieldData(entry.Hvo, LexEntryTags.kflidSenses, new BsonString("LF data"), null), Is.False); + + // Verify + int reports = logger.Messages.Split('\n').Count(line => line.Contains("not written to LCM")); + Assert.That(reports, Is.EqualTo(1)); + } + } +} diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs new file mode 100644 index 00000000..40cd4c3a --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs @@ -0,0 +1,881 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LfMerge.Core.DataConverters; +using LfMerge.Core.LanguageForge.Config; +using LfMerge.Core.LanguageForge.Model; +using LfMerge.Core.MongoConnector; +using LfMergeBridge.LfMergeModel; +using MongoDB.Bson; +using NUnit.Framework; +using SIL.LCModel; +using SIL.LCModel.Infrastructure; +using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.Text; +using SIL.LCModel.Core.WritingSystems; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// Language Forge stores a writing system tag exactly as it was typed, but LCM canonicalizes a + /// tag when it creates a writing system and thereafter knows that writing system by its Id. LCM's + /// own lookups -- WritingSystemManager.TryGet, GetWsFromStr -- match an Id regardless of case + /// but otherwise literally, so a tag differing from the Id in structure, a subtag absorbed or + /// dropped, is not found by them; LanguageTags.WsIdFromLfTag finds it. + /// + /// testlangproj already contains "qaa-x-kal" and "qaa-Zxxx-x-kal-audio", so a non-canonical LF + /// spelling of either exercises the mismatch without adding to the shared fixture project. The + /// tests that need a writing system of their own add it under a tag no other test uses, since + /// what is added to the fixture's LCM stays for the rest of the run. + /// + public class ConvertMongoToLcmWritingSystemsTests : LcmTestBase + { + // The apltw2016 shape: the private-use section repeats the "qaa" language subtag, which + // canonicalizing absorbs (apltw2016 itself has "qaa-x-qaa-v", which becomes "qaa-x-v"). + private const string RedundantPrivateUseTag = "qaa-x-qaa-kal"; + private const string RedundantPrivateUseId = "qaa-x-kal"; + + // The shape found in the testlangproj-derived projects, where canonicalizing only changes case. + // This one ALREADY works, because both LCM lookups (TryGet and GetWsFromStr) are + // case-insensitive; it is here as a guard so that a fix for the structural case does not + // regress it. Only differences in structure -- a subtag absorbed or dropped -- go unfound. + private const string MixedCaseTag = "qaa-Zxxx-x-kal-AUDIO"; + private const string MixedCaseId = "qaa-Zxxx-x-kal-audio"; + + /// + /// Adds one input system to the mock project record, and returns an action that puts the + /// record back as it was. Each test gets a fresh record, so the restore is only tidiness; + /// what does outlive a test is a writing system it adds to LCM, which is why the tests that + /// add one each use tags of their own. + /// + private Action AddLfInputSystem(string tag) + { + MongoProjectRecord record = _recordFactory.Create(_lfProj); + Assert.That(record, Is.Not.Null, "the mock project record factory should supply a record"); + Assert.That(record.InputSystems.ContainsKey(tag), Is.False, + "the fixture's input systems should not already contain {0}", tag); + + record.InputSystems[tag] = new LfInputSystemRecord { + Abbreviation = tag, + Tag = tag, + LanguageName = "Unlisted Language", + IsRightToLeft = false + }; + return () => record.InputSystems.Remove(tag); + } + + /// + /// Adds a tag to one config field's input systems, found by its path through the entry's + /// field lists, and returns an action that takes it out again. Like the input systems, the + /// config comes fresh with each test's record, so this too is only tidiness. + /// + private Action AddToConfigField(string tag, params string[] path) + { + LfConfigFieldBase field = _recordFactory.Create(_lfProj).Config.Entry; + foreach (string key in path) + field = ((LfConfigFieldList)field).Fields[key]; + var multiText = (LfConfigMultiText)field; + multiText.InputSystems.Add(tag); + return () => multiText.InputSystems.Remove(tag); + } + + private IEnumerable LcmWritingSystemIds() + { + return _cache.ServiceLocator.WritingSystemManager.WritingSystems.Select(ws => ws.Id); + } + + [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)] + [TestCase(MixedCaseTag, MixedCaseId)] + public void LfWsToLcmWs_NonCanonicalTag_ReusesTheCanonicalWritingSystem(string lfTag, string canonicalId) + { + // Setup: LF holds the writing system under a non-canonical tag, while the LCM project + // already holds it under the canonical id that LCM derives from that same tag. + Assert.That(LcmWritingSystemIds(), Contains.Item(canonicalId), + "testlangproj should already contain {0}", canonicalId); + int countBefore = LcmWritingSystemIds().Count(id => id == canonicalId); + Action restore = AddLfInputSystem(lfTag); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify: the existing writing system was reused, not duplicated, and the raw LF tag + // did not become an id of its own. + Assert.That(LcmWritingSystemIds().Count(id => id == canonicalId), Is.EqualTo(countBefore), + "{0} should still appear exactly once", canonicalId); + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(lfTag), + "the non-canonical tag {0} should not be stored as an id of its own", lfTag); + } + finally + { + restore(); + } + } + + [Test] + public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWritingSystem() + { + // Setup: a definition keyed by the non-canonical tag. LfMultiText used to resolve each key + // with GetWsFromStr and skip anything that returned 0, so an unresolved key dropped the + // value -- and the clearing pass then blanked whatever LCM had for that field. + const string newDefinition = "Definition stored under a non-canonical writing system tag"; + var data = new SampleData(); + data.bsonTestData["senses"][0]["definition"] = new BsonDocument { + { RedundantPrivateUseTag, new BsonDocument { { "value", newDefinition } } } + }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + int wsId = _cache.WritingSystemFactory.GetWsFromStr(RedundantPrivateUseId); + Assert.That(wsId, Is.Not.EqualTo(0), "{0} should resolve in testlangproj", RedundantPrivateUseId); + + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + Guid expectedGuid = Guid.Parse(data.bsonTestData["guid"].AsString); + var entry = _cache.ServiceLocator.GetObject(expectedGuid) as ILexEntry; + Assert.That(entry, Is.Not.Null); + Assert.That(entry.SensesOS[0].Definition.get_String(wsId).Text, Is.EqualTo(newDefinition), + "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId); + } + + /// + /// Text in a built-in multitext field under a key that names no writing system LCM has has + /// nowhere to go. It is skipped, as it always was, but no longer silently: the custom fields + /// and the single-string fields already said so, and this, the commonest path, did not. + /// + [Test] + public void SetMultiStringFrom_KeyNamingNoWritingSystem_IsSkippedWithAWarning() + { + const string tag = "zzz-x-nonesuch"; + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, tag), Is.EqualTo(0)); + var data = new SampleData(); + data.bsonTestData["senses"][0]["definition"][tag] = new BsonDocument { { "value", "a definition with nowhere to go" } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + SutMongoToLcm.Run(_lfProj); + + Assert.That(_env.Logger.GetMessages(), + Does.Contain("MongoToLcm: skipping text under \"zzz-x-nonesuch\" (a definition with nowhere to go)")); + } + + [Test] + public void SingleStringField_TextInNoKnownWritingSystem_KeepsTheLcmValueAndSyncsTheRest() + { + // Setup: LCM holds a scientific name, and then LF holds one only in a writing system LCM + // has never heard of. Building that into a string in writing system 0 threw, abandoning + // the rest of the entry half-written; returning null instead would clear the field and + // delete what LCM holds. Neither is acceptable. + const string original = "Homo sapiens"; + const string laterNote = "A note set after the scientific name"; + var data = new SampleData(); + BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument; + sense["scientificName"] = new BsonDocument { { "en", new BsonDocument { { "value", original } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + SutMongoToLcm.Run(_lfProj); + + sense["scientificName"] = new BsonDocument { + { "zzz-x-nonesuch", new BsonDocument { { "value", "Unplaceable" } } } }; + // Converted after the scientific name, so it only arrives if the entry is not abandoned. + sense["sociolinguisticsNote"] = new BsonDocument { + { "en", new BsonDocument { { "value", laterNote } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow.AddMinutes(1); + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + var entry = _cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)) as ILexEntry; + Assert.That(entry, Is.Not.Null); + ILexSense lcmSense = entry.SensesOS[0]; + Assert.That(lcmSense.ScientificName.Text, Is.EqualTo(original), + "text that cannot be placed must not replace or clear what LCM holds"); + Assert.That(lcmSense.SocioLinguisticsNote.get_String(_wsEn).Text, Is.EqualTo(laterNote), + "the rest of the entry should still have been converted"); + Assert.That(_env.Logger.GetMessages(), Does.Contain( + "MongoToLcm: skipping text under \"zzz-x-nonesuch\" (Unplaceable), which names no writing system LCM has; " + + "leaving the field as it was")); + } + + /// + /// A single-string field whose text is all under a key passed over for an empty one naming + /// the same writing system -- the config's spelling, which LF's editor fills in empty -- + /// keeps what LCM holds, and the log says so for that reason, not that LCM lacks the + /// writing system. + /// + [Test] + public void SingleStringField_TextOnlyUnderAKeyPassedOverForAnEmptyOne_KeepsTheLcmValueAndSaysWhy() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("ro-Latn-MD", "ro-MD"); + const string original = "Homo sapiens"; + var data = new SampleData(); + BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument; + sense["scientificName"] = new BsonDocument { { "en", new BsonDocument { { "value", original } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + SutMongoToLcm.Run(_lfProj); + + Action restore = AddToConfigField("ro-MD", "senses", "scientificName"); + try + { + sense["scientificName"] = new BsonDocument { + { "ro-Latn-MD", new BsonDocument { { "value", "the export's value" } } }, + { "ro-MD", new BsonDocument { { "value", "" } } } }; + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow.AddMinutes(1); + _conn.UpdateMockLfLexEntry(data.bsonTestData); + + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + var entry = _cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)) as ILexEntry; + Assert.That(entry.SensesOS[0].ScientificName.Text, Is.EqualTo(original)); + Assert.That(_env.Logger.GetMessages(), Does.Contain( + "MongoToLcm: keys \"ro-Latn-MD\" and \"ro-MD\" name the same writing system, and \"ro-MD\", the one to " + + "write, is empty; leaving the field as it was rather than write \"ro-Latn-MD\" (the export's value)")); + Assert.That(_env.Logger.GetMessages(), Does.Not.Contain("names no writing system LCM has")); + } + finally + { + restore(); + } + } + + [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)] + [TestCase(MixedCaseTag, MixedCaseId)] + [TestCase("fr-Latn", "fr")] + public void WsIdFromLfTag_NonCanonicalTag_ResolvesToTheCanonicalWritingSystem(string lfTag, string canonicalId) + { + int expected = _cache.WritingSystemFactory.GetWsFromStr(canonicalId); + Assert.That(expected, Is.Not.EqualTo(0), "{0} should resolve in testlangproj", canonicalId); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, lfTag), Is.EqualTo(expected)); + } + + /// + /// Adds a writing system whose Id is spelled differently from its LanguageTag, the way LCM + /// loads an older .ldml file: brb-flex-2022's "km-Khmr-KH.ldml" has km plus KH for its + /// identity, so it is held as Id "km-Khmr-KH" with LanguageTag "km-KH". The repository + /// keeps an Id as given whenever it is equivalent to the LanguageTag. + /// + private CoreWritingSystemDefinition AddWritingSystemWithId(string id, string languageTag) + { + WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager; + CoreWritingSystemDefinition ws = wsManager.Create(languageTag); + ws.Id = id; + wsManager.Set(ws); + Assert.That(ws.Id, Is.EqualTo(id), "precondition: LCM should hold the writing system as {0}", id); + Assert.That(ws.LanguageTag, Is.EqualTo(languageTag)); + return ws; + } + + [Test] + public void WsIdFromLfTag_ExactIdWinsOverAnotherWritingSystemWithTheCanonicalId() + { + // The brb-flex-2022 layout: the real writing system under a non-canonical Id, and an + // unused second one whose Id is the canonical form. Text keyed by the real one's Id is + // the real one's text; resolving the canonical form first sent it to the other, and the + // clearing pass then deleted the real writing system's text as missing from LF. + CoreWritingSystemDefinition real = AddWritingSystemWithId("de-Latn-AT", "de-AT"); + CoreWritingSystemDefinition stray = _cache.ServiceLocator.WritingSystemManager.Set("de-AT"); + Assert.That(stray.Handle, Is.Not.EqualTo(real.Handle)); + + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "de-Latn-AT"), Is.EqualTo(real.Handle)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "de-AT"), Is.EqualTo(stray.Handle)); + } + + [Test] + public void WsIdFromLfTag_CanonicalTagFindsAWritingSystemWhoseIdIsSpelledOtherwise() + { + // The spt-flex layout: LCM holds "hi-Deva-IN" and LF's input systems call it "hi-IN", + // which is neither the Id nor its canonical form spelled as an Id. + CoreWritingSystemDefinition real = AddWritingSystemWithId("fr-Latn-CA", "fr-CA"); + Assert.That(_cache.WritingSystemFactory.GetWsFromStr("fr-CA"), Is.EqualTo(0), + "precondition: no writing system has the Id fr-CA"); + + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "fr-CA"), Is.EqualTo(real.Handle)); + } + + [Test] + public void LfWsToLcmWs_InputSystemSpelledAsTheLanguageTag_DoesNotDuplicateTheWritingSystem() + { + // The spt-flex layout: LCM holds "hi-Deva-IN", whose LanguageTag is "hi-IN", and LF's + // input systems -- which LfMerge used to export by LanguageTag -- call it "hi-IN". GetOrSet looks + // only at Ids, so it created a second writing system "hi-IN" and put it in the lists. + CoreWritingSystemDefinition real = AddWritingSystemWithId("pt-Latn-AO", "pt-AO"); + Action restore = AddLfInputSystem("pt-AO"); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + Assert.That(LcmWritingSystemIds(), Does.Not.Contain("pt-AO"), + "no duplicate writing system should be created for the input system"); + Assert.That(_cache.ServiceLocator.WritingSystemManager.WritingSystems + .Count(ws => ws.LanguageTag == "pt-AO"), Is.EqualTo(1)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "pt-AO"), Is.EqualTo(real.Handle), + "LF text keyed \"pt-AO\" should go to the existing writing system"); + } + finally + { + restore(); + } + } + + /// + /// An input system spelled by Id -- as every project LfMerge set up under FieldWorks 8 has + /// them -- for a writing system whose Id differs from its LanguageTag. Replace looked the + /// writing system up by LanguageTag, which either found nothing and re-registered it under a + /// fresh handle, or found the unused writing system with that LanguageTag as its Id and + /// evicted it, the real one taking over its handle. + /// + [TestCase("es-Latn-MX", "es-MX", false)] + [TestCase("it-Latn-CH", "it-CH", true)] + public void LfWsToLcmWs_ExistingWritingSystemWhoseIdIsNotItsLanguageTag_KeepsItsHandle( + string id, string languageTag, bool withUnusedNamesake) + { + WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager; + CoreWritingSystemDefinition real = AddWritingSystemWithId(id, languageTag); + int realHandle = real.Handle; + CoreWritingSystemDefinition namesake = withUnusedNamesake ? wsManager.Set(languageTag) : null; + int namesakeHandle = withUnusedNamesake ? namesake.Handle : 0; + Action restore = AddLfInputSystem(id); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + Assert.That(real.Handle, Is.EqualTo(realHandle), "the writing system should keep its handle"); + Assert.That(wsManager.GetWsFromStr(id), Is.EqualTo(realHandle)); + Assert.That(real.Abbreviation, Is.EqualTo(id), + "the input system's properties should still have been applied"); + if (withUnusedNamesake) + { + Assert.That(wsManager.GetWsFromStr(languageTag), Is.EqualTo(namesakeHandle), + "the other writing system with that LanguageTag should be left alone"); + } + } + finally + { + restore(); + } + } + + /// + /// Two LF keys naming one writing system: the export's, spelled as its Id, and one spelled + /// as its LanguageTag. Only one can be written, and it is the one under the spelling LF's + /// config uses, wherever it comes, since that is the one LF's editor shows; with neither + /// spelling configured, the later one. The other is reported. + /// + /// The export's key comes first in an entry, since an LF user's new key is added after it. + /// Before the config is re-spelled by Id, the user's edit is under the LanguageTag and the + /// export's value is hidden and stale; afterwards, an Id-spelled edit is made in place, and a + /// LanguageTag key left from before is what is stale. + /// + // A pair each, since writing systems a test adds stay in the fixture's project + [TestCase("languageTag", true, "sv-Latn-FI", "sv-FI", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelledByLanguageTag_WritesTheEdit")] + [TestCase("id", true, "da-Latn-DE", "da-DE", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelledById_WritesTheEditMadeInPlace")] + [TestCase("languageTag", false, "nb-Latn-NO", "nb-NO", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelling_WinsWhereverItComes")] + [TestCase("neither", true, "fi-Latn-SE", "fi-SE", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_NeitherConfigured_WritesTheLaterKey")] + public void WriteToLcm_TwoKeysForOneWritingSystem(string configured, bool idFirst, string id, string languageTag) + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, languageTag); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, languageTag), Is.EqualTo(ws.Handle), + "precondition: the LanguageTag spelling should resolve to the same writing system"); + var multiText = new LfMultiText(); + if (idFirst) + multiText.Add(id, LfStringField.FromString("under the Id")); + multiText.Add(languageTag, LfStringField.FromString("under the LanguageTag")); + if (!idFirst) + multiText.Add(id, LfStringField.FromString("under the Id")); + var configuredTags = new HashSet(StringComparer.Ordinal) { "en", "fr" }; + if (configured == "id") + configuredTags.Add(id); + else if (configured == "languageTag") + configuredTags.Add(languageTag); + string expected = configured == "id" ? id : configured == "languageTag" ? languageTag : (idFirst ? languageTag : id); + var gloss = ((ILexEntry)_cache.ServiceLocator.GetObject(Guid.Parse(TestEntryGuidStr))).SensesOS[0].Gloss; + var reported = new List<(string NotWritten, string Written)>(); + + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + multiText.WriteToLcmMultiString(gloss, _cache.WritingSystemFactory, configuredTags, + onKeyNotWritten: (notWritten, written) => reported.Add((notWritten, written)))); + + Assert.That(gloss.get_String(ws.Handle).Text, Is.EqualTo(expected == id ? "under the Id" : "under the LanguageTag")); + Assert.That(reported, Is.EqualTo(new[] { (expected == id ? languageTag : id, expected) })); + } + + /// + /// The config's spellings reach every writer that chooses between two keys naming one + /// writing system, through a whole sync: the built-in multitext fields, and picture captions + /// with them; the single-string fields; and the multitext and string custom fields. The + /// config's spelling comes first here, so a writer going on key order alone would write the + /// export's value instead. + /// + // A pair each, since writing systems a test adds stay in the fixture's project + [TestCase("gloss", "hu-Latn-SK", "hu-SK")] + [TestCase("scientificName", "sk-Latn-HU", "sk-HU")] + [TestCase("customMultiText", "sl-Latn-IT", "sl-IT")] + [TestCase("customString", "et-Latn-FI", "et-FI")] + public void MongoToLcm_TwoKeysForOneWritingSystem_WritesTheConfigsSpellingInEveryKindOfField( + string field, string id, string languageTag) + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, languageTag); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, languageTag), Is.EqualTo(ws.Handle), + "precondition: the LanguageTag spelling should resolve to the same writing system"); + var twoKeys = new BsonDocument { + { languageTag, new BsonDocument { { "value", "the user's edit" } } }, + { id, new BsonDocument { { "value", "the export's value" } } } }; + var data = new SampleData(); + BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument; + BsonDocument customFields = data.bsonTestData["customFields"].AsBsonDocument; + string[] configPath; + switch (field) + { + case "gloss": + sense["gloss"] = twoKeys; + configPath = new[] { "senses", "gloss" }; + break; + case "scientificName": + sense["scientificName"] = twoKeys; + configPath = new[] { "senses", "scientificName" }; + break; + case "customMultiText": + customFields["customField_entry_Cust_Single_Line_All"] = twoKeys; + configPath = new[] { "customField_entry_Cust_Single_Line_All" }; + break; + default: + customFields["customField_entry_Cust_Single_Line"] = twoKeys; + configPath = new[] { "customField_entry_Cust_Single_Line" }; + break; + } + data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(data.bsonTestData); + Action restore = AddToConfigField(languageTag, configPath); + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + var entry = (ILexEntry)_cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)); + ISilDataAccess sda = _cache.DomainDataByFlid; + string written; + switch (field) + { + case "gloss": + written = entry.SensesOS[0].Gloss.get_String(ws.Handle).Text; + break; + case "scientificName": + written = entry.SensesOS[0].ScientificName.Text; + break; + case "customMultiText": + written = sda.get_MultiStringAlt(entry.Hvo, EntryCustomFieldId("Cust Single Line All"), ws.Handle).Text; + break; + default: + written = sda.get_StringProp(entry.Hvo, EntryCustomFieldId("Cust Single Line")).Text; + break; + } + Assert.That(written, Is.EqualTo("the user's edit")); + } + finally + { + restore(); + } + } + + private int EntryCustomFieldId(string name) + { + int flid = _cache.MetaDataCacheAccessor.GetFieldId2(LexEntryTags.kClassId, name, false); + Assert.That(flid, Is.Not.EqualTo(0), "testlangproj should have the custom field {0}", name); + return flid; + } + + /// + /// The xkk-flex-2022 layout, carried all the way round: one writing system whose Id is not + /// its LanguageTag, and a second whose Id IS that LanguageTag, each with text of its own. + /// LfMerge used to look the first up by LanguageTag when re-registering it, evict the second + /// and take its handle, so the two writing systems' text could end up under one of them. + /// Each must keep its own text, in LCM and in LF, through an export, an LF edit and an + /// import. The pair is one whose LanguageTag is the same under both SLDR datasets; xkk's own + /// is not, and the tests run on the older one. + /// + [Test] + public void RoundTrip_WritingSystemAndItsLanguageTagNamesake_EachKeepsItsOwnText() + { + // Setup: both writing systems in the analysis list, each with a gloss + const string id = "ca-Latn-AD", namesakeId = "ca-AD"; + WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager; + CoreWritingSystemDefinition real = AddWritingSystemWithId(id, namesakeId); + CoreWritingSystemDefinition namesake = wsManager.Set(namesakeId); + int realHandle = real.Handle, namesakeHandle = namesake.Handle; + Assert.That(namesakeHandle, Is.Not.EqualTo(realHandle)); + Guid entryGuid = Guid.Parse(TestEntryGuidStr); + var lcmEntry = (ILexEntry)_cache.ServiceLocator.GetObject(entryGuid); + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + { + _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(real); + _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(namesake); + lcmEntry.SensesOS[0].Gloss.set_String(realHandle, "glossa andorrana"); + lcmEntry.SensesOS[0].Gloss.set_String(namesakeHandle, "glossa de l'homonima"); + }); + + // Exercise: export, edit both in LF, import, export again + SutLcmToMongo.Run(_lfProj); + LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid); + Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo("glossa andorrana")); + Assert.That(lfEntry.Senses[0].Gloss[namesakeId].Value, Is.EqualTo("glossa de l'homonima")); + lfEntry.Senses[0].Gloss[id] = LfStringField.FromString("glossa andorrana, corregida a LF"); + lfEntry.Senses[0].Gloss[namesakeId] = LfStringField.FromString("glossa de l'homonima, corregida a LF"); + lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; + _conn.UpdateMockLfLexEntry(lfEntry); + var restores = new[] { AddLfInputSystem(id), AddLfInputSystem(namesakeId) }; + try + { + SutMongoToLcm.Run(_lfProj); + SutLcmToMongo.Run(_lfProj); + + // Verify: neither writing system took the other's place or its text + Assert.That(real.Handle, Is.EqualTo(realHandle)); + Assert.That(namesake.Handle, Is.EqualTo(namesakeHandle)); + Assert.That(wsManager.GetWsFromStr(id), Is.EqualTo(realHandle)); + Assert.That(wsManager.GetWsFromStr(namesakeId), Is.EqualTo(namesakeHandle)); + Assert.That(lcmEntry.SensesOS[0].Gloss.get_String(realHandle).Text, + Is.EqualTo("glossa andorrana, corregida a LF")); + Assert.That(lcmEntry.SensesOS[0].Gloss.get_String(namesakeHandle).Text, + Is.EqualTo("glossa de l'homonima, corregida a LF")); + lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid); + Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo("glossa andorrana, corregida a LF")); + Assert.That(lfEntry.Senses[0].Gloss[namesakeId].Value, Is.EqualTo("glossa de l'homonima, corregida a LF")); + Assert.That(_conn.GetInputSystems(_lfProj).Keys, Is.SupersetOf(new[] { id, namesakeId })); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } + + /// + /// LF's editor matches a field's input systems against an entry's keys exactly, so the two + /// must be spelled alike. The keys are spelled by Id; the input systems were spelled by + /// LanguageTag, so brb-flex-2022's Khmer text, keyed "km-Khmr-KH" under an input system + /// called "km-KH", showed as empty. + /// + [Test] + public void LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEverywhere() + { + // Setup: an analysis writing system held under a non-canonical Id, with a gloss in it. + const string id = "nl-Latn-BE"; + const string gloss = "Nederlandse glos"; + CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, "nl-BE"); + Guid entryGuid = Guid.Parse(TestEntryGuidStr); + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + { + _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); + var lcmEntry = (ILexEntry)_cache.ServiceLocator.GetObject(entryGuid); + lcmEntry.SensesOS[0].Gloss.set_String(ws.Handle, gloss); + }); + + // Exercise + SutLcmToMongo.Run(_lfProj); + + // Verify + Dictionary inputSystems = _conn.GetInputSystems(_lfProj); + Assert.That(inputSystems.Keys, Contains.Item(id)); + Assert.That(inputSystems[id].Tag, Is.EqualTo(id)); + Assert.That(inputSystems.Keys, Does.Not.Contain("nl-BE")); + LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid); + Assert.That(lfEntry.Senses[0].Gloss.Keys, Contains.Item(id), + "the multitext key and the input system should be spelled alike"); + Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo(gloss)); + Assert.That(_conn.LastAnalysisWss, Contains.Item(id)); + Assert.That(_conn.LastAnalysisWss, Does.Not.Contain("nl-BE")); + var customFieldTags = _conn.GetCustomFieldConfig(_lfProj).Values.OfType() + .SelectMany(field => field.InputSystems).ToList(); + Assert.That(customFieldTags, Contains.Item(id), "a custom field offering every analysis writing system"); + Assert.That(customFieldTags, Does.Not.Contain("nl-BE")); + } + + /// + /// testlangproj's current vernacular list names "qaa-Zxxx-x-kal-AUDIO", and LCM holds the + /// writing system as "qaa-Zxxx-x-kal-audio". It is in the vernacular list LF is given, so its + /// input system must be flagged vernacular too; the list's own Contains compares Ids exactly. + /// + [Test] + public void LcmToMongo_ListSpellingAnIdInAnotherCase_FlagsTheInputSystemAsInTheList() + { + Assert.That(_cache.LanguageProject.CurVernWss, Does.Contain(MixedCaseTag), "precondition"); + + SutLcmToMongo.Run(_lfProj); + + Assert.That(_conn.LastVernacularWss, Contains.Item(MixedCaseId)); + Assert.That(_conn.GetInputSystems(_lfProj)[MixedCaseId].VernacularWS, Is.True); + } + + [Test] + public void LcmToMongo_PronunciationListNamingAWritingSystemLcmDoesNotHave_LeavesItOut() + { + // Setup: a pronunciation list naming an Id LCM has no writing system for. (The same in + // the vernacular list would still fail, inside liblcm's own GetWritingSystemList.) + string vernacularId = _cache.LanguageProject.DefaultVernacularWritingSystem.Id; + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () => + _cache.LanguageProject.CurPronunWss = vernacularId + " zzz-x-nonesuch"); + + // Exercise + SutLcmToMongo.Run(_lfProj); + + // Verify: the list LF is given keeps the writing system LCM has and leaves out the other + Assert.That(_conn.LastPronunciationWss, Is.EqualTo(new[] { vernacularId })); + } + + [Test] + public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero() + { + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "zzz-x-nonesuch"), Is.EqualTo(0)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, ""), Is.EqualTo(0)); + Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, null), Is.EqualTo(0)); + } + + [Test] + public void BestStringAndWsId_NonCanonicalKey_StillMatchesThePreferredWritingSystem() + { + // The lexicon's single-string fields compared LF's keys with ws.Id, so "fr-Latn" never + // matched French and the English value was chosen although French comes first. + int wsFr = _cache.WritingSystemFactory.GetWsFromStr("fr"); + var multiText = new LfMultiText { + { "en", LfStringField.FromString("English") }, + { "fr-Latn", LfStringField.FromString("Français") }, + }; + + KeyValuePair best = multiText.BestStringAndWsId(new[] { wsFr, _wsEn }, _cache.WritingSystemFactory); + + Assert.That(best.Key, Is.EqualTo(wsFr)); + Assert.That(best.Value, Is.EqualTo("Français")); + } + + [Test] + public void BestStringAndWsId_FallsBackToTheFirstValueInAKnownWritingSystem() + { + // An unidentified key comes first, but a value LCM has no writing system for is no use. + var multiText = new LfMultiText { + { "zzz-x-nonesuch", LfStringField.FromString("Unplaceable") }, + { "en", LfStringField.FromString("English") }, + }; + + KeyValuePair best = multiText.BestStringAndWsId(new int[0], _cache.WritingSystemFactory); + + Assert.That(best.Key, Is.EqualTo(_wsEn)); + Assert.That(best.Value, Is.EqualTo("English")); + } + + /// + /// A single-string field chooses between two keys naming one writing system as a multitext + /// field does: the one spelled as the config spells it. Here the export's hidden value is + /// under the Id and the LF user's under the LanguageTag, which the config uses. + /// + [Test] + public void BestStringAndWsId_TwoKeysForOneWritingSystem_TakesTheConfigsSpelling() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("pl-Latn-PL", "pl-PL"); + var multiText = new LfMultiText { + { "pl-Latn-PL", LfStringField.FromString("the export's value") }, + { "pl-PL", LfStringField.FromString("the user's edit") }, + }; + var configuredTags = new HashSet(StringComparer.Ordinal) { "pl-PL" }; + + KeyValuePair best = multiText.BestStringAndWsId(new[] { ws.Handle }, _cache.WritingSystemFactory, configuredTags); + + Assert.That(best.Key, Is.EqualTo(ws.Handle)); + Assert.That(best.Value, Is.EqualTo("the user's edit")); + } + + /// + /// The key is chosen before empty values are left out: a value the LF user has cleared, + /// under the config's spelling, is not replaced by the export's hidden one. + /// + [Test] + public void BestStringAndWsId_ValueClearedUnderTheConfigsSpelling_IsNotReplacedByTheHiddenOne() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("cs-Latn-CZ", "cs-CZ"); + var multiText = new LfMultiText { + { "cs-Latn-CZ", LfStringField.FromString("the export's value") }, + { "cs-CZ", LfStringField.FromString("") }, + }; + var configuredTags = new HashSet(StringComparer.Ordinal) { "cs-CZ" }; + + KeyValuePair best = multiText.BestStringAndWsId(new[] { ws.Handle }, _cache.WritingSystemFactory, configuredTags); + + Assert.That(best.Value, Is.Null, "nothing should be placed for the cleared writing system"); + } + + [Test] + public void KeysPassedOver_NamesEachKeyWithTextLeftOutAndWhy() + { + CoreWritingSystemDefinition ws = AddWritingSystemWithId("nl-Latn-SR", "nl-SR"); + var multiText = new LfMultiText { + { "nl-Latn-SR", LfStringField.FromString("the export's value") }, + { "nl-SR", LfStringField.FromString("") }, + { "zzz-x-nonesuch", LfStringField.FromString("unplaceable") }, + { "en", LfStringField.FromString("placed") }, + { "fr", LfStringField.FromString("") }, + }; + var configuredTags = new HashSet(StringComparer.Ordinal) { "nl-SR", "en", "fr" }; + + var passedOver = multiText.KeysPassedOver(_cache.WritingSystemFactory, configuredTags); + + Assert.That(passedOver, Is.EquivalentTo(new[] { + new KeyValuePair("nl-Latn-SR", "nl-SR"), + new KeyValuePair("zzz-x-nonesuch", null) })); + } + + [Test] + public void SpanStrToTsString_NonCanonicalSpanLang_UsesTheCanonicalWritingSystem() + { + int wsFr = _cache.WritingSystemFactory.GetWsFromStr("fr"); + + var tss = ConvertMongoToLcmTsStrings.SpanStrToTsString( + "English français", _wsEn, _cache.WritingSystemFactory); + + Assert.That(tss.RunCount, Is.EqualTo(2)); + Assert.That(tss.get_WritingSystem(1), Is.EqualTo(wsFr)); + } + + /// + /// A span whose lang names no writing system LCM has used to make the string build throw, + /// abandoning the entry half-written. Its text is now kept in the string's own writing + /// system, and the lang reported. + /// + [Test] + public void SpanStrToTsString_SpanLangNamingNoWritingSystem_KeepsTheTextInTheMainWritingSystem() + { + var tss = ConvertMongoToLcmTsStrings.SpanStrToTsString( + "English unplaceable", _wsEn, _cache.WritingSystemFactory); + + Assert.That(tss.Text, Is.EqualTo("English unplaceable")); + for (int run = 0; run < tss.RunCount; run++) + Assert.That(tss.get_WritingSystem(run), Is.EqualTo(_wsEn)); + Assert.That(_env.Logger.GetMessages(), Does.Contain("span lang \"zzz-x-nonesuch\" names no writing system LCM has")); + } + + [Test] + public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis() + { + // Setup: a writing system LCM does not have yet, configured for the lexeme and for the + // definition, as "en" so often is. Being vernacular must not stop it being analysis too, + // or the definitions written in it would have no analysis writing system to go to. + const string tag = "qaa-x-both"; + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag); + var restores = new[] { + AddLfInputSystem(tag), + AddToConfigField(tag, "lexeme"), + AddToConfigField(tag, "senses", "definition"), + }; + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + ILangProject langProj = _cache.LanguageProject; + Assert.That(langProj.CurrentVernacularWritingSystems.Select(ws => ws.Id), Contains.Item(tag)); + Assert.That(langProj.CurrentAnalysisWritingSystems.Select(ws => ws.Id), Contains.Item(tag)); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } + + /// + /// An input system LF spells non-canonically, while the config's fields spell it + /// canonically: the lexeme field says "qaa-x-fresh" for the input system "qaa-x-qaa-fresh". The + /// writing system is created under the canonical form, and must still get the vernacular + /// role the lexeme field gives it; looked up under the input system's spelling only, it + /// was not found, and was made analysis. + /// + [Test] + public void LfWsToLcmWs_NewInputSystemSpelledOtherwiseThanInTheConfig_GetsTheConfigsRole() + { + const string inputSystemTag = "qaa-x-qaa-fresh", configTag = "qaa-x-fresh"; + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(configTag), "testlangproj should not contain {0}", configTag); + var restores = new[] { + AddLfInputSystem(inputSystemTag), + AddToConfigField(configTag, "lexeme"), + }; + + try + { + // Exercise + SutMongoToLcm.Run(_lfProj); + + // Verify + ILangProject langProj = _cache.LanguageProject; + Assert.That(LcmWritingSystemIds(), Contains.Item(configTag)); + Assert.That(langProj.CurrentVernacularWritingSystems.Select(ws => ws.Id), Contains.Item(configTag)); + Assert.That(langProj.CurrentAnalysisWritingSystems.Select(ws => ws.Id), Does.Not.Contain(configTag)); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } + + /// + /// Classifying a writing system counts the text in the whole lexicon, a second pass over + /// Mongo. Only a writing system new to LCM is classified, so with none the lexicon is read + /// once, for the entries themselves. + /// + [Test] + public void MongoToLcm_NoInputSystemNewToLcm_ReadsTheLexiconOnce() + { + int readsBefore = _conn.LexiconReads; + + SutMongoToLcm.Run(_lfProj); + + Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(1)); + } + + /// + /// With writing systems new to LCM, the lexicon is counted once more, and only once however + /// many there are: the classification is worked out for the first and kept for the rest. + /// + [Test] + public void MongoToLcm_InputSystemsNewToLcm_CountTheLexiconOnceMore() + { + var tags = new[] { "qaa-x-new", "qaa-x-newtoo" }; + var restores = new List(); + foreach (string tag in tags) + { + Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag); + restores.Add(AddLfInputSystem(tag)); + } + try + { + int readsBefore = _conn.LexiconReads; + + SutMongoToLcm.Run(_lfProj); + + Assert.That(LcmWritingSystemIds(), Is.SupersetOf(tags), "precondition: both should have been created"); + Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(2)); + } + finally + { + foreach (Action restore in restores) + restore(); + } + } + } +} diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs new file mode 100644 index 00000000..d5da98e1 --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) 2016-2018 SIL International +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +using System.Collections.Generic; +using LfMerge.Core.DataConverters; +using LfMerge.Core.LanguageForge.Model; +using NUnit.Framework; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// Counting the text a writing system actually holds, field by field. This is a pure function + /// over the Language Forge model, so the fixture needs neither Mongo nor an LCM project. It + /// must not call MongoConnectionDouble.Initialize, which throws once another fixture in this + /// assembly has registered the conventions. + /// + public class LfWritingSystemUsageTests + { + private static LfMultiText Text(params string[] tagThenValue) + { + var result = new LfMultiText(); + for (int i = 0; i < tagThenValue.Length; i += 2) + result[tagThenValue[i]] = new LfStringField { Value = tagThenValue[i + 1] }; + return result; + } + + private static LfLexEntry Entry(LfMultiText lexeme, params LfSense[] senses) + { + return new LfLexEntry { Lexeme = lexeme, Senses = new List(senses) }; + } + + private static LfSense Sense(LfMultiText gloss, params LfExample[] examples) + { + return new LfSense { Gloss = gloss, Examples = new List(examples) }; + } + + [Test] + public void FromLexicon_CountsOneEntryPerNonEmptyAlternative() + { + var lexicon = new[] { + Entry(Text("qaa-x-kal", "first", "en", "one")), + Entry(Text("qaa-x-kal", "second")), + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.EntriesCounted, Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_DoesNotCountEmptyOrWhitespaceOrMissingText() + { + // A field Language Forge offers but nobody has filled in is exactly what the counts are + // meant to tell apart from a field that is really in use. + var lexicon = new[] { + Entry(Text("qaa-x-kal", "", "en", " ", "fr", "real")), + new LfLexEntry { Lexeme = null }, + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.EntriesCounted, Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(0)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(0)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "fr"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_SkipsDeletedEntries() + { + // Deleted entries are tombstones kept so the deletion can be replayed; their text is + // not evidence that anyone is using that writing system. + var lexicon = new[] { + Entry(Text("qaa-x-kal", "kept")), + new LfLexEntry { Lexeme = Text("qaa-x-kal", "gone"), IsDeleted = true }, + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.EntriesCounted, Is.EqualTo(1)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_CountsNestedSenseAndExampleFields() + { + var lexicon = new[] { + Entry(Text("kal", "word"), + Sense(Text("en", "meaning"), + new LfExample { Sentence = Text("kal", "a sentence"), Translation = Text("en", "a translation") }, + new LfExample { Sentence = Text("kal", "another") })), + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Gloss, "en"), Is.EqualTo(1)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.ExampleSentence, "kal"), Is.EqualTo(2)); + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.ExampleTranslation, "en"), Is.EqualTo(1)); + } + + [Test] + public void NonEmptyCount_SumsOverSeveralFields() + { + var lexicon = new[] { + Entry(Text("kal", "word"), + Sense(null, new LfExample { Sentence = Text("kal", "a sentence") })), + }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.NonEmptyCount( + new[] { LfWritingSystemUsage.Lexeme, LfWritingSystemUsage.ExampleSentence }, "kal"), + Is.EqualTo(2)); + } + + [Test] + public void NonEmptyCount_MatchesTagsCaseInsensitively() + { + // Every LCM writing system lookup is case-insensitive, and the config and the lexicon do + // not always agree on the case of a tag. + var lexicon = new[] { Entry(Text("qaa-x-IPA-dupl1", "text")) }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-ipa-dupl1"), Is.EqualTo(1)); + } + + [Test] + public void FromLexicon_ToleratesAnEmptyOrNullLexicon() + { + Assert.That(LfWritingSystemUsage.FromLexicon(null).EntriesCounted, Is.EqualTo(0)); + Assert.That(LfWritingSystemUsage.FromLexicon(new LfLexEntry[0]).EntriesCounted, Is.EqualTo(0)); + Assert.That(LfWritingSystemUsage.FromLexicon(new LfLexEntry[0]) + .NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(0)); + } + + [Test] + public void TagsWithText_ListsOnlyTheWritingSystemsThatHoldText() + { + var lexicon = new[] { Entry(Text("kal", "word", "en", "", "fr", "mot")) }; + + LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon); + + Assert.That(usage.TagsWithText(LfWritingSystemUsage.Lexeme), Is.EquivalentTo(new[] { "kal", "fr" })); + Assert.That(usage.TagsWithText(LfWritingSystemUsage.Gloss), Is.Empty); + } + } +} diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs new file mode 100644 index 00000000..3494b1a0 --- /dev/null +++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs @@ -0,0 +1,790 @@ +using System; +using System.Collections.Generic; +using LfMerge.Core.DataConverters; +using LfMerge.Core.LanguageForge.Config; +using LfMerge.Core.LanguageForge.Model; +using NUnit.Framework; + +namespace LfMerge.Core.Tests.Lcm.DataConverters +{ + /// + /// LF has no vernacular/analysis flag on a writing system: which ones are vernacular is derived + /// from the project's own config, not from languageCode. Configs are built by hand here, since + /// this is a pure function needing no Mongo and no LCM project. The fixture must not call + /// MongoConnectionDouble.Initialize -- it throws if another fixture in this assembly has already + /// registered the conventions, and nothing here needs it. + /// + /// Most cases are named after a real project in the 2026-07-06 corpus that has that shape. + /// + public class VernacularClassificationTests + { + private static LfConfigMultiText Field(params string[] inputSystems) + { + return new LfConfigMultiText { InputSystems = new List(inputSystems) }; + } + + /// + /// Builds a config from dotted paths spelled as Mongo spells them, so nesting goes through + /// "fields" ("senses.fields.examples.fields.sentence"). + /// + private static LfProjectConfig Config(params (string Path, LfConfigFieldBase Field)[] fields) + { + var entry = new LfConfigFieldList(); + var senses = new LfConfigFieldList(); + var examples = new LfConfigFieldList(); + foreach ((string path, LfConfigFieldBase field) in fields) + { + if (path.StartsWith("senses.fields.examples.fields.")) + examples.Fields[path.Substring("senses.fields.examples.fields.".Length)] = field; + else if (path.StartsWith("senses.fields.")) + senses.Fields[path.Substring("senses.fields.".Length)] = field; + else + entry.Fields[path] = field; + } + if (examples.Fields.Count > 0) senses.Fields["examples"] = examples; + if (senses.Fields.Count > 0) entry.Fields["senses"] = senses; + return new LfProjectConfig { Entry = entry }; + } + + /// + /// One entry holding text at the given field path, so a lexicon of them gives a writing + /// system a known amount of text in a field of known role. + /// + private static LfLexEntry EntryWithTextAt(string fieldPath, string tag) + { + var text = new LfMultiText { { tag, LfStringField.FromString("text") } }; + var entry = new LfLexEntry(); + switch (fieldPath) + { + case LfWritingSystemUsage.Lexeme: entry.Lexeme = text; break; + case LfWritingSystemUsage.CitationForm: entry.CitationForm = text; break; + case LfWritingSystemUsage.Definition: + entry.Senses = new List { new LfSense { Definition = text } }; break; + case LfWritingSystemUsage.Gloss: + entry.Senses = new List { new LfSense { Gloss = text } }; break; + case LfWritingSystemUsage.ExampleSentence: + entry.Senses = new List { + new LfSense { Examples = new List { new LfExample { Sentence = text } } } }; + break; + case LfWritingSystemUsage.ExampleTranslation: + entry.Senses = new List { + new LfSense { Examples = new List { new LfExample { Translation = text } } } }; + break; + case LfWritingSystemUsage.Note: entry.Note = text; break; + case LfWritingSystemUsage.LiteralMeaning: entry.LiteralMeaning = text; break; + case LfWritingSystemUsage.Pronunciation: entry.Pronunciation = text; break; + case "etymology": entry.Etymology = text; break; + default: throw new ArgumentException("no test entry shape for " + fieldPath); + } + return entry; + } + + private static LfWritingSystemUsage Usage(params (string Path, string Tag, int Count)[] spec) + { + var lexicon = new List(); + foreach ((string path, string tag, int count) in spec) + for (int i = 0; i < count; i++) + lexicon.Add(EntryWithTextAt(path, tag)); + return LfWritingSystemUsage.FromLexicon(lexicon); + } + + /// + /// A null or empty tag in a config field's input systems names no writing system. The + /// 2026-10-01 corpus has none, but a null one made counting the text throw, which would + /// have ended the whole transfer. + /// + [Test] + public void ANullOrEmptyTagInTheConfigIsIgnored() + { + var config = Config(("lexeme", Field("kal", null)), ("senses.fields.gloss", Field("en", ""))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + + [Test] + public void LexemeAndCitationFormAreAlwaysVernacular() + { + var config = Config(("lexeme", Field("seh")), ("citationForm", Field("seh-fonipa-x-etic")), + ("senses.fields.definition", Field("en", "pt"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" })); + Assert.That(result.Unresolved, Is.Empty); + } + + [Test] + public void UnionsLexemeAndCitationFormWithoutDuplicating() + { + var config = Config( + ("lexeme", Field("qaa-x-kal", "qaa-fonipa-x-kal")), + ("citationForm", Field("qaa-x-kal", "seh")), + ("senses.fields.definition", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, + Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "seh" })); + } + + [Test] + public void IgnoresTheAnalysisAnchorFields() + { + var config = Config(("lexeme", Field("qaa-x-kal")), + ("senses.fields.definition", Field("en", "fr")), ("senses.fields.gloss", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal" })); + Assert.That(result.Vernacular, Does.Not.Contain("en")); + Assert.That(result.Vernacular, Does.Not.Contain("fr")); + } + + /// + /// apltw2016, the case that started this: its lexeme field uses a phonetic writing system + /// alongside the project language, and the rule this replaces -- vernacular is whatever + /// equals languageCode -- filed that phonetic one as analysis. + /// + [Test] + public void TreatsASecondLexemeWritingSystemAsVernacular() + { + var config = Config(("lexeme", Field("qaa-x-IPA-dupl1", "th"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th"); + Assert.That(result.Vernacular, Contains.Item("th")); + Assert.That(result.Vernacular, Contains.Item("qaa-x-IPA-dupl1"), + "a writing system the lexeme field uses is vernacular even though it is not the language code"); + } + + [Test] + public void MatchesTagsCaseInsensitively() + { + var config = Config(("lexeme", Field("qaa-x-IPA-dupl1"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th"); + Assert.That(result.Vernacular.Contains("qaa-x-ipa-dupl1"), Is.True, + "LCM's own writing system lookups are case-insensitive, so this should be too"); + } + + /// The 741-project shape: etymology written in the analysis language. + [Test] + public void EtymologyInAnAnalysisWritingSystemStaysAnalysis() + { + var config = Config(("lexeme", Field("yog")), ("senses.fields.gloss", Field("en")), + ("etymology", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "yog"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "yog" })); + Assert.That(result.Vernacular, Does.Not.Contain("en")); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + + /// The 378-project shape, e.g. test-india-sena-01. + [Test] + public void EtymologyInTheVernacularIsVernacular() + { + var config = Config(("lexeme", Field("seh", "seh-fonipa-x-etic")), + ("senses.fields.gloss", Field("en", "pt")), + ("etymology", Field("seh", "seh-fonipa-x-etic"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" })); + } + + /// + /// A vernacular field is evidence about the field, not about every writing system in it: + /// Portuguese is exclusive to the analysis anchors, so an etymology that also carries the + /// vernacular must not make it vernacular -- a new Portuguese writing system would then be + /// created vernacular-only, and the glosses in it would have no analysis writing system. + /// + [Test] + public void AVernacularFieldDoesNotMakeAnAnalysisOnlyWritingSystemVernacular() + { + var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("en", "pt")), + ("etymology", Field("seh", "pt"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "pt" })); + } + + /// + /// A field offering a vernacular-only and an analysis-only writing system side by side says + /// nothing about the role of the others it offers. It used to count as vernacular, because + /// that was checked first, which made kam-flex's Spanish and French vernacular; FLEx has + /// both as analysis. + /// + [Test] + public void AFieldOfferingBothRolesIsNotEvidenceOfEither() + { + var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("en")), + ("customField_senses_Wordlist", Field("seh", "en", "fr"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "fr" })); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "fr" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "fr" })); + } + + /// + /// grc-vie-flex: etymologies in Hebrew and Aramaic, neither vernacular nor analysis. With no + /// evidence either way they are both, so neither a vernacular-typed nor an analysis-typed + /// field is left without them. + /// + [Test] + public void WritingSystemsInNeitherAnchorAreBothRolesAndReported() + { + var config = Config(("lexeme", Field("grc")), ("senses.fields.gloss", Field("en", "vi")), + ("etymology", Field("hbo", "arc"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "grc"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "grc", "hbo", "arc" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "vi", "hbo", "arc" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo", "arc" })); + } + + /// + /// A French-only notes field overlaps neither anchor. French must reach the analysis list, or + /// an analysis-typed FieldWorks custom field would not offer it. + /// + [Test] + public void AnUnresolvedCustomFieldWritingSystemIsAnalysisToo() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("senses.fields.customField_senses_note", Field("fr"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + Assert.That(result.Analysis, Contains.Item("fr")); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "fr" })); + } + + /// + /// With no analysis anchor there is nothing to rule a field out, so every other field's + /// writing systems are unresolved -- and therefore both vernacular and analysis. Unless + /// their language has already been spoken for: qaa-fonipa-x-kal is a phonetic spelling of + /// the same language as the lexeme's qaa-x-kal, so it is vernacular like its relative, + /// while Hebrew keeps no such company and stays in doubt. + /// + [Test] + public void WithNoAnalysisAnchorEveryOtherFieldIsUnresolved() + { + var config = Config(("lexeme", Field("qaa-x-kal")), + ("etymology", Field("qaa-fonipa-x-kal", "hbo"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" })); + Assert.That(result.Vernacular, + Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "hbo" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "hbo" })); + } + + /// flh-flex: languageCode names a writing system only etymology uses. + [Test] + public void LexemeWinsWhenLanguageCodeDisagrees() + { + var config = Config(("lexeme", Field("flh-x-cm")), ("senses.fields.gloss", Field("en", "id")), + ("etymology", Field("flh-x-ortho"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "flh-x-ortho"); + Assert.That(result.Vernacular, Contains.Item("flh-x-cm")); + } + + /// odo: languageCode "th" is not among the project's writing systems at all. + [Test] + public void LexemeIsVernacularEvenWhenLanguageCodeIsAbsent() + { + var config = Config(("lexeme", Field("en")), + ("senses.fields.definition", Field("qaa-Syrc-IQ-x-syr", "en", "acm"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" })); + } + + /// + /// A writing system in both anchors is no evidence of any other field's role, so the Hebrew + /// sharing the etymology with it is in doubt. "en" itself is not: the anchors placed it. + /// + [Test] + public void AWritingSystemInBothAnchorsIsNotEvidence() + { + var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("en")), + ("etymology", Field("en", "hbo"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en", "hbo" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "hbo" })); + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" }), + "etymology overlaps neither anchor exclusively, so the writing system it alone places is in doubt"); + } + + /// + /// The "en" in lexeme and gloss alike: it is vernacular because the lexeme uses it, and it is + /// analysis because the gloss does. Being one must not stop it being the other. + /// + [Test] + public void AWritingSystemInBothAnchorsIsBothVernacularAndAnalysis() + { + var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal", "en" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + + /// + /// The example sentence and translation are anchors nested in a field list, so this covers + /// the recursion into one: the sentence is vernacular and the translation analysis. + /// + [Test] + public void TheNestedExampleAnchorsAreFound() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.definition", Field("fr")), + ("senses.fields.gloss", Field("en")), + ("senses.fields.examples.fields.sentence", Field("kal")), + ("senses.fields.examples.fields.translation", Field("en"))); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Vernacular, Does.Not.Contain("en"), + "the example translation is not a vernacular field"); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "fr" })); + Assert.That(result.Unresolved, Is.Empty); + } + + [Test] + public void ToleratesAPartiallyMissingNestedPath() + { + // senses exists but has no children, so the walk has to give up cleanly. + var config = Config(("lexeme", Field("qaa-x-kal")), ("senses", new LfConfigFieldList())); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal" })); + } + + [Test] + public void FallsBackToLanguageCodeWhenTheConfigSaysNothing() + { + Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(null, "fr").Vernacular, + Is.EquivalentTo(new[] { "fr" })); + Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(new LfProjectConfig(), "fr").Vernacular, + Is.EquivalentTo(new[] { "fr" })); + } + + [Test] + public void FallsBackToLanguageCodeWhenLexemeListsNoInputSystems() + { + var config = Config(("lexeme", Field())); + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "fr"); + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "fr" })); + } + + /// + /// The case that motivates counting at all: the config offers "en" for the lexeme field, so + /// the config alone calls it vernacular, but no headword is written in it. + /// + [Test] + public void AWritingSystemConfiguredForTheLexemeButUnusedThereIsNotVernacular() + { + var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 100), + (LfWritingSystemUsage.Gloss, "en", 100)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + Assert.That(result.Vernacular, Does.Not.Contain("en"), + "nothing is written in en in the lexeme field, so it is no evidence of vernacular"); + } + + /// + /// brb-flex-2022: the phonetic writing system has 4,030 headwords and one stray string in an + /// example reference, and FieldWorks has it as vernacular only. + /// + [Test] + public void AFewStrayStringsDoNotMakeAVernacularWritingSystemAnalysisAsWell() + { + var config = Config(("lexeme", Field("seh-fonipa-x-etic")), ("senses.fields.gloss", Field("en", "seh-fonipa-x-etic"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "seh-fonipa-x-etic", 4030), + (LfWritingSystemUsage.Gloss, "seh-fonipa-x-etic", 1), + (LfWritingSystemUsage.Gloss, "en", 4000)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh", usage); + + Assert.That(result.Vernacular, Contains.Item("seh-fonipa-x-etic")); + Assert.That(result.Analysis, Does.Not.Contain("seh-fonipa-x-etic")); + Assert.That(result.Analysis, Contains.Item("en")); + } + + /// + /// brb-flex-2022 again: the vernacular is also glossed in, 72/28, and FieldWorks really does + /// list it in both. + /// + [Test] + public void AWritingSystemUsedSubstantiallyForBothRolesGetsBoth() + { + var config = Config(("lexeme", Field("brb")), ("senses.fields.gloss", Field("brb", "en"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "brb", 7256), + (LfWritingSystemUsage.Gloss, "brb", 2801)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "brb", usage); + + Assert.That(result.Vernacular, Contains.Item("brb")); + Assert.That(result.Analysis, Contains.Item("brb")); + Assert.That(result.Unresolved, Is.Empty, "28% is a real second role, not an unknown one"); + } + + [Test] + public void TextOutweighsTheConfigInBothDirections() + { + // Configured the wrong way round: the lexeme offers "en" and the gloss offers "kal", + // but every headword is in kal and every gloss in en. + var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("kal"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 50), + (LfWritingSystemUsage.Gloss, "en", 50)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en", usage); + + Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" })); + Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" })); + } + + [Test] + public void AWritingSystemUsedOnlyInAFieldOfUnknownRoleIsStillUnresolved() + { + // Text in the etymology says nothing: the corpus has 33 projects writing etymologies in + // the vernacular and 50 in an analysis language. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("etymology", Field("hbo"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), ("etymology", "hbo", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" })); + } + + [Test] + public void AWritingSystemTheConfigForgotIsClassifiedFromItsTextAnyway() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Lexeme, "kal-fonipa", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Contains.Item("kal-fonipa")); + } + + [Test] + public void AnEmptyLexiconLeavesTheConfigInCharge() + { + var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en"))); + + var withoutData = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", + LfWritingSystemUsage.FromLexicon(new LfLexEntry[0])); + var configOnly = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + + Assert.That(withoutData.Vernacular, Is.EquivalentTo(configOnly.Vernacular)); + Assert.That(withoutData.Analysis, Is.EquivalentTo(configOnly.Analysis)); + Assert.That(withoutData.Vernacular, Contains.Item("en")); + } + + /// + /// FieldWorks fixes the role of these fields, and the corpus agrees: counting the projects + /// whose text in each is exclusively one role, the example sentence runs 332 vernacular to 5 + /// analysis, and the translation, entry note and literal meaning run the other way. + /// + [TestCase("senses.fields.examples.fields.sentence", true)] + [TestCase("senses.fields.examples.fields.translation", false)] + [TestCase("note", false)] + [TestCase("literalMeaning", false)] + public void TheFieldsFieldWorksFixesTheRoleOfAreEvidenceToo(string fieldPath, bool isVernacular) + { + // Only the field under test carries any text for "xyz", so nothing else can decide it. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + (fieldPath, Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (fieldPath, "xyz", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Does.Not.Contain("xyz")); + if (isVernacular) + { + Assert.That(result.Vernacular, Contains.Item("xyz")); + Assert.That(result.Analysis, Does.Not.Contain("xyz")); + } + else + { + Assert.That(result.Analysis, Contains.Item("xyz")); + Assert.That(result.Vernacular, Does.Not.Contain("xyz")); + } + } + + /// + /// The pronunciation field is not an anchor. It looks vernacular in the corpus at 114 + /// projects to 16, but 12% is above the share a single writing system may hold in the other + /// role and still count as one thing, so projects evidently use it for more than one. Being + /// offered there, with nothing written in it, therefore settles nothing. + /// + [Test] + public void ThePronunciationFieldIsNotAnAnchor() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("pronunciation", Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Contains.Item("xyz")); + } + + /// + /// But text in it makes a writing system vernacular, since FieldWorks draws its + /// pronunciation writing systems from the vernacular ones. xin-flex's "xin" holds 226 + /// pronunciations and nothing else. + /// + [Test] + public void PronunciationTextMakesAWritingSystemVernacular() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("pronunciation", Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "xyz", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Does.Not.Contain("xyz")); + Assert.That(result.Vernacular, Contains.Item("xyz")); + Assert.That(result.Analysis, Does.Not.Contain("xyz")); + } + + /// + /// A writing system offered only in a custom field, with pronunciation text and nothing + /// else, is vernacular only, by design. Without the pronunciation text the custom field + /// would say nothing about it and it would be both; with it, it counts as placed, so that + /// field cannot put it in doubt. In the 2026-10-01 corpus every writing system in that + /// position is a pronunciation writing system -- a phonetic, IPA or audio one -- for which + /// vernacular only is the right answer. + /// + /// The writing system's language is one the project has nowhere else: a phonetic spelling of + /// the vernacular, "kal-fonipa", would be made vernacular only by its language whether or not + /// it had pronunciation text, and the test would not show the rule at work. + /// + [TestCase(true)] + [TestCase(false)] + public void PronunciationTextAloneMakesAWritingSystemOfferedOnlyInACustomFieldVernacularOnly(bool hasPronunciationText) + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_senses_Phonetic", Field("xyz-fonipa"))); + var counts = new List<(string, string, int)> { + (LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10) }; + if (hasPronunciationText) + counts.Add((LfWritingSystemUsage.Pronunciation, "xyz-fonipa", 1)); + var usage = Usage(counts.ToArray()); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Contains.Item("xyz-fonipa")); + Assert.That(result.Analysis.Contains("xyz-fonipa"), Is.EqualTo(!hasPronunciationText), + "with pronunciation text, vernacular only; without, both"); + Assert.That(result.Unresolved.Contains("xyz-fonipa"), Is.EqualTo(!hasPronunciationText)); + } + + /// + /// Pronunciation text only adds the vernacular role; it does not take away an analysis role + /// the config gives. In dtlat-flex a single pronunciation string in Swedish would otherwise + /// make Swedish, configured for the gloss, vernacular only. + /// + [Test] + public void PronunciationTextDoesNotTakeAwayAnAnalysisRole() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en", "sv")), + ("pronunciation", Field("kal", "sv"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "sv", 1)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Vernacular, Contains.Item("sv")); + Assert.That(result.Analysis, Contains.Item("sv")); + } + + /// + /// A writing system offered both in a field that says nothing about its role and in one + /// whose company settles it is settled by the latter. The field of no role is listed first + /// on purpose: placing as each field came, it used to put the writing system in doubt. + /// + [TestCase("kal", true)] + [TestCase("en", false)] + public void AWritingSystemInDoubtIsSettledByTheCompanyItKeeps(string companion, bool isVernacular) + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Doubtful", Field("abc", "xyz")), + ("customField_entry_Settled", Field(companion, "abc"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Does.Not.Contain("abc"), + "abc shares a field with {0}, which is settled, so abc is settled too", companion); + Assert.That(result.Vernacular.Contains("abc"), Is.EqualTo(isVernacular)); + Assert.That(result.Analysis.Contains("abc"), Is.EqualTo(!isVernacular)); + // xyz is offered only in the field of no role, so the field loop puts it in doubt; only + // the company it keeps there, abc, settles it + Assert.That(result.Unresolved, Does.Not.Contain("xyz"), "xyz is settled by the company it keeps"); + Assert.That(result.Vernacular.Contains("xyz"), Is.EqualTo(isVernacular)); + Assert.That(result.Analysis.Contains("xyz"), Is.EqualTo(!isVernacular)); + } + + /// + /// The answer must not depend on the order the config lists its fields in. Here en-fonipa is + /// offered in a vernacular field and an analysis field, so it plays both roles. Placing as + /// each field came, a field of no role listed before them put it in doubt, and language + /// affinity with "en" then stripped its vernacular role; listed after them, it did not. + /// + [TestCase(true, TestName = "TheFieldOrderDoesNotChangeTheAnswerWithTheNoRoleFieldFirst")] + [TestCase(false, TestName = "TheFieldOrderDoesNotChangeTheAnswerWithTheNoRoleFieldLast")] + public void TheFieldOrderDoesNotChangeTheAnswer(bool noRoleFieldFirst) + { + var noRole = ("customField_entry_NoRole", (LfConfigFieldBase)Field("en-fonipa")); + var fields = new List<(string, LfConfigFieldBase)> { + ("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Vernacular", Field("kal", "en-fonipa")), + ("customField_entry_Analysis", Field("en", "en-fonipa")), + }; + fields.Insert(noRoleFieldFirst ? 2 : fields.Count, noRole); + var config = Config(fields.ToArray()); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal"); + + Assert.That(result.Vernacular, Contains.Item("en-fonipa")); + Assert.That(result.Analysis, Contains.Item("en-fonipa")); + Assert.That(result.Unresolved, Does.Not.Contain("en-fonipa")); + } + + [Test] + public void WritingSystemsWhoseOnlyCompanyIsEachOtherStayUnresolved() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Doubtful", Field("abc", "xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "abc", "xyz" })); + } + + /// + /// "en" is in both anchors, so it plays both roles. Turning up alone in a sense's general + /// note, a field that gives no evidence, must not put it in doubt: if it did, the company it keeps in the + /// lexeme field (the vernacular) would then strip it of its analysis role, and a new "en" + /// writing system would be created vernacular only. + /// + [Test] + public void AWritingSystemAlreadyPlacedIsNotPutInDoubtByAFieldWithNoEvidence() + { + var config = Config(("lexeme", Field("seh", "en")), ("senses.fields.gloss", Field("en")), + ("senses.fields.definition", Field("en")), ("senses.fields.generalNote", Field("en"))); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh"); + + Assert.That(result.Unresolved, Is.Empty); + Assert.That(result.Vernacular, Contains.Item("en")); + Assert.That(result.Analysis, Contains.Item("en")); + } + + [Test] + public void AWritingSystemWithNoCompanyAtAllStaysUnresolved() + { + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Note", Field("xyz"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "xyz" })); + } + + /// + /// A writing system with nothing written in it and no company still has a language, and the + /// other writing systems of that language have already been placed. + /// + [TestCase("seh", "seh-fonipa-x-etic", true, TestName = "AffinityPhoneticVariantOfTheVernacular")] + [TestCase("qaa-x-kal", "qaa-Zxxx-x-kal-audio", true, TestName = "AffinityAudioVariantOfTheVernacular")] + [TestCase("qaa-x-kal", "qaa-x-kal-dupl1", true, TestName = "AffinityDuplicateOfTheVernacular")] + [TestCase("x-kal", "x-kal-dupl1", true, TestName = "AffinityDuplicateOfAWhollyPrivateUseVernacular")] + public void AWritingSystemOfTheSameLanguageAsTheVernacularIsVernacular( + string vernacularTag, string relative, bool isVernacular) + { + var config = Config(("lexeme", Field(vernacularTag)), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field(relative))); + var usage = Usage((LfWritingSystemUsage.Lexeme, vernacularTag, 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, vernacularTag, usage); + + Assert.That(result.Unresolved, Is.Empty); + Assert.That(result.Vernacular.Contains(relative), Is.EqualTo(isVernacular)); + Assert.That(result.Analysis.Contains(relative), Is.EqualTo(!isVernacular)); + } + + [Test] + public void AWritingSystemOfTheSameLanguageAsTheAnalysisLanguageIsAnalysis() + { + // Script and region are not the language: "en-GB" is English. + var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field("en-GB"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage); + + Assert.That(result.Unresolved, Is.Empty); + Assert.That(result.Analysis, Contains.Item("en-GB")); + Assert.That(result.Vernacular, Does.Not.Contain("en-GB")); + } + + /// + /// Under "qaa" the private-use subtags are the language, so two of them are two languages. + /// + [Test] + public void ADifferentPrivateUseLanguageIsNotTheSameLanguage() + { + var config = Config(("lexeme", Field("qaa-x-kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field("qaa-x-hbo"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "qaa-x-kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-x-hbo" })); + } + + /// + /// A tag can be private-use from its first subtag (the corpus has "x-IPA" and "x-v"), and + /// then too the private-use subtags are the language. + /// + [Test] + public void AWhollyPrivateUseTagIsNotTheSameLanguageAsAnother() + { + var config = Config(("lexeme", Field("x-kal")), ("senses.fields.gloss", Field("en")), + ("customField_entry_Extra", Field("x-hbo"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "x-kal", 10), + (LfWritingSystemUsage.Gloss, "en", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "x-kal", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "x-hbo" })); + } + + [Test] + public void ALanguageUsedForBothRolesSettlesNothing() + { + // seh is the vernacular and seh-fonipa is glossed in, so the language says nothing. + var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("seh-fonipa")), + ("customField_entry_Extra", Field("seh-Latn"))); + var usage = Usage((LfWritingSystemUsage.Lexeme, "seh", 10), + (LfWritingSystemUsage.Gloss, "seh-fonipa", 10)); + + var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh", usage); + + Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "seh-Latn" })); + } + + [Test] + public void ReturnsNothingWhenThereIsNoConfigAndNoLanguageCode() + { + Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(null, null).Vernacular, + Is.Empty); + } + } +} diff --git a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs index 3a7d8b55..1f425769 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016 SIL International +// Copyright (c) 2016 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; @@ -58,18 +58,14 @@ public void Action_IsInitialClone_ShouldPopulateMongoInputSystems() // Verify const int expectedNumVernacularWS = 3; const int expectedNumAnalysisWS = 2; - // TODO: Investigate why qaa-Zxxx-x-kal-audio is in CurrentVernacularWritingSystems, but somehow in - // UpdateMongoDbFromLcm.LcmWsToLfWs() is not contained in _cache.LangProject.CurrentVernacularWritingSystems - const string notVernacularWs = "qaa-Zxxx-x-kal-audio"; lfWsList = _conn.GetInputSystems(lfProject); var languageProj = lfProject.FieldWorksProject.Cache.LangProject; + // Every one, qaa-Zxxx-x-kal-audio included: the list spells it qaa-Zxxx-x-kal-AUDIO, + // which the list's own Contains, comparing Ids exactly, did not match foreach (var lcmVernacularWs in languageProj.CurrentVernacularWritingSystems) - { - if (lcmVernacularWs.Id != notVernacularWs) - Assert.That(lfWsList[lcmVernacularWs.Id].VernacularWS); - } + Assert.That(lfWsList[lcmVernacularWs.Id].VernacularWS, "{0} should be flagged vernacular", lcmVernacularWs.Id); Assert.That(languageProj.CurrentVernacularWritingSystems.Count, Is.EqualTo(expectedNumVernacularWS)); foreach (var lcmAnalysisWs in languageProj.CurrentAnalysisWritingSystems) diff --git a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs index 0013eca5..86dc7aa1 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs @@ -745,6 +745,39 @@ public void Action_NeverSyncedProjectWhoseEntriesLfMergeWrote_KeepsFieldWorksVer Assert.That(_counts.Modified, Is.EqualTo(0)); } + [Test] + public void Action_NeverSyncedProjectWithEntryImportedIntoLf_SendsItToFieldWorks() + { + // Setup: an LF project's first Send/Receive, as when one is migrated. A LIFT import into + // LF leaves no user references on its entries, the same as LfMerge's own writes do, so + // the entry is stored with UpdateRecord rather than as an LF user's edit. + var lfProj = _lfProj; + _conn.SetLastSyncedDate(lfProj, MagicValues.UnixEpoch); + FwProject fwProject = lfProj.FieldWorksProject; + string vernacularWS = fwProject.Cache.LanguageProject.DefaultVernacularWritingSystem.Id; + Guid importedGuid = Guid.NewGuid(); + var imported = new LfLexEntry(); + imported.Guid = importedGuid; + imported.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, "imported into LF"); + imported.AuthorInfo = new LfAuthorInfo(); + imported.AuthorInfo.CreatedDate = new DateTime(2016, 4, 10, 0, 0, 0, DateTimeKind.Utc); + imported.AuthorInfo.ModifiedDate = imported.AuthorInfo.CreatedDate; + imported.DateModified = new DateTime(2016, 7, 25, 0, 0, 0, DateTimeKind.Utc); + _conn.UpdateRecord(lfProj, imported); + + // Exercise + SutMongoToLcm.Run(lfProj); + + // Verify + ILexEntry sent; + Assert.That(fwProject.ServiceLocator.GetInstance().TryGetObject(importedGuid, out sent), Is.True, + "an entry only LF has must reach FieldWorks, or the transfer back to Mongo marks it deleted"); + Assert.That(sent.LexemeFormOA.Form.VernacularDefaultWritingSystem.Text, Is.EqualTo("imported into LF")); + Assert.That(_counts.Added, Is.EqualTo(1)); + Assert.That(_counts.Modified, Is.EqualTo(0)); + Assert.That(_counts.Deleted, Is.EqualTo(0)); + } + // TODO: Move custom field tests to their own test class, and move these helper functions with them public void AddCustomFieldToSense(LfSense sense, string fieldName, BsonValue fieldValue) { diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index e8667743..9584c5e2 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -116,7 +116,9 @@ public static void Initialize() } private readonly Dictionary _storedInputSystems = new Dictionary(); - private readonly Dictionary _storedLfLexEntries = new Dictionary(); + // Held as documents, as Mongo holds them: an entry LF wrote keeps what LfMerge's own + // serializer would leave out, such as a multitext holding an empty value + private readonly Dictionary _storedLfLexEntries = new Dictionary(); private readonly Dictionary _storedLfOptionLists = new Dictionary(); private Dictionary _storedCustomFieldConfig = new Dictionary(); private Dictionary _storedLastSyncDate = new Dictionary(); @@ -154,11 +156,19 @@ public Dictionary GetInputSystems(ILfProject projec return _storedInputSystems; } + /// The current vernacular, analysis and pronunciation lists the last export gave. + public List LastVernacularWss { get; private set; } + public List LastAnalysisWss { get; private set; } + public List LastPronunciationWss { get; private set; } + public bool SetInputSystems(ILfProject project, Dictionary inputSystems, List vernacularWss, List analysisWss, List pronunciationWss) { foreach (var ws in inputSystems.Keys) _storedInputSystems[ws] = inputSystems[ws]; + LastVernacularWss = vernacularWss; + LastAnalysisWss = analysisWss; + LastPronunciationWss = pronunciationWss; if (project.IsInitialClone) { @@ -182,30 +192,36 @@ public Dictionary GetCustomFieldConfig(ILfProject pro return _storedCustomFieldConfig; } - public void UpdateMockLfLexEntry(BsonDocument mockData) - { - LfLexEntry data = BsonSerializer.Deserialize(mockData); - UpdateMockLfLexEntry(data); - } - // The user SampleData's entries were last modified by private static readonly ObjectId LfUserRef = ObjectId.Parse("561b666c0f87096a35c3cf2d"); /// /// Store an entry the way an LF user's edit does: LF's MapperModel::write stamps DateModified, /// and LexEntryCommands::updateEntry records who made the edit. LfMerge's own writes go - /// through UpdateRecord, which does neither. + /// through UpdateRecord, which does neither. The document is stored as given, empty values + /// and all, since LF writes what its editor sends. /// - public void UpdateMockLfLexEntry(LfLexEntry mockData) + public void UpdateMockLfLexEntry(BsonDocument mockData) { - Guid guid = mockData.Guid ?? Guid.Empty; - var stored = DeepCopy(mockData); - stored.DateModified = DateTime.UtcNow; - if (stored.AuthorInfo != null && stored.AuthorInfo.ModifiedByUserRef == null) - stored.AuthorInfo.ModifiedByUserRef = LfUserRef; + var stored = (BsonDocument)mockData.DeepClone(); + Guid guid = BsonSerializer.Deserialize(stored).Guid ?? Guid.Empty; + stored["dateModified"] = new BsonDateTime(DateTime.UtcNow); + BsonValue authorInfo; + if (stored.TryGetValue("authorInfo", out authorInfo) && authorInfo.IsBsonDocument) + { + BsonValue userRef; + if (!authorInfo.AsBsonDocument.TryGetValue("modifiedByUserRef", out userRef) || userRef.IsBsonNull) + authorInfo.AsBsonDocument["modifiedByUserRef"] = LfUserRef; + } _storedLfLexEntries[guid] = stored; } + /// As UpdateMockLfLexEntry(BsonDocument), for an entry built as an object. + public void UpdateMockLfLexEntry(LfLexEntry mockData) + { + UpdateMockLfLexEntry(mockData.ToBsonDocument()); + } + public void UpdateMockOptionList(BsonDocument mockData) { LfOptionList data = BsonSerializer.Deserialize(mockData); @@ -220,14 +236,14 @@ public void UpdateMockOptionList(LfOptionList mockData) public IEnumerable GetLfLexEntries() { - return new List(_storedLfLexEntries.Values.Select(entry => DeepCopy(entry))); + return new List(_storedLfLexEntries.Values.Select(entry => BsonSerializer.Deserialize(entry))); } public LfLexEntry GetLfLexEntryByGuid(ILfProject _project, Guid key) { - LfLexEntry result; + BsonDocument result; if (_storedLfLexEntries.TryGetValue(key, out result)) - return result; + return BsonSerializer.Deserialize(result); return null; } @@ -238,10 +254,26 @@ public IEnumerable GetLfOptionLists() public IEnumerable GetRecords(ILfProject project, string collectionName, Expression> filter) { - return GetRecords(project, collectionName).Where(filter.Compile()); + // A filtered request finds a few records, not a pass over the collection, so it is not + // counted in LexiconReads + return Records(collectionName).Where(filter.Compile()); } + /// + /// How many times the whole lexicon has been asked for, each request being a pass over + /// Mongo. Filtered requests, such as the comment converter's one per comment, are not + /// counted. + /// + public int LexiconReads { get; private set; } + public IEnumerable GetRecords(ILfProject project, string collectionName) + { + if (collectionName == MagicStrings.LfCollectionNameForLexicon) + LexiconReads++; + return Records(collectionName); + } + + private IEnumerable Records(string collectionName) { switch (collectionName) { @@ -257,7 +289,8 @@ public IEnumerable GetRecords(ILfProject project, string c public Dictionary GetAllModifiedDatesForEntries(ILfProject project) { - return _storedLfLexEntries.ToDictionary(kv => kv.Key, kv => kv.Value.AuthorInfo.ModifiedDate); + return _storedLfLexEntries.ToDictionary(kv => kv.Key, + kv => BsonSerializer.Deserialize(kv.Value).AuthorInfo.ModifiedDate); } public LfOptionList GetLfOptionListByCode(ILfProject project, string listCode) @@ -284,7 +317,7 @@ public IMongoDatabase GetMainDatabase() public bool UpdateRecord(ILfProject project, LfLexEntry data) { - _storedLfLexEntries[data.Guid ?? Guid.Empty] = DeepCopy(data); + _storedLfLexEntries[data.Guid ?? Guid.Empty] = data.ToBsonDocument(); return true; } @@ -302,7 +335,7 @@ public bool RemoveRecord(ILfProject project, Guid guid) public bool SetLastSyncedDate(ILfProject project, DateTime? newSyncedDate) { - // Mongo keeps dates to the millisecond, as DeepCopy does for the entries' dates + // Mongo keeps dates to the millisecond, as the entries' dates are kept here in BSON if (newSyncedDate != null) newSyncedDate = newSyncedDate.Value.AddTicks(-(newSyncedDate.Value.Ticks % TimeSpan.TicksPerMillisecond)); _storedLastSyncDate[project.ProjectCode] = newSyncedDate; diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs index 3a736398..f560084f 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -156,11 +156,7 @@ Dictionary lfCustomFieldTypes // Single line or MultiText fields need writing systems int fieldWs = LcmMetaData.GetFieldWs(flid); // That's a "magic" ws, which we need to expand into a (list of) real writing system(s). -#if FW8_COMPAT - var wsesForThisField = new List(); -#else var wsesForThisField = new List(); -#endif // GetWritingSystemList() in FW 8.3 is buggy and doesn't properly handle the kwsAnal and kwsVern cases, so we handle them here instead. switch (fieldWs) { case WritingSystemServices.kwsAnal: @@ -173,11 +169,10 @@ Dictionary lfCustomFieldTypes wsesForThisField = WritingSystemServices.GetWritingSystemList(cache, fieldWs, forceIncludeEnglish: false); break; } -#if FW8_COMPAT - IEnumerable inputSystems = wsesForThisField.Select(LcmWs => LcmWs.Id); -#else - IEnumerable inputSystems = wsesForThisField.Select(LcmWs => LcmWs.LanguageTag); -#endif + // Spelled as every multitext key is, by GetStrFromWs: LF matches a field's input + // systems against an entry's keys exactly. See ConvertLcmToMongoLexicon.LfTagOf. + IEnumerable inputSystems = wsesForThisField.Select( + LcmWs => servLoc.WritingSystemManager.GetStrFromWs(LcmWs.Handle)); // GetWritingSystemList returns all analysis WSes even when asked for just one, so if this // is a single-line custom field, trim the WSes down to just the first one if (lfCustomFieldType.StartsWith("Single")) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index 413bdd9a..0a74f7be 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -14,6 +14,7 @@ using MongoDB.Bson; using SIL.LCModel; using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.WritingSystems; using SIL.Progress; namespace LfMerge.Core.DataConverters @@ -65,15 +66,9 @@ public ConvertLcmToMongoLexicon(ILfProject lfProject, ILogger logger, IMongoConn // Reconcile writing systems from Lcm and Mongo Dictionary lfWsList = LcmWsToLfWs(); - #if FW8_COMPAT - List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(ws => ws.Id).ToList(); - List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(ws => ws.Id).ToList(); - List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(ws => ws.Id).ToList(); - #else - List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(ws => ws.LanguageTag).ToList(); - List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(ws => ws.LanguageTag).ToList(); - List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(ws => ws.LanguageTag).ToList(); - #endif + List VernacularWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentVernacularWritingSystems, "vernacular"); + List AnalysisWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems, "analysis"); + List PronunciationWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems, "pronunciation"); Connection.SetInputSystems(LfProject, lfWsList, VernacularWss, AnalysisWss, PronunciationWss); ListConverters = new Dictionary(); @@ -585,15 +580,77 @@ private static string LcmPictureFilenameToLfPictureFilename(string LcmInternalFi return Regex.Replace(LcmInternalFilename.Normalize(System.Text.NormalizationForm.FormC), @"^Pictures[/\\]", ""); } + /// + /// How Language Forge spells an LCM writing system: exactly as every multitext key LfMerge + /// writes is spelled, since those come from GetStrFromWs too (see LfMultiText). That is the + /// writing system's Id, falling back to its LanguageTag should it have no Id. + /// + /// Input systems and the config's field lists must be spelled the same way, because LF's + /// editor matches a field's input systems against an entry's keys exactly and + /// case-sensitively. They were spelled by LanguageTag, which in older projects is not the + /// Id: brb-flex-2022's Khmer is Id "km-Khmr-KH" with LanguageTag "km-KH", and che-flex's + /// vernacular is Id "ce-la" with LanguageTag "ce-LA", so LF showed all of that text as empty. + /// + private string LfTagOf(CoreWritingSystemDefinition ws) + { + return ServiceLocator.WritingSystemManager.GetStrFromWs(ws.Handle); + } + + /// + /// The LF tags of one of the project's current writing-system lists, leaving out any Id the + /// list names that LCM has no writing system for, rather than ending the transfer. That + /// protects the pronunciation list. A missing Id in a current vernacular or analysis list is + /// almost always in the full list too, and LcmWsToLfWs, which reads the full lists through + /// AllWritingSystems, still fails on it first -- as FieldWorks itself would. + /// + /// The list is read with foreach on purpose. liblcm's enumerator yields null for such an + /// Id, but its indexer and CopyTo throw KeyNotFoundException, and LINQ's Select and ToList + /// use those on an IList. + /// + private List LfTagsOf(IEnumerable wss, string listName) + { + var tags = new List(); + int missing = 0; + foreach (CoreWritingSystemDefinition ws in wss) + { + if (ws == null) + missing++; + else + tags.Add(LfTagOf(ws)); + } + if (missing > 0) + Logger.Warning("LcmToMongo: the current {0} writing systems list names {1} writing system(s) that LCM does not have; leaving them out", + listName, missing); + return tags; + } + + /// + /// The handles of the writing systems in one of the project's current lists, leaving out any + /// Id it names that LCM has no writing system for; see LfTagsOf. + /// + private static HashSet Handles(IEnumerable wss) + { + var handles = new HashSet(); + foreach (CoreWritingSystemDefinition ws in wss) + { + if (ws != null) + handles.Add(ws.Handle); + } + return handles; + } + /// /// Converts Lcm writing systems to LF input systems /// /// The list of LF input systems. private Dictionary LcmWsToLfWs() { - // Using var here so that we'll stay compatible with both FW 8 and 9 (the type of these two lists changed between 8 and 9). - var vernacularWSList = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems; - var analysisWSList = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems; + // By handle, from the same enumerator LfTagsOf reads the lists with. The lists' own + // Contains compares Ids exactly, while LCM resolves them ignoring case, so a list + // naming "qaa-Zxxx-x-kal-AUDIO" for the writing system "qaa-Zxxx-x-kal-audio" put it in + // the vernacular list LF is given but flagged it as not vernacular. + var vernacularHandles = Handles(ServiceLocator.LanguageProject.CurrentVernacularWritingSystems); + var analysisHandles = Handles(ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems); var lfWsList = new Dictionary(); foreach (var LcmWs in ServiceLocator.LanguageProject.AllWritingSystems) @@ -605,20 +662,12 @@ private Dictionary LcmWsToLfWs() Abbreviation = LcmWs.Abbreviation, IsRightToLeft = LcmWs.RightToLeftScript, LanguageName = LcmWs.LanguageName, - #if FW8_COMPAT - Tag = LcmWs.Id, - #else - Tag = LcmWs.LanguageTag, - #endif - VernacularWS = vernacularWSList.Contains(LcmWs), - AnalysisWS = analysisWSList.Contains(LcmWs) + Tag = LfTagOf(LcmWs), + VernacularWS = vernacularHandles.Contains(LcmWs.Handle), + AnalysisWS = analysisHandles.Contains(LcmWs.Handle) }; - #if FW8_COMPAT - lfWsList[LcmWs.Id] = lfWs; - #else - lfWsList[LcmWs.LanguageTag] = lfWs; - #endif + lfWsList[lfWs.Tag] = lfWs; } return lfWsList; } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index a8de39d0..b301b2b3 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -25,14 +25,21 @@ public class ConvertMongoToLcmCustomField private IFwMetaDataCacheManaged lcmMetaData; private ILogger logger; private int wsEn; + // Fields already reported as not writable, so each is reported once per sync rather than + // once for every entry, sense or example that has it + private readonly HashSet unwritableFieldsReported = new HashSet(); + // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm + private readonly ISet configuredTags; - public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn) + public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn, + ISet configuredTags = null) { this.cache = cache; this.servLoc = serviceLocator; this.lcmMetaData = (IFwMetaDataCacheManaged)cache.MetaDataCacheAccessor; this.logger = logger; this.wsEn = wsEn; + this.configuredTags = configuredTags; } public Guid ParseGuidOrDefault(string input) @@ -89,7 +96,9 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui if (fieldName == null) return false; - // Valid field types in LCM are GenDate, Integer, String, OwningAtomic, ReferenceAtomic, and ReferenceCollection, so that's all we implement. + // Valid field types in LCM are GenDate, Integer, String, MultiUnicode, OwningAtomic, + // ReferenceAtomic, and ReferenceCollection, so that's all we implement. MultiUnicode is the + // one every text custom field created in LF has, since LF has no single-string custom field. switch (fieldType) { case CellarPropertyType.GenDate: @@ -176,7 +185,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui if (multiPara.InputSystem == null) wsId = lcmMetaData.GetFieldWs(flid); else - wsId = servLoc.WritingSystemFactory.GetWsFromStr(multiPara.InputSystem); + wsId = LanguageTags.WsIdFromLfTag(servLoc.WritingSystemFactory, multiPara.InputSystem); ConvertUtilities.SetCustomStTextValues(text, multiPara.Paragraphs, wsId); return true; @@ -356,17 +365,53 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui return true; } + case CellarPropertyType.MultiUnicode: + { + var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); + + var existingWsIds = new List(); + ITsMultiString oldValues = data.get_MultiStringProp(hvo, flid); + if (oldValues != null) + { + for (int index = 0; index < oldValues.StringCount; index++) + { + int oldWsId; + oldValues.GetStringFromIndex(index, out oldWsId); + existingWsIds.Add(oldWsId); + } + } + + // The same writer as the built-in multitext fields, reaching LCM through + // ISilDataAccess because a custom field has no IMultiAccessorBase. + return valueAsMultiText.WriteToLcm(existingWsIds, + wsId => data.get_MultiStringAlt(hvo, flid, wsId), + (wsId, tss) => data.SetMultiStringAlt(hvo, flid, wsId, tss), + servLoc.WritingSystemFactory, + configuredTags, + tag => logger.Warning("Custom field {0}: skipping unidentified writing system {1}", + fieldName, tag), + (notWritten, written) => logger.Warning( + "Custom field {0}: keys \"{1}\" and \"{2}\" name the same writing system; wrote \"{2}\" ({3}), not \"{1}\" ({4})", + fieldName, notWritten, written, valueAsMultiText.Excerpt(written), + valueAsMultiText.Excerpt(notWritten))); + } + case CellarPropertyType.String: { var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument); int wsIdForField = lcmMetaData.GetFieldWs(flid); - string wsStrForField = servLoc.WritingSystemFactory.GetStrFromWs(wsIdForField); - KeyValuePair kv = valueAsMultiText.BestStringAndWs(new string[] { wsStrForField }); - string foundWs = kv.Key ?? string.Empty; + KeyValuePair kv = valueAsMultiText.BestStringAndWsId( + new[] { wsIdForField }, servLoc.WritingSystemFactory, configuredTags); + int foundWsId = kv.Key; string foundData = kv.Value ?? string.Empty; - int foundWsId = servLoc.WritingSystemFactory.GetWsFromStr(foundWs); if (foundWsId == 0) - return false; // Skip any unidentified writing systems + { + // Nothing placeable, so the field is left as it was; if LF holds text, say why + if (!valueAsMultiText.IsEmpty) + ConvertMongoToLcmLexicon.LogTextNotPlaced(valueAsMultiText, "Custom field " + fieldName, + logger, servLoc.WritingSystemFactory, configuredTags); + return false; + } ITsString oldValue = data.get_StringProp(hvo, flid); ITsString newValue = ConvertMongoToLcmTsStrings.SpanStrToTsString(foundData, foundWsId, servLoc.WritingSystemFactory); if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) // GetDiffsInTsStrings() returns null when there are no changes @@ -379,8 +424,12 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui } default: + if (unwritableFieldsReported.Add(flid)) + logger.Warning( + "Custom field {0} not written to LCM: CellarPropertyType.{1} is not implemented " + + "for the LF to LCM direction, so any data LF holds in it is being dropped", + fieldName, fieldType.ToString()); return false; - // TODO: Maybe issue a proper warning (or error) log message for "field type not recognized"? } } @@ -421,4 +470,4 @@ public void SetCustomFieldsForThisCmObject(ICmObject cmObj, string objectType, B } } } -} \ No newline at end of file +} diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 70d172fb..c86a1065 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -6,6 +6,7 @@ using System.Text.RegularExpressions; using LfMerge.Core.DataConverters.CanonicalSources; using LfMerge.Core.FieldWorks; +using LfMerge.Core.LanguageForge.Config; using LfMerge.Core.LanguageForge.Model; using LfMergeBridge.LfMergeModel; using LfMerge.Core.Logging; @@ -39,9 +40,13 @@ public class ConvertMongoToLcmLexicon private int _wsEn; private ConvertMongoToLcmCustomField _convertCustomField; + // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm + private ISet _configuredTags; - // Entries where LF's copy differs from FieldWorks but no LF user has touched it since the - // last sync, so FieldWorks holds the newer version (see EditedInLfSinceLastSync) + // Entries where LF's copy differs from FieldWorks, or FieldWorks lacks it, but no LF user has + // touched it since the last sync, so FieldWorks holds the newer version (see + // EditedInLfSinceLastSync). An entry FieldWorks lacks in a project that has never synced is + // not counted: it is LF's own, and is sent (see LfLexEntryToLcmLexEntry). private int _changedOnlyInFieldWorks; private int _absentOnlyFromFieldWorks; @@ -138,14 +143,18 @@ public ConversionError RunConversion() // Logger.Debug("Running \"fake\" MtFComments, should see comments show up below:"); var entryObjectIdToGuidMappings = Connection.GetGuidsByObjectIdForCollection(LfProject, MagicStrings.LfCollectionNameForLexicon); EntryCounts.Reset(); - // Update writing systems from project config input systems. Won't commit till the end + // Update writing systems from project config input systems. Won't commit till the end. + // Which list a new writing system belongs in depends on what it is actually used for, so + // the lexicon is counted -- a second streaming pass over Mongo, which GetLexicon yields + // from a cursor -- but only if some input system is new to LCM. UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", Cache.ActionHandlerAccessor, () => - LfWsToLcmWs(ProjectRecord.InputSystems)); + LfWsToLcmWs(ProjectRecord.InputSystems, () => LfWritingSystemUsage.FromLexicon(GetLexicon(LfProject)))); // Set English ws handle again in case it changed _wsEn = ServiceLocator.WritingSystemFactory.GetWsFromStr("en"); - _convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn); + _configuredTags = ConfiguredTags(ProjectRecord.Config); + _convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn, _configuredTags); IEnumerable lexicon = GetLexicon(LfProject); _changedOnlyInFieldWorks = 0; @@ -201,45 +210,446 @@ private IEnumerable GetLexicon(ILfProject project) return Connection.GetRecords(project, MagicStrings.LfCollectionNameForLexicon); } + // The field groups whose vernacular/analysis role is never in doubt, because FieldWorks fixes + // it: a headword and an example sentence are vernacular, a gloss and a translation are not. + // Everything else is classified per project by which of these its writing systems appear in + // -- a fixed list cannot be right for every project: in the 2026-07-06 corpus 741 projects + // configure etymology with analysis writing systems and 378 with vernacular ones. + // + // Counting that corpus, for each field, the projects whose text there is exclusively + // vernacular against exclusively analysis: the example sentence is 332 to 5, the example + // translation 1 to 320, the entry note 0 to 255 and the literal meaning 4 to 136. + // + // The pronunciation field is deliberately NOT here. It looks vernacular at 114 to 16, but + // that is 12% analysis, above the MinorityShare a single writing system would have to stay + // under to count as one role alone, so projects evidently use it for more than one thing. + // Its text still counts for something in ClassifyByText, though: see there. + private static readonly string[] VernacularAnchorFields = { + LfWritingSystemUsage.Lexeme, + LfWritingSystemUsage.CitationForm, + LfWritingSystemUsage.ExampleSentence, + }; + private static readonly string[] AnalysisAnchorFields = { + LfWritingSystemUsage.Definition, + LfWritingSystemUsage.Gloss, + LfWritingSystemUsage.ExampleTranslation, + LfWritingSystemUsage.Note, + LfWritingSystemUsage.LiteralMeaning, + }; + + /// + /// Work out which writing systems this project treats as vernacular and which as analysis, + /// from its own config and the text in its lexicon. + /// + /// The anchor fields have a role FieldWorks fixes: the lexeme, citation form and example + /// sentence are vernacular; the definition, gloss, example translation, note and literal + /// meaning are analysis (see VernacularAnchorFields). + /// + /// Where the lexicon has text in the anchors for a writing system, that text decides: one + /// holding nearly all of it in the vernacular anchors is vernacular whatever the config + /// offers, and vice versa, and one whose text is split is both. See + /// . A writing system whose only text is pronunciation text is + /// vernacular; see . + /// + /// The rest go by the config. A writing system the anchors offer is placed by them, and one + /// in both anchor groups -- "en" very often is -- is both. Every other field carrying input + /// systems (etymology, custom fields) is assigned by overlap: a writing system exclusive to + /// the vernacular role is evidence the field is vernacular, and vice versa. Evidence decides + /// the role of the field, not of every writing system in it, and it has to point one way: a + /// field offering writing systems exclusive to each role, such as an etymology configured + /// as [seh, pt], is evidence of neither, so it does not make Portuguese vernacular, nor + /// anything else it offers that the anchors have not placed. All the fields' evidence is + /// weighed before any writing system is placed by it, so the order of the config's fields + /// does not matter. What is left is settled, where it can be, by the company a writing + /// system keeps and then by its language. + /// + /// This replaces comparing each tag against ProjectRecord.LanguageCode, which named exactly one + /// vernacular writing system and got it wrong whenever languageCode disagreed with the lexeme + /// field: flh-flex has languageCode "flh-x-ortho" (used only by etymology) while its lexeme + /// uses "flh-x-cm", and odo has languageCode "th" which is not among its writing systems at all, + /// leaving its 24,460 "en" headwords classified as analysis. + /// + /// + /// How much text each writing system holds in each field, or null to go on the config alone. + /// + /// + /// The vernacular tags; the analysis tags; and the tags that could not be resolved. The first + /// two sets overlap wherever a writing system plays both roles. A tag in neither set -- one + /// no config field uses -- is analysis, as it always has been. + /// + /// Unresolved writing systems are treated as BOTH vernacular and analysis, since nothing says + /// which they are and a writing system missing from the list a field draws on leaves that + /// field's data with nowhere to go. The FieldWorks fields such a writing system most often + /// feeds (etymology form, example sentence) are vernacular-typed, which is why being only + /// analysis will not do; but a custom field can be analysis-typed, as a French-only notes + /// field would be, which is why being only vernacular will not do either. 7 projects in the + /// 2026-10-01 corpus need this, e.g. kam-flex, whose Spanish and French are offered only in + /// a wordlist field that offers its vernacular and its analysis language as well. + /// + public static (ISet Vernacular, ISet Analysis, ISet Unresolved) + ClassifyVernacularWritingSystems(LfProjectConfig config, string languageCode, + LfWritingSystemUsage usage = null) + { + var byPath = new Dictionary>(); + CollectInputSystems((config == null) ? null : config.Entry, "", byPath); + + var vernacular = new HashSet(StringComparer.OrdinalIgnoreCase); + var analysis = new HashSet(StringComparer.OrdinalIgnoreCase); + var unresolved = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Writing systems whose text has already answered for them. They keep that answer: the + // config below may offer them for fields nobody writes in, and must not overrule it. + var decidedByText = ClassifyByText(usage, byPath, vernacular, analysis); + + var vernacularAnchor = Union(byPath, VernacularAnchorFields); + var analysisAnchor = Union(byPath, AnalysisAnchorFields); + vernacular.UnionWith(vernacularAnchor.Where(tag => !decidedByText.Contains(tag))); + analysis.UnionWith(analysisAnchor.Where(tag => !decidedByText.Contains(tag))); + + // Only a writing system exclusive to one role is evidence of a field's role. + var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis); + + // Every field's evidence is gathered before any writing system is placed by it, so that + // the answer does not depend on the order the config lists its fields in. Placing as + // each field came used to let a field with no evidence, listed before the one that + // placed a writing system, put that writing system in doubt, and the passes below could + // then strip one of the roles that field gave it. + var anchors = new HashSet(VernacularAnchorFields.Concat(AnalysisAnchorFields)); + var inVernacularField = new HashSet(StringComparer.OrdinalIgnoreCase); + var inAnalysisField = new HashSet(StringComparer.OrdinalIgnoreCase); + var inFieldOfNoRole = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var field in byPath) + { + if (anchors.Contains(field.Key) || field.Value.Count == 0) + continue; + // A writing system its own text has spoken for is still evidence of what this field + // is for, but nothing here may change the answer it was given. + var undecided = field.Value.Where(tag => !decidedByText.Contains(tag)).ToList(); + bool withVernacular = field.Value.Any(vernacularOnly.Contains); + bool withAnalysis = field.Value.Any(analysisOnly.Contains); + if (withVernacular && !withAnalysis) + inVernacularField.UnionWith(undecided); + else if (withAnalysis && !withVernacular) + inAnalysisField.UnionWith(undecided); + else + { + // Overlaps neither role, or both: nothing says which role this field plays. A field + // offering a vernacular-only and an analysis-only writing system side by side is + // no more evidence for one role than the other, just as only unanimous company + // counts in ResolveByCompanions. + inFieldOfNoRole.UnionWith(undecided); + } + } + // A field of no role puts in doubt only the writing systems nothing else places, neither + // the anchors nor a field that has a role. One already placed keeps its place: "en" in + // both the lexeme and the gloss is not made doubtful by also turning up in a sense's + // general note, and if it were, the passes below could strip one of its roles on the + // strength of the company it keeps there. (The entry's note is an anchor, so never comes + // this far.) + var doubtful = inFieldOfNoRole + .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag) + && !inVernacularField.Contains(tag) && !inAnalysisField.Contains(tag)) + .ToList(); + vernacular.UnionWith(inVernacularField); + analysis.UnionWith(inAnalysisField); + vernacular.UnionWith(doubtful); + analysis.UnionWith(doubtful); + unresolved.UnionWith(doubtful); + + ResolveByCompanions(byPath, vernacular, analysis, unresolved); + ResolveByLanguageAffinity(vernacular, analysis, unresolved); + + // Safety net for a config LfMerge cannot read: without this such a project would get no + // vernacular writing system at all, which is worse than the old rule. + if (vernacular.Count == 0 && !string.IsNullOrEmpty(languageCode)) + vernacular.Add(languageCode); + + return (vernacular, analysis, unresolved); + } + + /// + /// The writing systems that play one role only: those vernacular and not analysis, and those + /// analysis and not vernacular. Each pass that weighs evidence recomputes them, since the + /// passes before it change the two sets. + /// + private static (HashSet VernacularOnly, HashSet AnalysisOnly) ExclusiveRoles( + ISet vernacular, ISet analysis) + { + var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase); + vernacularOnly.ExceptWith(analysis); + var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase); + analysisOnly.ExceptWith(vernacular); + return (vernacularOnly, analysisOnly); + } + + /// + /// Settles what is left by the company a writing system keeps. What is left is offered only + /// in fields that said nothing of their role -- a field with a writing system of one role + /// alone has already placed the rest -- but the company it keeps there may since have been + /// placed by another field: in a custom field offering "abc" and "xyz" and nothing else of + /// known role, "xyz" is vernacular if another custom field, offering "abc" beside the + /// vernacular, has made "abc" vernacular. + /// + /// Only unanimous company counts. A field holding writing systems of both roles says + /// nothing, which is the same reason a writing system in both anchors is no evidence of a + /// field's role. One pass is made, not one until nothing changes: a writing system whose + /// only company is another left in doubt stays in doubt, and keeps both roles, even if that + /// other is settled here. + /// + private static void ResolveByCompanions(IDictionary> byPath, + ISet vernacular, ISet analysis, ISet unresolved) + { + if (unresolved.Count == 0) + return; + + // Unresolved writing systems sit in both sets, so they are in neither of these and + // cannot vote for each other. + var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis); + + foreach (string tag in unresolved.ToList()) + { + var companions = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var field in byPath) + { + if (field.Value.Contains(tag)) + companions.UnionWith(field.Value); + } + companions.Remove(tag); + bool withVernacular = companions.Any(vernacularOnly.Contains); + bool withAnalysis = companions.Any(analysisOnly.Contains); + if (withVernacular == withAnalysis) + continue; // No company, or company of both roles: still nothing to go on. + if (withVernacular) + analysis.Remove(tag); + else + vernacular.Remove(tag); + unresolved.Remove(tag); + } + } + + /// + /// Settles what is left by language. A writing system plays the same role as the others + /// that spell the same language: a phonetic or audio spelling of the vernacular is + /// vernacular too, and a regional spelling of the analysis language is analysis. + /// + private static void ResolveByLanguageAffinity(ISet vernacular, ISet analysis, + ISet unresolved) + { + if (unresolved.Count == 0) + return; + + // Unresolved writing systems sit in both sets, so they are in neither of these and + // cannot vote for each other. + var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis); + var vernacularLanguages = new HashSet(vernacularOnly.Select(LanguageOf), StringComparer.OrdinalIgnoreCase); + var analysisLanguages = new HashSet(analysisOnly.Select(LanguageOf), StringComparer.OrdinalIgnoreCase); + + foreach (string tag in unresolved.ToList()) + { + string language = LanguageOf(tag); + bool withVernacular = vernacularLanguages.Contains(language); + bool withAnalysis = analysisLanguages.Contains(language); + if (withVernacular == withAnalysis) + continue; // The language is spoken for by both roles, or by neither. + if (withVernacular) + analysis.Remove(tag); + else + vernacular.Remove(tag); + unresolved.Remove(tag); + } + } + + // Subtags that mark a variant of a language rather than a language of its own. FieldWorks + // also appends "dupl1", "dupl2" and so on when a project needs a second writing system for + // the same thing. + private static readonly HashSet VariantMarkers = new HashSet( + new[] { "fonipa", "etic", "emic", "audio" }, StringComparer.OrdinalIgnoreCase); + + /// + /// The language a tag names, as far as telling one project's writing systems apart goes: + /// the language subtag plus any private-use subtags that name the language rather than a + /// variant of it. Script and region are left out, so "en" and "en-GB" are one language. + /// + /// The private-use subtags have to stay, because under "qaa" -- the code for a language + /// with no code of its own, which Language Forge projects lean on heavily -- they ARE the + /// language: "qaa-x-kal" and "qaa-x-hbo" are no more the same language than "fr" and "de". + /// So "qaa-x-kal", "qaa-fonipa-x-kal" and "qaa-Zxxx-x-kal-audio" all come out "qaa-kal", + /// while "seh" and "seh-fonipa-x-etic" both come out "seh". The same goes for a tag that is + /// private-use from the start: "x-kal" comes out "x-kal", not "x". + /// + private static string LanguageOf(string tag) + { + if (string.IsNullOrEmpty(tag)) + return tag; + string[] parts = tag.Split('-'); + var language = new List { parts[0] }; + int privateUse = Array.FindIndex(parts, + part => part.Equals("x", StringComparison.OrdinalIgnoreCase)); + if (privateUse >= 0) + { + for (int i = privateUse + 1; i < parts.Length; i++) + { + if (VariantMarkers.Contains(parts[i]) || + parts[i].StartsWith("dupl", StringComparison.OrdinalIgnoreCase)) + continue; + language.Add(parts[i]); + } + } + return string.Join("-", language); + } + + /// + /// A writing system holding at most this share of its text in the fields of one role is + /// taken to belong to the other role alone; in between, it plays both. + /// + /// A few strays are not evidence of a second role. In brb-flex-2022 the phonetic writing + /// system has 4,030 headwords and one stray string in an example reference, and FieldWorks + /// has it as vernacular only; in cmo-khm-flex the vernacular has 7,815 vernacular strings + /// against 300 in glosses, and is likewise vernacular only. The one writing system the four + /// FieldWorks projects on hand really do list as BOTH sits at 28%, so anything between 4% + /// and 28% separates the two, and 10% is the round number in the middle. + /// + public const double MinorityShare = 0.10; + + /// + /// Classifies every writing system the lexicon has text for, by where that text sits. + /// Adds them to and , and returns + /// the ones it answered for. + /// + /// A writing system whose only text is in the pronunciation field is made vernacular, + /// because FieldWorks draws its pronunciation writing systems from the vernacular ones + /// (InitializePronunciationWritingSystems considers no others). It is not returned as + /// answered for, so the anchor fields can still add analysis: Swedish offered for the + /// gloss, with a stray pronunciation string, is both. But from here on it counts as + /// vernacular, as any placed writing system does: a field of no role does not put it in + /// doubt, and a custom field it alone is offered in is taken for a vernacular field. So + /// a writing system offered only in custom fields, with pronunciation text and nothing + /// else, is vernacular only. In the 2026-10-01 corpus all 57 writing systems with + /// pronunciation text and no anchor text are used that way -- phonetic, IPA and audio + /// writing systems, and xin-flex's "xin", which holds 226 pronunciations and nothing else + /// -- and FLEx has as vernacular every one of them whose role LfMerge's export records. + /// The pronunciation field is still not an anchor; see VernacularAnchorFields. + /// + private static ISet ClassifyByText(LfWritingSystemUsage usage, + IDictionary> byPath, ISet vernacular, ISet analysis) + { + var decided = new HashSet(StringComparer.OrdinalIgnoreCase); + if (usage == null || usage.EntriesCounted == 0) + return decided; + + // Every writing system the config mentions, plus any the config forgot but the lexicon + // uses anyway -- those need classifying just as much. + var candidates = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (ISet tags in byPath.Values) + candidates.UnionWith(tags); + foreach (string path in VernacularAnchorFields.Concat(AnalysisAnchorFields)) + candidates.UnionWith(usage.TagsWithText(path)); + candidates.UnionWith(usage.TagsWithText(LfWritingSystemUsage.Pronunciation)); + + foreach (string tag in candidates) + { + int vernacularText = usage.NonEmptyCount(VernacularAnchorFields, tag); + int analysisText = usage.NonEmptyCount(AnalysisAnchorFields, tag); + int total = vernacularText + analysisText; + if (total == 0) + { + // No text in a field of known role, so its text decides nothing. + if (usage.NonEmptyCount(LfWritingSystemUsage.Pronunciation, tag) > 0) + vernacular.Add(tag); + continue; + } + double analysisShare = (double)analysisText / total; + if (analysisShare <= MinorityShare) + vernacular.Add(tag); + else if (analysisShare >= 1.0 - MinorityShare) + analysis.Add(tag); + else + { + vernacular.Add(tag); + analysis.Add(tag); + } + decided.Add(tag); + } + return decided; + } + + private static ISet Union(IDictionary> byPath, IEnumerable paths) + { + var result = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var path in paths) + { + ISet tags; + if (byPath.TryGetValue(path, out tags)) + result.UnionWith(tags); + } + return result; + } + + /// + /// Every writing-system spelling the config's fields use, exactly as spelled: LF's editor + /// matches them against an entry's keys exactly. + /// + private static ISet ConfiguredTags(LfProjectConfig config) + { + var byPath = new Dictionary>(); + CollectInputSystems((config == null) ? null : config.Entry, "", byPath); + var tags = new HashSet(StringComparer.Ordinal); + foreach (ISet fieldTags in byPath.Values) + tags.UnionWith(fieldTags); + return tags; + } + + /// + /// Record every config field's input systems, keyed by its dotted path as Mongo spells it, so + /// nesting goes through "fields" ("senses.fields.examples.fields.sentence"). + /// + private static void CollectInputSystems(LfConfigFieldList fieldList, string path, + IDictionary> byPath) + { + if (fieldList == null || fieldList.Fields == null) + return; + foreach (var child in fieldList.Fields) + { + string childPath = string.IsNullOrEmpty(path) ? child.Key : path + "." + child.Key; + var multiText = child.Value as LfConfigMultiText; + // A null or empty tag names no writing system, and a null one would make the text + // counts throw, ending the whole transfer + if (multiText != null && multiText.InputSystems != null) + byPath[childPath] = new HashSet(multiText.InputSystems.Where(tag => !string.IsNullOrEmpty(tag)), + StringComparer.OrdinalIgnoreCase); + var nested = child.Value as LfConfigFieldList; + if (nested != null) + CollectInputSystems(nested, childPath + ".fields", byPath); + } + } + /// /// Converts the list of LF input systems and adds them to Lcm writing systems /// /// List of LF input systems. - private void LfWsToLcmWs(Dictionary lfWsList) - { - // Between FW 8.2 and 9, a few classes and interfaces were renamed. The ones most relevant here are - // IWritingSystemManager (interface was removed and replaced with the WritingSystemManager concrete class), - // and PalasoWritingSystem which was replaced with CoreWritingSystemDefinition. Since their internals - // didn't change much (and the only changes were in areas we don't access), we can use a simple compiler - // define to choose the type of the wsm variable here (and the ws variable later) and we're fine. - // HOWEVER, if the code inside #if...#endif blocks starts to grow, this is not an ideal solution. A better - // solution if the code grows complex will be to write several classes to the same interface, each of which - // can deal with one particular version of FW or Lcm. Register them all with Autofac with a way to choose among them - // (http://docs.autofac.org/en/stable/register/registration.html#selection-of-an-implementation-by-parameter-value) - // and then, at runtime, we can instantiate the particular class that's needed for dealing with *this* FW project. - // - // But for now, these #if...#endif blocks are enough. - 2016-03 RM -#if FW8_COMPAT - // Note that we can't use ILgWritingSystemFactory here, because it doesn't have some methods we need later on. - IWritingSystemManager wsManager = ServiceLocator.WritingSystemManager; -#else + /// + /// Counts the text in the lexicon, for classifying a writing system new to LCM. Called only + /// if there is one, since it reads the whole lexicon. + /// + private void LfWsToLcmWs(Dictionary lfWsList, + Func countUsage = null) + { WritingSystemManager wsManager = ServiceLocator.WritingSystemManager; -#endif if (wsManager == null) { Logger.Error("Failed to find the writing system manager"); return; } - string vernacularLanguageCode = ProjectRecord.LanguageCode; + // Worked out when the first writing system new to LCM turns up, and not at all if none + // does, which after a project's first sync is nearly always: only a new writing system is + // put in a list, and working out which ones counts the whole lexicon. It depends only on + // LF's config and text, so the writing systems created before then cannot change it. + (ISet Vernacular, ISet Analysis, ISet Unresolved)? classification = null; // TODO: Split the inside of this foreach() out into its own function foreach (var lfWs in lfWsList.Values) { -#if FW8_COMPAT - IWritingSystem ws; -#else CoreWritingSystemDefinition ws; -#endif // It would be nice to call this to add to both analysis and vernacular WS. // But we need the flexibility of bringing in LF WS properties. @@ -253,46 +663,112 @@ private void LfWsToLcmWs(Dictionary lfWsList) } */ - // TODO: It might be possible to rewrite this code to NOT rely on TryGet() after all, in which case we could - // use the ILgWritingSystemFactory interface and remove one point of FW 8-to-9 API incompatibility. - - if (wsManager.TryGet(lfWs.Tag, out ws)) + // Find the writing system the way every other LF tag is found before creating one. + // LfMerge used to export input systems by LanguageTag, while LCM knows a writing + // system by its Id, and in older projects the two differ: spt-flex's input system "hi-IN" is + // the writing system LCM holds as "hi-Deva-IN". GetOrSet looks a tag up only as an + // Id, so it did not find it, created a duplicate "hi-IN" and put it in the current + // lists; WsIdFromLfTag, which every multitext key goes through, then matched the + // duplicate exactly, and text typed in LF went to it rather than to hi-Deva-IN. + int existingHandle = LanguageTags.WsIdFromLfTag(wsManager, lfWs.Tag); + bool wsAlreadyExisted; + if (existingHandle != 0) { - // The WS does check that a property has a different value before setting it - // (and thus setting IsChanged flag), but for Abbreviation the WS returns - // Language if not set, and it fails to check that. - if (ws.Abbreviation != lfWs.Abbreviation) - ws.Abbreviation = lfWs.Abbreviation; - ws.RightToLeftScript = lfWs.IsRightToLeft; - wsManager.Replace(ws); + ws = wsManager.Get(existingHandle); + wsAlreadyExisted = true; } else { - ws = wsManager.Create(lfWs.Tag); - ws.Abbreviation = lfWs.Abbreviation; - ws.RightToLeftScript = lfWs.IsRightToLeft; - wsManager.Set(ws); - - // LF doesn't distinguish between vernacular/analysis WS, so we'll - // only assign the project language code to vernacular. - // All other WS assigned to analysis. + // Nothing to find, so create it. GetOrSet creates under the canonical form of the + // tag -- "qaa-x-qaa-v" becomes "qaa-x-v", "th-Thai" becomes "th" -- and adds the + // writing system to the manager, so no Set() call is needed here. + wsAlreadyExisted = wsManager.GetOrSet(lfWs.Tag, out ws); + } - // TODO: What if our vernacular was Thai, but we added th-ipa? This logic needs to be a bit "fuzzier", really. - if (lfWs.Tag.Equals(vernacularLanguageCode)) + // The WS does check that a property has a different value before setting it + // (and thus setting IsChanged flag), but for Abbreviation the WS returns + // Language if not set, and it fails to check that. + if (ws.Abbreviation != lfWs.Abbreviation) + { + ws.Abbreviation = lfWs.Abbreviation; + } + ws.RightToLeftScript = lfWs.IsRightToLeft; + // No wsManager.Replace(ws): ws is the manager's own object, so the edits above are + // already in place. Replace looks the writing system up by LanguageTag, and where + // the Id is spelled otherwise that lookup does harm. If it finds nothing, Replace + // re-registers ws under a fresh handle mid-sync; if it finds another writing system + // with that LanguageTag -- brb-flex-2022's unused "km-KH" beside "km-Khmr-KH" -- + // Replace evicts that one and gives ws its handle. + + if (!wsAlreadyExisted) + { + // LF doesn't distinguish between vernacular/analysis WS, so the roles are worked + // out per project from its config and text -- see ClassifyVernacularWritingSystems. A + // writing system can play both, "en" in lexeme and gloss alike being the usual + // case, and one no field claims is analysis. The input system, the config and + // the lexicon need not spell the writing system alike, so the roles are looked + // up under any spelling of it; see HasRole. + if (classification == null) + classification = ClassifyForNewWritingSystems(countUsage); + bool isVernacular = HasRole(classification.Value.Vernacular, lfWs.Tag); + if (isVernacular) ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws); - else + if (!isVernacular || HasRole(classification.Value.Analysis, lfWs.Tag)) ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws); - } } } + /// + /// Whether a writing system new to LCM plays this role, under any spelling of its tag. The + /// classification spells each writing system as the config and the lexicon do, and the + /// input system can spell it otherwise: an LF user who adds "qaa-x-qaa-new" may configure + /// fields with "qaa-x-new". Spellings sharing a canonical form can be taken for one writing + /// system here, because this one is new: had LCM held any writing system with that canonical + /// form, WsIdFromLfTag would have found it. A role found under one spelling therefore counts + /// for all of them, even where the text decided otherwise under another: a writing system + /// whose text makes it vernacular only, offered by the config in the gloss under another + /// spelling, is both. That can only add a role, never take one away. + /// + private static bool HasRole(ISet role, string tag) + { + if (role.Contains(tag)) + return true; + string canonical = LanguageTags.Canonical(tag); + return role.Any(other => string.Equals(LanguageTags.Canonical(other), canonical, StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Which writing systems this project treats as vernacular and which as analysis, derived + /// from its own config and text rather than from languageCode alone; see + /// ClassifyVernacularWritingSystems. + /// + private (ISet Vernacular, ISet Analysis, ISet Unresolved) + ClassifyForNewWritingSystems(Func countUsage) + { + var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config, + ProjectRecord.LanguageCode, countUsage?.Invoke()); + if (classification.Unresolved.Count > 0) + { + Logger.Notice("MongoToLcm: writing system(s) {0} appear only in config fields whose " + + "vernacular/analysis role could not be determined; treating them as both", + string.Join(", ", classification.Unresolved)); + } + return classification; + } + + /// + /// The best string for a single-string LCM field, and the writing system it is in. Null when + /// LF holds no text in any writing system LCM has -- which is not the same as holding no text + /// at all; see . + /// private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool isAnalysisField = true) { if (input == null) return null; if (input.Count == 0) { - Logger.Warning("BestStringAndWsFromMultiText got a non-null multitext, but it was empty. Empty LF MultiText objects should be nulls in Mongo. Unfortunately, at this point in the code it's hard to know which multitext it was."); + // Some Language Forge fields, like scientificName on senses, have empty but + // non-null multitexts; those should also be empty in LCM. return null; } @@ -303,38 +779,79 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool // _analysisWsIdsAndNamesInSearchOrder : // _vernacularWsIdsAndNamesInSearchOrder; - foreach (ILgWritingSystem ws in wsesToSearch) - { - LfStringField field; - if (input.TryGetValue(ws.Id, out field) && field != null && !field.IsEmpty) - { -// Logger.Debug("Returning TsString from {0} for writing system {1}", field.Value, ws.Id); - return new Tuple(field.Value, ws.Handle); - } - } - - // Last-ditch option: just grab the first non-empty string we can find - KeyValuePair kv = input.WsIdAndFirstNonEmptyString(Cache); - if (kv.Value == null) return null; -// Logger.Debug("Returning first non-empty TsString from {0} for writing system with ID {1}", -// kv.Value, kv.Key); - return new Tuple(kv.Value, kv.Key); + // By handle, not by comparing LF's keys with ws.Id, which would miss a key LF spells + // differently from LCM. Falls back to the first value in any writing system LCM knows. + KeyValuePair best = input.BestStringAndWsId( + wsesToSearch.Select(ws => ws.Handle), ServiceLocator.WritingSystemFactory, _configuredTags); + if (best.Value == null) + return null; + return new Tuple(best.Value, best.Key); } - private ITsString BestTsStringFromMultiText(LfMultiText input, bool isAnalysisField = true) + /// + /// The value to give a single-string LCM field, whose current value is + /// . + /// + /// Null, clearing the field, when LF holds no text for it. But when LF holds text and every + /// alternative is in a writing system LCM does not have, there is nowhere to put it, and the + /// field keeps its current value. Neither alternative will do: building the string in + /// writing system 0 throws, which abandons the rest of the entry half-written, and clearing + /// the field would delete what LCM holds on account of text that could not be placed. + /// + /// The field also keeps its value when LF's text is all under keys passed over for an empty + /// key naming the same writing system. That empty key may be a clear made under the config's + /// spelling, or the empty value LF's editor fills in for every configured input system an + /// entry lacks, beside the export's value under the Id; nothing here tells the two apart, and + /// keeping what LCM holds loses nothing that LF can show. + /// + private ITsString BestTsStringFromMultiText(LfMultiText input, ITsString current, bool isAnalysisField = true) { Tuple stringAndWsId = BestStringAndWsFromMultiText(input, isAnalysisField); - if (stringAndWsId == null) - return null; - return ConvertMongoToLcmTsStrings.SpanStrToTsString(stringAndWsId.Item1, stringAndWsId.Item2, ServiceLocator.WritingSystemFactory); + if (stringAndWsId != null) + return ConvertMongoToLcmTsStrings.SpanStrToTsString(stringAndWsId.Item1, stringAndWsId.Item2, ServiceLocator.WritingSystemFactory); + return NothingToPlace(input) ? null : current; } - private string BestStringFromMultiText(LfMultiText input, bool isAnalysisField = true) + /// As , for a plain string field. + private string BestStringFromMultiText(LfMultiText input, string current, bool isAnalysisField = true) { Tuple stringAndWsId = BestStringAndWsFromMultiText(input, isAnalysisField); - if (stringAndWsId == null) - return null; - return stringAndWsId.Item1; + if (stringAndWsId != null) + return stringAndWsId.Item1; + return NothingToPlace(input) ? null : current; + } + + /// + /// Whether LF holds no text at all here. When it does hold text that nevertheless could not + /// be placed, says why in the log, since that text is not reaching FieldWorks. + /// + private bool NothingToPlace(LfMultiText input) + { + if (input == null || input.IsEmpty) + return true; + LogTextNotPlaced(input, "MongoToLcm", Logger, ServiceLocator.WritingSystemFactory, _configuredTags); + return false; + } + + /// + /// Logs why each key holding text in a single-string field's multitext was not written + /// there. Only for when nothing was: every key with text then either names no writing system + /// LCM has or was passed over for an empty key, and the messages say so. Called after a + /// value has been placed, it would call the key written empty. + /// + internal static void LogTextNotPlaced(LfMultiText input, string context, ILogger logger, + ILgWritingSystemFactory wsManager, ISet configuredTags) + { + foreach (KeyValuePair passedOver in input.KeysPassedOver(wsManager, configuredTags)) + { + if (passedOver.Value == null) + logger.Warning("{0}: skipping text under \"{1}\" ({2}), which names no writing system LCM has; " + + "leaving the field as it was", context, passedOver.Key, input.Excerpt(passedOver.Key)); + else + logger.Warning("{0}: keys \"{1}\" and \"{2}\" name the same writing system, and \"{2}\", the one " + + "to write, is empty; leaving the field as it was rather than write \"{1}\" ({3})", + context, passedOver.Key, passedOver.Value, input.Excerpt(passedOver.Key)); + } } // This GetOrCreate() function takes an extra out parameter so we can correctly update @@ -543,6 +1060,9 @@ private DateTime? LastSyncedDate /// LfMerge's: the initial clone's transfer to Mongo does not always get as far as recording a /// sync. LfMerge clears the user references on every entry it writes, and LF sets one on every /// edit and creation; a deletion sets none, but deleting what FieldWorks lacks is harmless. + /// A LIFT import into LF sets none either, so this says "not edited" of every imported entry + /// in a never-synced LF project; LfLexEntryToLcmLexEntry sends such an entry anyway when + /// FieldWorks does not have it. /// private bool EditedInLfSinceLastSync(LfLexEntry lfEntry) { @@ -564,12 +1084,25 @@ private void LfLexEntryToLcmLexEntry(LfLexEntry lfEntry) if (lfEntry.IsDeleted) { // An LF deletion that an earlier sync has already sent + return; } - else if (!GetInstance().TryGetObject(guid, out existing)) + if (GetInstance().TryGetObject(guid, out existing)) + { + if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified) + _changedOnlyInFieldWorks++; + return; + } + if (LastSyncedDate != null) + { _absentOnlyFromFieldWorks++; - else if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified) - _changedOnlyInFieldWorks++; - return; + return; + } + // Never synced, and FieldWorks does not have it: an LF project's own entry, going to + // FieldWorks for the first time. A LIFT import into LF sets no user references either, + // so EditedInLfSinceLastSync cannot tell it from one LfMerge wrote; but FieldWorks + // cannot hold a newer version of an entry it does not have, so there is nothing to + // keep. Leaving it out would also lose it: the transfer back to Mongo marks every + // entry FieldWorks lacks as deleted. } bool createdEntry = false; bool wantCreation = !lfEntry.IsDeleted; @@ -693,7 +1226,7 @@ private void LfExampleToLcmExample(LfExample lfExample, ILexSense owner) // LcmExample.Guid, // LcmExample.Hvo // ); - LcmExample.Reference = BestTsStringFromMultiText(lfExample.Reference); + LcmExample.Reference = BestTsStringFromMultiText(lfExample.Reference, LcmExample.Reference); ICmTranslation t = FindOrCreateTranslationByGuid(lfExample.TranslationGuid, LcmExample, _freeTranslationType); SetMultiStringFrom(t.Translation, lfExample.Translation); @@ -719,9 +1252,17 @@ private void LfPictureToLcmPicture(LfPicture lfPicture, ILexSense owner) string caption = ""; if (lfPicture.Caption != null) { - KeyValuePair kv = lfPicture.Caption.WsIdAndFirstNonEmptyString(Cache); - captionWs = kv.Key; - caption = kv.Value; + // A caption in no writing system LCM has would come back as writing system 0, which + // cannot be built into a string; the picture then starts with an empty caption, and + // SetMultiStringFrom below fills in every alternative that can be placed. + KeyValuePair kv = lfPicture.Caption.BestStringAndWsId( + ServiceLocator.LanguageProject.AnalysisWritingSystems.Select(ws => ws.Handle), + ServiceLocator.WritingSystemFactory, _configuredTags); + if (kv.Value != null) + { + captionWs = kv.Key; + caption = kv.Value; + } } // Lcm expects internal pictures in a certain path. If an external path already @@ -789,19 +1330,19 @@ private void LfSenseToLcmSense(LfSense lfSense, ILexEntry owner) SetMultiStringFrom(LcmSense.PhonologyNote, lfSense.PhonologyNote); // LcmSense.ReversalEntriesRC = lfSense.ReversalEntries; // TODO: More complex than // that. Handle it correctly. Maybe. - LcmSense.ScientificName = BestTsStringFromMultiText(lfSense.ScientificName); + LcmSense.ScientificName = BestTsStringFromMultiText(lfSense.ScientificName, LcmSense.ScientificName); ListConverters[SemDomListCode].UpdatePossibilitiesFromStringArray(LcmSense.SemanticDomainsRC, lfSense.SemanticDomain); SetMultiStringFrom(LcmSense.SemanticsNote, lfSense.SemanticsNote); SetMultiStringFrom(LcmSense.Bibliography, lfSense.SenseBibliography); // lfSense.SenseId; // TODO: What do I do with this one? - LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue); + LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue, LcmSense.ImportResidue); SetMultiStringFrom(LcmSense.Restrictions, lfSense.SenseRestrictions); LcmSense.SenseTypeRA = ListConverters[SenseTypeListCode].FromStringField(lfSense.SenseType); SetMultiStringFrom(LcmSense.SocioLinguisticsNote, lfSense.SociolinguisticsNote); - LcmSense.Source = BestTsStringFromMultiText(lfSense.Source); + LcmSense.Source = BestTsStringFromMultiText(lfSense.Source, LcmSense.Source); LcmSense.StatusRA = ListConverters[StatusListCode].FromStringArrayFieldWithOneCase(lfSense.Status); ListConverters[UsageTypeListCode].UpdatePossibilitiesFromStringArray(LcmSense.UsageTypesRC, lfSense.Usages); @@ -903,7 +1444,15 @@ private void SetMultiStringFrom(IMultiStringAccessor dest, LfMultiText source) if (source == null) ClearMultiString(dest); else - source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager); + source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager, _configuredTags, + // Text under a key that names no writing system LCM has never reaches FieldWorks, + // and leaves LF too once FieldWorks next changes the entry and it is exported again + tag => Logger.Warning( + "MongoToLcm: skipping text under \"{0}\" ({1}), which names no writing system LCM has", + tag, source.Excerpt(tag)), + (notWritten, written) => Logger.Warning( + "MongoToLcm: keys \"{0}\" and \"{1}\" name the same writing system; wrote \"{1}\" ({2}), not \"{0}\" ({3})", + notWritten, written, source.Excerpt(written), source.Excerpt(notWritten))); } /// @@ -952,7 +1501,7 @@ private void SetEtymologyFields(ILexEntry LcmEntry, LfLexEntry lfEntry) SetMultiStringFrom(LcmEtymology.Gloss, lfEntry.EtymologyGloss); if (lfEntry.EtymologySource != null) #if DBVERSION_7000068 - LcmEtymology.Source = BestStringFromMultiText(lfEntry.EtymologySource); + LcmEtymology.Source = BestStringFromMultiText(lfEntry.EtymologySource, LcmEtymology.Source); #else SetMultiStringFrom(LcmEtymology.LanguageNotes, lfEntry.EtymologySource); #endif @@ -994,8 +1543,8 @@ private void SetPronunciation(ILexEntry LcmEntry, LfLexEntry lfEntry) LcmPronunciation = GetInstance().Create(); LcmEntry.PronunciationsOS.Add(LcmPronunciation); } - LcmPronunciation.CVPattern = BestTsStringFromMultiText(lfEntry.CvPattern); - LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone); + LcmPronunciation.CVPattern = BestTsStringFromMultiText(lfEntry.CvPattern, LcmPronunciation.CVPattern); + LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone, LcmPronunciation.Tone); SetMultiStringFrom(LcmPronunciation.Form, lfEntry.Pronunciation); LcmPronunciation.LocationRA = (ICmLocation)ListConverters[LocationListCode].FromStringField(lfEntry.Location); diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs index e33a4f96..1edbc320 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -289,7 +289,7 @@ public void PopulateCmPossibilityFromOptionListItem(ICmPossibility poss, LfOptio // Should only be called when NO canonical item can be found if (wsStr == null) wsStr = "en"; // TODO: Set this from LF user's writing system rather than English, once LF allows setting interface languages - int wsId = poss.Cache.WritingSystemFactory.GetWsFromStr(wsStr); + int wsId = LanguageTags.WsIdFromLfTag(poss.Cache.WritingSystemFactory, wsStr); poss.Abbreviation.set_String(wsId, item.Abbreviation); poss.Name.set_String(wsId, item.Value); } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs index cd1b1b7f..56d8242f 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -89,7 +89,16 @@ public static ITsString SpanStrToTsString(string source, int mainWs, ILgWritingS builder.ClearProps(); // Make sure there aren't leftover properties from previous run // To remove a string property, you set it to null, so we can just use StyleName directly whether or not it's null. builder.SetStrPropValue((int)FwTextPropType.ktptNamedStyle, run.StyleName); - int runWs = (run.Lang == null) ? mainWs : wsf.GetWsFromStr(run.Lang); + int runWs = (run.Lang == null) ? mainWs : LanguageTags.WsIdFromLfTag(wsf, run.Lang); + if (runWs == 0) + { + // A lang naming no writing system LCM has cannot be built into the string, and the + // exception that follows abandons the whole entry half-written. The run's text is + // kept in the string's own writing system; only its language mark is lost. + MainClass.Logger?.Warning("MongoToLcm: span lang \"{0}\" names no writing system LCM has; " + + "keeping \"{1}\" in the field's writing system", run.Lang, run.Content); + runWs = mainWs; + } builder.SetIntPropValues((int)FwTextPropType.ktptWs, (int)FwTextPropVar.ktpvDefault, runWs); // We don't care about Guids in this function, so run.Guid is ignored // But we do need to set any other int or string properties that were in the original diff --git a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs new file mode 100644 index 00000000..62d213a6 --- /dev/null +++ b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs @@ -0,0 +1,142 @@ +// Copyright (c) 2016-2018 SIL International +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) +using System.Collections.Generic; +using LfMerge.Core.LanguageForge.Model; + +namespace LfMerge.Core.DataConverters +{ + /// + /// How much text each writing system actually holds in each Language Forge field, counted over + /// a project's lexicon. + /// + /// The project config says which writing systems a field OFFERS, which is a much weaker claim + /// than it looks: Language Forge routinely offers every input system in every field, so a tag + /// configured for the lexeme field may well have no headword in it anywhere. Counting the text + /// tells us what a writing system is really being used for, which is what + /// ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems needs to tell a vernacular writing + /// system from an analysis one. + /// + /// Fields are keyed by the dotted path the config spells them with, nesting through "fields" + /// ("senses.fields.examples.fields.sentence"), so the counts line up with the paths that + /// classification already works in. + /// + public class LfWritingSystemUsage + { + // Field paths, spelled as the config spells them. + public const string Lexeme = "lexeme"; + public const string CitationForm = "citationForm"; + public const string Definition = "senses.fields.definition"; + public const string Gloss = "senses.fields.gloss"; + public const string ExampleSentence = "senses.fields.examples.fields.sentence"; + public const string ExampleTranslation = "senses.fields.examples.fields.translation"; + public const string Note = "note"; + public const string LiteralMeaning = "literalMeaning"; + public const string Pronunciation = "pronunciation"; + + private readonly Dictionary> _countsByPath = + new Dictionary>(); + + /// How many entries were counted; zero means we have no evidence to go on. + public int EntriesCounted { get; private set; } + + /// The number of non-empty strings this writing system holds in this field. + public int NonEmptyCount(string fieldPath, string tag) + { + Dictionary byTag; + int count; + if (_countsByPath.TryGetValue(fieldPath, out byTag) && byTag.TryGetValue(tag, out count)) + return count; + return 0; + } + + /// + /// The total number of non-empty strings this writing system holds across the given fields. + /// + public int NonEmptyCount(IEnumerable fieldPaths, string tag) + { + int total = 0; + foreach (string path in fieldPaths) + total += NonEmptyCount(path, tag); + return total; + } + + /// The writing systems that hold any text in this field. + public IEnumerable TagsWithText(string fieldPath) + { + Dictionary byTag; + if (!_countsByPath.TryGetValue(fieldPath, out byTag)) + return new string[0]; + return byTag.Keys; + } + + /// + /// Counts the text in the fields classification consults: the vernacular and analysis + /// anchors and the pronunciation field. Counting every other field too would only lengthen + /// a pass over the whole lexicon for numbers nothing reads. Deleted entries are skipped: + /// they are tombstones Language Forge keeps so the deletion can be replayed, and their text + /// is not evidence of anything. + /// + public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon) + { + var usage = new LfWritingSystemUsage(); + if (lexicon == null) + return usage; + foreach (LfLexEntry entry in lexicon) + { + if (entry == null || entry.IsDeleted) + continue; + usage.EntriesCounted++; + usage.Add(Lexeme, entry.Lexeme); + usage.Add(CitationForm, entry.CitationForm); + usage.Add(Note, entry.Note); + usage.Add(LiteralMeaning, entry.LiteralMeaning); + usage.Add(Pronunciation, entry.Pronunciation); + if (entry.Senses == null) + continue; + foreach (LfSense sense in entry.Senses) + { + if (sense == null) + continue; + usage.Add(Definition, sense.Definition); + usage.Add(Gloss, sense.Gloss); + if (sense.Examples == null) + continue; + foreach (LfExample example in sense.Examples) + { + if (example == null) + continue; + usage.Add(ExampleSentence, example.Sentence); + usage.Add(ExampleTranslation, example.Translation); + } + } + } + return usage; + } + + /// + /// Counts one field's non-empty alternatives. Whitespace counts as empty; span markup is + /// not stripped first, since a string made only of markup carries no text to begin with. + /// + private void Add(string fieldPath, LfMultiText text) + { + if (text == null) + return; + foreach (KeyValuePair kv in text) + { + if (kv.Value == null || string.IsNullOrWhiteSpace(kv.Value.Value)) + continue; + Dictionary byTag; + if (!_countsByPath.TryGetValue(fieldPath, out byTag)) + { + // Case-insensitively, as every LCM writing system lookup is: the config and the + // lexicon do not always agree on the case of a tag, and they mean the same one. + byTag = new Dictionary(System.StringComparer.OrdinalIgnoreCase); + _countsByPath[fieldPath] = byTag; + } + int count; + byTag.TryGetValue(kv.Key, out count); + byTag[kv.Key] = count + 1; + } + } + } +} diff --git a/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs b/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs new file mode 100644 index 00000000..21c50e88 --- /dev/null +++ b/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs @@ -0,0 +1,24 @@ +// Copyright (c) 2026 SIL Global +// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) + +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; + +namespace LfMerge.Core.LanguageForge.Model +{ + public class CustomMongoSerializerForInt32 : Int32Serializer + { + public override int Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + IBsonReader bsonReader = context.Reader; + if (bsonReader.GetCurrentBsonType() == BsonType.Null) + { + bsonReader.ReadNull(); + return 0; + } + return base.Deserialize(context, args); + } + } +} \ No newline at end of file diff --git a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs index 28fb02a6..92ea82e1 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016 SIL International +// Copyright (c) 2016 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; @@ -19,6 +19,9 @@ public class LfLexEntry : LfFieldBase, IHasNullableGuid public string MercurialSha { get; set; } public DateTime DateCreated { get; set; } public DateTime DateModified { get; set; } + [BsonDefaultValue(0)] + // CustomMongoSerializerForInt32 converts null to 0 + [BsonSerializer(typeof(CustomMongoSerializerForInt32))] public int DirtySR { get; set ; } // Data properties diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 96dd29dd..02e6b5d9 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 SIL International +// Copyright (c) 2016-2018 SIL International // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Linq; @@ -112,13 +112,40 @@ public string FirstNonEmptyString() return (result == null) ? null : result.Value; } - public KeyValuePair WsIdAndFirstNonEmptyString(LcmCache cache) + /// + /// The value for the first writing system in wsSearchOrder that has a non-empty one, falling + /// back to the first non-empty value in any writing system LCM knows, together with that + /// writing system's handle. Keys are resolved to handles before they are compared, so a key + /// LF spells differently from LCM's id for the writing system still matches it. + /// + /// Where two keys name one writing system, the value is the one WriteToLcm would write: the + /// key spelled as the config spells it, or else the later one. That is chosen before empty + /// values are left out, so a value an LF user has cleared under the config's spelling is + /// not replaced by the export's hidden, stale one. + /// + /// + /// Every writing-system spelling the project's config uses; see WriteToLcm. + /// + /// The handle and the value, or (0, null) when no non-empty value resolves. + public KeyValuePair BestStringAndWsId(IEnumerable wsSearchOrder, ILgWritingSystemFactory wsManager, + ISet configuredTags = null) { - KeyValuePair kv = FirstNonEmptyKeyValue(); - if (kv.Key == null) return new KeyValuePair(); - ILgWritingSystemFactory wsManager = cache.ServiceLocator.WritingSystemManager; - int wsId = wsManager.GetWsFromStr(kv.Key); - return new KeyValuePair(wsId, kv.Value); + var resolved = new List>(); + foreach (KeyValuePair chosen in ChooseKeys(wsManager, configuredTags)) + { + LfStringField field = this[chosen.Value]; + if (field != null && !field.IsEmpty) + resolved.Add(new KeyValuePair(chosen.Key, field.Value)); + } + foreach (int wsId in wsSearchOrder) + { + foreach (KeyValuePair candidate in resolved) + { + if (candidate.Key == wsId) + return candidate; + } + } + return resolved.FirstOrDefault(); } public KeyValuePair FirstNonEmptyKeyValue() @@ -129,24 +156,171 @@ public KeyValuePair FirstNonEmptyKeyValue() new KeyValuePair(result.Key, result.Value.Value); } - public void WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager) + public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager, + ISet configuredTags = null, Action onUnidentifiedTag = null, + Action onKeyNotWritten = null) { if (dest == null) - return; - HashSet destWsIdsToClear = new HashSet(dest.AvailableWritingSystemIds); + return false; + return WriteToLcm(dest.AvailableWritingSystemIds, dest.get_String, dest.set_String, wsManager, + configuredTags, onUnidentifiedTag, onKeyNotWritten); + } + + /// + /// Writes every alternative into an LCM multistring, then clears each alternative LCM holds + /// that has no key here. The built-in multitext fields and the MultiUnicode custom fields + /// both come through here, reaching LCM through an IMultiAccessorBase and through + /// ISilDataAccess respectively, which is why LCM is reached through delegates. + /// + /// Clearing is right because the LCM-to-Mongo direction exports every alternative LCM holds: + /// one missing from LF is one a user removed there, not one LF never saw. + /// + /// Each key is resolved with LanguageTags.WsIdFromLfTag. Resolving it any other way would + /// skip a non-canonical key as unidentified, and the clearing pass would then blank that + /// writing system's text in LCM: the data would not merely be dropped but deleted. + /// + /// An alternative whose value has not changed is left alone, so it stays out of the .fwdata + /// XML and out of the Mercurial commit. + /// + /// Two keys can name the same writing system, spelled two ways that both resolve to it, and + /// only one can be written. That happens when LF's config spells a writing system otherwise + /// than LfMerge's export: the export keys every value by the writing system's Id, an LF user + /// edits under the config's spelling, and the export's value stays in the entry, hidden + /// from the editor, until the entry is next exported -- which, the edit once written to LCM, + /// may be a long time. In spt-flex, whose config says "hi-IN" for the Id "hi-Deva-IN", an + /// edited entry holds both. The value to write is the one under the spelling the config + /// uses, since that is what LF's editor shows and so the one a user can have edited; the + /// other is the hidden export, or, after the config has been re-spelled, an edit made under + /// the old spelling that has already been written. Where both spellings or neither are in + /// the config, it is the later key, as it always was: LF adds a new key after the existing + /// ones. + /// + /// The writing systems LCM currently holds alternatives for. + /// Reads LCM's alternative for a writing system. + /// Writes LCM's alternative for a writing system. + /// Resolves LF's keys to LCM writing systems. + /// + /// Every writing-system spelling the project's config uses, to choose between two keys + /// naming one writing system; null to go on key order alone. + /// + /// Told each key that resolves to no writing system. + /// + /// Told each key left unwritten because another key names the same writing system, and the + /// key written instead. + /// + /// Whether anything in LCM changed. + public bool WriteToLcm(IEnumerable existingWsIds, Func getAlternative, + Action setAlternative, ILgWritingSystemFactory wsManager, + ISet configuredTags = null, Action onUnidentifiedTag = null, + Action onKeyNotWritten = null) + { + // Each writing system's key, chosen before anything is written + List> keyFor = ChooseKeys(wsManager, configuredTags, onUnidentifiedTag, onKeyNotWritten); + + var wsIdsToClear = new HashSet(existingWsIds); + bool changed = false; + foreach (KeyValuePair chosen in keyFor) + { + int wsId = chosen.Key; + LfStringField field = this[chosen.Value]; + wsIdsToClear.Remove(wsId); + string text = (field == null) ? string.Empty : (field.Value ?? string.Empty); + ITsString newValue = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(text, wsId, wsManager); + ITsString oldValue = getAlternative(wsId); + // GetDiffsInTsStrings() returns null when there are no changes + if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) + continue; + setAlternative(wsId, newValue); + changed = true; + } + foreach (int wsId in wsIdsToClear) + { + ITsString oldValue = getAlternative(wsId); + if (oldValue == null || string.IsNullOrEmpty(oldValue.Text)) + continue; + setAlternative(wsId, TsStringUtils.EmptyString(wsId)); + changed = true; + } + return changed; + } + + /// The start of one key's text, enough to find the entry it is in. + public string Excerpt(string key) + { + LfStringField field; + string text = (TryGetValue(key, out field) && field != null) ? (field.Value ?? string.Empty) : string.Empty; + return text.Length <= 40 ? text : text.Substring(0, 40) + "..."; + } + + /// + /// The keys holding text that passes over, each with the + /// reason: null where the key names no writing system LCM has, or else the key naming the + /// same writing system that is chosen over it. + /// + public List> KeysPassedOver(ILgWritingSystemFactory wsManager, + ISet configuredTags = null) + { + var chosenFor = ChooseKeys(wsManager, configuredTags).ToDictionary(kv => kv.Key, kv => kv.Value); + var passedOver = new List>(); foreach (KeyValuePair kv in this) { - int wsId = wsManager.GetWsFromStr(kv.Key); - if (wsId == 0) continue; // Skip any unidentified writing systems - string value = kv.Value.Value; - ITsString tss = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(value, wsId, wsManager); - dest.set_String(wsId, tss); - destWsIdsToClear.Remove(wsId); + if (kv.Value == null || kv.Value.IsEmpty) + continue; + int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key); + if (wsId == 0) + passedOver.Add(new KeyValuePair(kv.Key, null)); + else if (chosenFor[wsId] != kv.Key) + passedOver.Add(new KeyValuePair(kv.Key, chosenFor[wsId])); } - foreach (int wsId in destWsIdsToClear) + return passedOver; + } + + /// + /// Each writing system's key, in the order the writing systems first appear: the only key + /// naming it, or of two, the one picks. + /// + private List> ChooseKeys(ILgWritingSystemFactory wsManager, ISet configuredTags, + Action onUnidentifiedTag = null, Action onKeyNotWritten = null) + { + var keyFor = new Dictionary(); + var order = new List(); + foreach (string key in Keys) + { + int wsId = LanguageTags.WsIdFromLfTag(wsManager, key); + if (wsId == 0) + { + onUnidentifiedTag?.Invoke(key); + continue; + } + string other; + if (keyFor.TryGetValue(wsId, out other)) + { + string kept = PreferredKey(other, key, configuredTags); + onKeyNotWritten?.Invoke(kept == key ? other : key, kept); + keyFor[wsId] = kept; + } + else + { + keyFor[wsId] = key; + order.Add(wsId); + } + } + return order.Select(wsId => new KeyValuePair(wsId, keyFor[wsId])).ToList(); + } + + /// + /// Of two keys naming one writing system, the one to write: the one spelled as the config + /// spells it, or else the later one. LF's editor matches spellings exactly, so this does too. + /// + private static string PreferredKey(string earlier, string later, ISet configuredTags) + { + if (configuredTags != null) { - dest.set_String(wsId, string.Empty); + bool earlierConfigured = configuredTags.Contains(earlier); + if (earlierConfigured != configuredTags.Contains(later)) + return earlierConfigured ? earlier : later; } + return later; } } diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs new file mode 100644 index 00000000..620c47b0 --- /dev/null +++ b/src/LfMerge.Core/LanguageTags.cs @@ -0,0 +1,99 @@ +using System; +using System.Linq; +using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.WritingSystems; +using SIL.WritingSystems; + +namespace LfMerge.Core +{ + public static class LanguageTags + { + /// + /// The canonical form of a language tag: the id LCM knows a writing system by, once it has + /// created one from this tag. + /// + /// Language Forge stores a tag exactly as it was typed. LCM canonicalizes it when the writing + /// system is created -- WritingSystemManager.GetOrSet and Create both do -- and thereafter + /// GetWsFromStr matches that canonical id literally. It is case-insensitive, so a difference + /// of case resolves on its own, but it is not structure-aware, so a tag that loses or absorbs + /// a subtag is never found: + /// + /// qaa-x-qaa-v -> qaa-x-v the private-use section repeats the "qaa" language subtag, + /// and canonicalizing absorbs it + /// th-Thai -> th Thai's suppress-script is dropped + /// + /// Every lookup of an LF tag, LfWsToLcmWs's included, should go through + /// rather than calling this and GetWsFromStr itself. GetOrSet is + /// no substitute: it looks a tag up only as an Id. + /// + /// + /// The canonical tag, or the tag unchanged when it is empty or cannot be parsed -- + /// IetfLanguageTag.Canonicalize throws on those, and an unparseable tag should go on failing + /// where it failed before rather than here. + /// + public static string Canonical(string tag) + { + if (string.IsNullOrEmpty(tag) || !IetfLanguageTag.IsValid(tag)) + { + return tag; + } + return IetfLanguageTag.Canonicalize(tag); + } + + /// + /// The handle of the LCM writing system a Language Forge tag names, or 0 when there is none. + /// + /// Every place that resolves a tag LF supplied -- a multitext key, a span's lang attribute, + /// a multi-paragraph's input system -- goes through here, so that a non-canonical spelling + /// resolves the same way everywhere. + /// + /// LCM knows a writing system by its Id, which is whatever its .ldml file is named, and also + /// has its LanguageTag, which is always canonical. The two differ in older projects: + /// brb-flex-2022 holds "km-Khmr-KH.ldml", whose identity is km plus KH, so its Id is + /// "km-Khmr-KH" and its LanguageTag "km-KH". LfMerge writes LF's multitext keys by Id, but it + /// used to write the project's input systems by LanguageTag, and LF users type under the + /// spelling the config gives, so LF can hand back either spelling. Tried in turn: + /// + /// 1. The tag as LF spells it, as an Id. This must come first. brb-flex-2022 also holds a + /// second, unused writing system whose Id is "km-KH"; trying the canonical form first + /// sent the text under "km-Khmr-KH" there, and WriteToLcm then cleared the real + /// writing system's text as missing from LF. + /// 2. Its canonical form, as an Id: the Id LCM gives any writing system it creates. + /// 3. Its canonical form as a LanguageTag, for an Id spelled some other way: + /// spt-flex's "hi-IN" is the writing system LCM holds as "hi-Deva-IN". Should several + /// writing systems share the LanguageTag, the one with the lowest handle wins: LCM + /// numbers writing systems in the order it loads them, not the order they were made. + /// + public static int WsIdFromLfTag(ILgWritingSystemFactory wsf, string tag) + { + if (string.IsNullOrEmpty(tag)) + { + return 0; + } + int wsId = wsf.GetWsFromStr(tag); + if (wsId != 0) + { + return wsId; + } + string canonical = Canonical(tag); + if (!string.Equals(canonical, tag, StringComparison.OrdinalIgnoreCase)) + { + wsId = wsf.GetWsFromStr(canonical); + if (wsId != 0) + { + return wsId; + } + } + var wsManager = wsf as WritingSystemManager; + if (wsManager == null) + { + return 0; + } + CoreWritingSystemDefinition byLanguageTag = wsManager.WritingSystems + .Where(ws => string.Equals(ws.LanguageTag, canonical, StringComparison.OrdinalIgnoreCase)) + .OrderBy(ws => ws.Handle) + .FirstOrDefault(); + return (byLanguageTag == null) ? 0 : byLanguageTag.Handle; + } + } +}