diff --git a/docker/scripts/create-installation-tarball.sh b/docker/scripts/create-installation-tarball.sh
index b9f37e15..556d0995 100755
--- a/docker/scripts/create-installation-tarball.sh
+++ b/docker/scripts/create-installation-tarball.sh
@@ -52,53 +52,11 @@ install -m 755 output/${BUILD}/${FRAMEWORK}/LfMerge ${DBDESTDIR}/${LIB} 2>/dev/n
install -m 755 output/${BUILD}/${FRAMEWORK}/LfMergeQueueManager ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/LfMergeQueueManager ${DBDESTDIR}/${LIB}
install -m 755 output/${BUILD}/${FRAMEWORK}/chorusmerge ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/chorusmerge ${DBDESTDIR}/${LIB}
chmod 755 ${DBDESTDIR}/${LIB}/FixFwData.exe
-install -d ${DBDESTDIR}/${LIB}/Mercurial
-install -d ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport
-install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext
-install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert
-install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight
-install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles
-install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils
-install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils
-install -d ${DBDESTDIR}/${LIB}/MercurialExtensions
-install -d ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8
-install -m 755 Mercurial/chg ${DBDESTDIR}/${LIB}/Mercurial
-install -m 755 Mercurial/hg ${DBDESTDIR}/${LIB}/Mercurial
-install -m 644 Mercurial/mercurial.ini ${DBDESTDIR}/${LIB}/Mercurial
-install -m 644 Mercurial/hgdemandimport/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport
-install -m 644 Mercurial/hgext/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext
-install -m 644 Mercurial/hgext/convert/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert
-install -m 644 Mercurial/hgext/highlight/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight
-install -m 644 Mercurial/hgext/largefiles/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles
-install -m 644 Mercurial/hgext/zeroconf/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf
-install -m 644 Mercurial/mercurial/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial
-install -m 644 Mercurial/mercurial/defaultrc/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc
-install -m 644 Mercurial/mercurial/dirstateutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils
-install -m 644 Mercurial/mercurial/hgweb/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb
-install -m 644 Mercurial/mercurial/httpclient/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true
-install -m 644 Mercurial/mercurial/interfaces/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces
-install -m 644 Mercurial/mercurial/pure/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure
-install -m 644 Mercurial/mercurial/revlogutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils
-install -m 644 Mercurial/mercurial/stabletailgraph/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph
-install -m 644 Mercurial/mercurial/testing/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing
-install -m 644 Mercurial/mercurial/thirdparty/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty
-install -m 644 Mercurial/mercurial/thirdparty/attr/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr
-install -m 644 Mercurial/mercurial/utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils
-install -m 644 Mercurial/mercurial/upgrade_utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils
-install -m 644 MercurialExtensions/fixutf8/*.* ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8
+cp -a Mercurial ${DBDESTDIR}/${LIB}/
+cp -a MercurialExtensions ${DBDESTDIR}/${LIB}/
+# Ensure hg and chg are world-executable
+chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/hg
+chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/chg
install -d ${DBDESTDIR}/${LIB}/runtimes
install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64
install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64/lib
diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs
new file mode 100644
index 00000000..3773e8e2
--- /dev/null
+++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmCustomFieldTests.cs
@@ -0,0 +1,184 @@
+// Copyright (c) 2016-2018 SIL International
+// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
+
+using System;
+using System.Linq;
+using LfMerge.Core.DataConverters;
+using MongoDB.Bson;
+using NUnit.Framework;
+using SIL.LCModel;
+
+namespace LfMerge.Core.Tests.Lcm.DataConverters
+{
+ ///
+ /// Writing custom fields from LF into LCM.
+ ///
+ /// Cust_Single_Line_All in the test project is a MultiUnicode field with English and French
+ /// alternatives. That is the shape every text custom field an LF project owns takes: LF has
+ /// never offered a single-string custom field, so a field created there is always multilingual
+ /// even when only one writing system is filled in.
+ ///
+ public class ConvertMongoToLcmCustomFieldTests : LcmTestBase
+ {
+ private const string MultiTextField = "customField_entry_Cust_Single_Line_All";
+ // A String field, whose writing system is the magic "first analysis" selector.
+ private const string SingleStringField = "customField_entry_Cust_Single_Line";
+
+ private static BsonDocument MultiText(params string[] wsThenValue)
+ {
+ var multiText = new BsonDocument();
+ for (int i = 0; i < wsThenValue.Length; i += 2)
+ multiText.Add(wsThenValue[i], new BsonDocument("value", wsThenValue[i + 1]));
+ return multiText;
+ }
+
+ private ILexEntry TestEntry()
+ {
+ var entry = _servLoc.GetInstance().GetObject(Guid.Parse(TestEntryGuidStr));
+ Assert.That(entry, Is.Not.Null);
+ return entry;
+ }
+
+ private ConvertMongoToLcmCustomField Converter()
+ {
+ return new ConvertMongoToLcmCustomField(_cache, _servLoc,
+ new TestLogger(TestContext.CurrentContext.Test.Name), _wsEn);
+ }
+
+ /// Reads the entry's custom fields back out through the converter going the other way.
+ private BsonDocument ReadBack(ILexEntry entry)
+ {
+ var reader = new ConvertLcmToMongoCustomField(_cache, _servLoc,
+ new TestLogger(TestContext.CurrentContext.Test.Name));
+ return reader.GetCustomFieldsForThisCmObject(entry, "entry", _listConverters)[0].AsBsonDocument;
+ }
+
+ [Test]
+ public void SetCustomFieldsForThisCmObject_ShouldWriteEveryMultiUnicodeAlternative()
+ {
+ // Setup
+ ILexEntry entry = TestEntry();
+ var customFields = new BsonDocument(MultiTextField,
+ MultiText("en", "Updated English", "fr", "Updated French"));
+
+ // Exercise
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry", customFields, null);
+
+ // Verify
+ BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
+ Assert.That(written["en"]["value"].AsString, Is.EqualTo("Updated English"));
+ Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Updated French"));
+ }
+
+ [Test]
+ public void SetCustomFieldsForThisCmObject_ShouldWriteAFieldThatWasEmptyInLcm()
+ {
+ // The migration case: LF holds the only copy, and LCM has nothing in the field yet.
+ // Emptying it leaves the alternatives in place holding empty strings rather than removing
+ // them, so the precondition checks the contents rather than the presence of the key.
+ ILexEntry entry = TestEntry();
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry",
+ new BsonDocument(MultiTextField, MultiText("en", "", "fr", "")), null);
+ foreach (BsonElement alternative in ReadBack(entry)[MultiTextField].AsBsonDocument)
+ Assert.That(alternative.Value["value"].AsString, Is.Empty,
+ $"precondition: {alternative.Name} should have been emptied");
+
+ // Exercise
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry",
+ new BsonDocument(MultiTextField, MultiText("en", "Brand new")), null);
+
+ // Verify
+ Assert.That(ReadBack(entry)[MultiTextField]["en"]["value"].AsString, Is.EqualTo("Brand new"));
+ }
+
+ [Test]
+ public void SetCustomFieldsForThisCmObject_ShouldClearAnAlternativeLfNoLongerHas()
+ {
+ // The reverse converter exports every alternative LCM holds, so one missing from LF is
+ // one the user deleted there -- not one LF never knew about.
+ ILexEntry entry = TestEntry();
+ Assert.That(ReadBack(entry)[MultiTextField].AsBsonDocument.Contains("fr"), Is.True,
+ "precondition: the test project's field starts with a French alternative");
+
+ // Exercise
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry",
+ new BsonDocument(MultiTextField, MultiText("en", "English only")), null);
+
+ // Verify
+ BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
+ Assert.That(written["en"]["value"].AsString, Is.EqualTo("English only"));
+ Assert.That(written.Contains("fr"), Is.False, "French should have been cleared, not left stale");
+ }
+
+ [Test]
+ public void SetCustomFieldsForThisCmObject_ShouldSkipUnidentifiedWritingSystems()
+ {
+ // Setup
+ ILexEntry entry = TestEntry();
+
+ // Exercise
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry",
+ new BsonDocument(MultiTextField,
+ MultiText("en", "Kept", "zzz-x-nonesuch", "Dropped")), null);
+
+ // Verify
+ BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
+ Assert.That(written["en"]["value"].AsString, Is.EqualTo("Kept"));
+ Assert.That(written.Contains("zzz-x-nonesuch"), Is.False);
+ }
+
+ [Test]
+ public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagToTheCanonicalWritingSystem()
+ {
+ // LF keeps a tag as typed, while LCM knows the writing system only by its canonical id:
+ // "fr-Latn" canonicalizes to "fr", French's suppress-script being dropped. Looked up
+ // literally the key resolves to 0, and French would then be cleared as missing from LF.
+ ILexEntry entry = TestEntry();
+
+ // Exercise
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry",
+ new BsonDocument(MultiTextField, MultiText("en", "English", "fr-Latn", "Français")), null);
+
+ // Verify
+ BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
+ Assert.That(written.Contains("fr"), Is.True, "French should have been written, not cleared");
+ Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Français"));
+ }
+
+ [Test]
+ public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagInAStringField()
+ {
+ // The String case resolved the key it picked with a literal GetWsFromStr too, and gave
+ // up on the whole field when that returned 0, so the LF edit never reached LCM.
+ ILexEntry entry = TestEntry();
+
+ // Exercise
+ Converter().SetCustomFieldsForThisCmObject(entry, "entry",
+ new BsonDocument(SingleStringField, MultiText("fr-Latn", "Texte")), null);
+
+ // Verify
+ BsonDocument written = ReadBack(entry)[SingleStringField].AsBsonDocument;
+ Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Texte"));
+ }
+
+ [Test]
+ public void SetCustomFieldData_FieldOfATypeItCannotWrite_IsReportedOncePerSyncNotOncePerObject()
+ {
+ // Setup: an owning sequence, a type the LF to LCM direction does not implement, on
+ // three entries. The test project has no custom field of that type, so a built-in
+ // field stands in for one; the writer does not tell them apart.
+ var logger = new TestLogger(TestContext.CurrentContext.Test.Name);
+ var converter = new ConvertMongoToLcmCustomField(_cache, _servLoc, logger, _wsEn);
+ var entries = _servLoc.GetInstance().AllInstances().Take(3).ToList();
+ Assert.That(entries, Has.Count.EqualTo(3));
+
+ // Exercise
+ foreach (ILexEntry entry in entries)
+ Assert.That(converter.SetCustomFieldData(entry.Hvo, LexEntryTags.kflidSenses, new BsonString("LF data"), null), Is.False);
+
+ // Verify
+ int reports = logger.Messages.Split('\n').Count(line => line.Contains("not written to LCM"));
+ Assert.That(reports, Is.EqualTo(1));
+ }
+ }
+}
diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs
new file mode 100644
index 00000000..40cd4c3a
--- /dev/null
+++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/ConvertMongoToLcmWritingSystemsTests.cs
@@ -0,0 +1,881 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using LfMerge.Core.DataConverters;
+using LfMerge.Core.LanguageForge.Config;
+using LfMerge.Core.LanguageForge.Model;
+using LfMerge.Core.MongoConnector;
+using LfMergeBridge.LfMergeModel;
+using MongoDB.Bson;
+using NUnit.Framework;
+using SIL.LCModel;
+using SIL.LCModel.Infrastructure;
+using SIL.LCModel.Core.KernelInterfaces;
+using SIL.LCModel.Core.Text;
+using SIL.LCModel.Core.WritingSystems;
+
+namespace LfMerge.Core.Tests.Lcm.DataConverters
+{
+ ///
+ /// Language Forge stores a writing system tag exactly as it was typed, but LCM canonicalizes a
+ /// tag when it creates a writing system and thereafter knows that writing system by its Id. LCM's
+ /// own lookups -- WritingSystemManager.TryGet, GetWsFromStr -- match an Id regardless of case
+ /// but otherwise literally, so a tag differing from the Id in structure, a subtag absorbed or
+ /// dropped, is not found by them; LanguageTags.WsIdFromLfTag finds it.
+ ///
+ /// testlangproj already contains "qaa-x-kal" and "qaa-Zxxx-x-kal-audio", so a non-canonical LF
+ /// spelling of either exercises the mismatch without adding to the shared fixture project. The
+ /// tests that need a writing system of their own add it under a tag no other test uses, since
+ /// what is added to the fixture's LCM stays for the rest of the run.
+ ///
+ public class ConvertMongoToLcmWritingSystemsTests : LcmTestBase
+ {
+ // The apltw2016 shape: the private-use section repeats the "qaa" language subtag, which
+ // canonicalizing absorbs (apltw2016 itself has "qaa-x-qaa-v", which becomes "qaa-x-v").
+ private const string RedundantPrivateUseTag = "qaa-x-qaa-kal";
+ private const string RedundantPrivateUseId = "qaa-x-kal";
+
+ // The shape found in the testlangproj-derived projects, where canonicalizing only changes case.
+ // This one ALREADY works, because both LCM lookups (TryGet and GetWsFromStr) are
+ // case-insensitive; it is here as a guard so that a fix for the structural case does not
+ // regress it. Only differences in structure -- a subtag absorbed or dropped -- go unfound.
+ private const string MixedCaseTag = "qaa-Zxxx-x-kal-AUDIO";
+ private const string MixedCaseId = "qaa-Zxxx-x-kal-audio";
+
+ ///
+ /// Adds one input system to the mock project record, and returns an action that puts the
+ /// record back as it was. Each test gets a fresh record, so the restore is only tidiness;
+ /// what does outlive a test is a writing system it adds to LCM, which is why the tests that
+ /// add one each use tags of their own.
+ ///
+ private Action AddLfInputSystem(string tag)
+ {
+ MongoProjectRecord record = _recordFactory.Create(_lfProj);
+ Assert.That(record, Is.Not.Null, "the mock project record factory should supply a record");
+ Assert.That(record.InputSystems.ContainsKey(tag), Is.False,
+ "the fixture's input systems should not already contain {0}", tag);
+
+ record.InputSystems[tag] = new LfInputSystemRecord {
+ Abbreviation = tag,
+ Tag = tag,
+ LanguageName = "Unlisted Language",
+ IsRightToLeft = false
+ };
+ return () => record.InputSystems.Remove(tag);
+ }
+
+ ///
+ /// Adds a tag to one config field's input systems, found by its path through the entry's
+ /// field lists, and returns an action that takes it out again. Like the input systems, the
+ /// config comes fresh with each test's record, so this too is only tidiness.
+ ///
+ private Action AddToConfigField(string tag, params string[] path)
+ {
+ LfConfigFieldBase field = _recordFactory.Create(_lfProj).Config.Entry;
+ foreach (string key in path)
+ field = ((LfConfigFieldList)field).Fields[key];
+ var multiText = (LfConfigMultiText)field;
+ multiText.InputSystems.Add(tag);
+ return () => multiText.InputSystems.Remove(tag);
+ }
+
+ private IEnumerable LcmWritingSystemIds()
+ {
+ return _cache.ServiceLocator.WritingSystemManager.WritingSystems.Select(ws => ws.Id);
+ }
+
+ [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)]
+ [TestCase(MixedCaseTag, MixedCaseId)]
+ public void LfWsToLcmWs_NonCanonicalTag_ReusesTheCanonicalWritingSystem(string lfTag, string canonicalId)
+ {
+ // Setup: LF holds the writing system under a non-canonical tag, while the LCM project
+ // already holds it under the canonical id that LCM derives from that same tag.
+ Assert.That(LcmWritingSystemIds(), Contains.Item(canonicalId),
+ "testlangproj should already contain {0}", canonicalId);
+ int countBefore = LcmWritingSystemIds().Count(id => id == canonicalId);
+ Action restore = AddLfInputSystem(lfTag);
+
+ try
+ {
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify: the existing writing system was reused, not duplicated, and the raw LF tag
+ // did not become an id of its own.
+ Assert.That(LcmWritingSystemIds().Count(id => id == canonicalId), Is.EqualTo(countBefore),
+ "{0} should still appear exactly once", canonicalId);
+ Assert.That(LcmWritingSystemIds(), Does.Not.Contain(lfTag),
+ "the non-canonical tag {0} should not be stored as an id of its own", lfTag);
+ }
+ finally
+ {
+ restore();
+ }
+ }
+
+ [Test]
+ public void SetMultiStringFrom_NonCanonicalTagInLexicon_WritesToTheCanonicalWritingSystem()
+ {
+ // Setup: a definition keyed by the non-canonical tag. LfMultiText used to resolve each key
+ // with GetWsFromStr and skip anything that returned 0, so an unresolved key dropped the
+ // value -- and the clearing pass then blanked whatever LCM had for that field.
+ const string newDefinition = "Definition stored under a non-canonical writing system tag";
+ var data = new SampleData();
+ data.bsonTestData["senses"][0]["definition"] = new BsonDocument {
+ { RedundantPrivateUseTag, new BsonDocument { { "value", newDefinition } } }
+ };
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow;
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+
+ int wsId = _cache.WritingSystemFactory.GetWsFromStr(RedundantPrivateUseId);
+ Assert.That(wsId, Is.Not.EqualTo(0), "{0} should resolve in testlangproj", RedundantPrivateUseId);
+
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ Guid expectedGuid = Guid.Parse(data.bsonTestData["guid"].AsString);
+ var entry = _cache.ServiceLocator.GetObject(expectedGuid) as ILexEntry;
+ Assert.That(entry, Is.Not.Null);
+ Assert.That(entry.SensesOS[0].Definition.get_String(wsId).Text, Is.EqualTo(newDefinition),
+ "the definition should have been written to the canonical writing system {0}", RedundantPrivateUseId);
+ }
+
+ ///
+ /// Text in a built-in multitext field under a key that names no writing system LCM has has
+ /// nowhere to go. It is skipped, as it always was, but no longer silently: the custom fields
+ /// and the single-string fields already said so, and this, the commonest path, did not.
+ ///
+ [Test]
+ public void SetMultiStringFrom_KeyNamingNoWritingSystem_IsSkippedWithAWarning()
+ {
+ const string tag = "zzz-x-nonesuch";
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, tag), Is.EqualTo(0));
+ var data = new SampleData();
+ data.bsonTestData["senses"][0]["definition"][tag] = new BsonDocument { { "value", "a definition with nowhere to go" } };
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow;
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+
+ SutMongoToLcm.Run(_lfProj);
+
+ Assert.That(_env.Logger.GetMessages(),
+ Does.Contain("MongoToLcm: skipping text under \"zzz-x-nonesuch\" (a definition with nowhere to go)"));
+ }
+
+ [Test]
+ public void SingleStringField_TextInNoKnownWritingSystem_KeepsTheLcmValueAndSyncsTheRest()
+ {
+ // Setup: LCM holds a scientific name, and then LF holds one only in a writing system LCM
+ // has never heard of. Building that into a string in writing system 0 threw, abandoning
+ // the rest of the entry half-written; returning null instead would clear the field and
+ // delete what LCM holds. Neither is acceptable.
+ const string original = "Homo sapiens";
+ const string laterNote = "A note set after the scientific name";
+ var data = new SampleData();
+ BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument;
+ sense["scientificName"] = new BsonDocument { { "en", new BsonDocument { { "value", original } } } };
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow;
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+ SutMongoToLcm.Run(_lfProj);
+
+ sense["scientificName"] = new BsonDocument {
+ { "zzz-x-nonesuch", new BsonDocument { { "value", "Unplaceable" } } } };
+ // Converted after the scientific name, so it only arrives if the entry is not abandoned.
+ sense["sociolinguisticsNote"] = new BsonDocument {
+ { "en", new BsonDocument { { "value", laterNote } } } };
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow.AddMinutes(1);
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ var entry = _cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)) as ILexEntry;
+ Assert.That(entry, Is.Not.Null);
+ ILexSense lcmSense = entry.SensesOS[0];
+ Assert.That(lcmSense.ScientificName.Text, Is.EqualTo(original),
+ "text that cannot be placed must not replace or clear what LCM holds");
+ Assert.That(lcmSense.SocioLinguisticsNote.get_String(_wsEn).Text, Is.EqualTo(laterNote),
+ "the rest of the entry should still have been converted");
+ Assert.That(_env.Logger.GetMessages(), Does.Contain(
+ "MongoToLcm: skipping text under \"zzz-x-nonesuch\" (Unplaceable), which names no writing system LCM has; " +
+ "leaving the field as it was"));
+ }
+
+ ///
+ /// A single-string field whose text is all under a key passed over for an empty one naming
+ /// the same writing system -- the config's spelling, which LF's editor fills in empty --
+ /// keeps what LCM holds, and the log says so for that reason, not that LCM lacks the
+ /// writing system.
+ ///
+ [Test]
+ public void SingleStringField_TextOnlyUnderAKeyPassedOverForAnEmptyOne_KeepsTheLcmValueAndSaysWhy()
+ {
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId("ro-Latn-MD", "ro-MD");
+ const string original = "Homo sapiens";
+ var data = new SampleData();
+ BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument;
+ sense["scientificName"] = new BsonDocument { { "en", new BsonDocument { { "value", original } } } };
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow;
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+ SutMongoToLcm.Run(_lfProj);
+
+ Action restore = AddToConfigField("ro-MD", "senses", "scientificName");
+ try
+ {
+ sense["scientificName"] = new BsonDocument {
+ { "ro-Latn-MD", new BsonDocument { { "value", "the export's value" } } },
+ { "ro-MD", new BsonDocument { { "value", "" } } } };
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow.AddMinutes(1);
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ var entry = _cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString)) as ILexEntry;
+ Assert.That(entry.SensesOS[0].ScientificName.Text, Is.EqualTo(original));
+ Assert.That(_env.Logger.GetMessages(), Does.Contain(
+ "MongoToLcm: keys \"ro-Latn-MD\" and \"ro-MD\" name the same writing system, and \"ro-MD\", the one to " +
+ "write, is empty; leaving the field as it was rather than write \"ro-Latn-MD\" (the export's value)"));
+ Assert.That(_env.Logger.GetMessages(), Does.Not.Contain("names no writing system LCM has"));
+ }
+ finally
+ {
+ restore();
+ }
+ }
+
+ [TestCase(RedundantPrivateUseTag, RedundantPrivateUseId)]
+ [TestCase(MixedCaseTag, MixedCaseId)]
+ [TestCase("fr-Latn", "fr")]
+ public void WsIdFromLfTag_NonCanonicalTag_ResolvesToTheCanonicalWritingSystem(string lfTag, string canonicalId)
+ {
+ int expected = _cache.WritingSystemFactory.GetWsFromStr(canonicalId);
+ Assert.That(expected, Is.Not.EqualTo(0), "{0} should resolve in testlangproj", canonicalId);
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, lfTag), Is.EqualTo(expected));
+ }
+
+ ///
+ /// Adds a writing system whose Id is spelled differently from its LanguageTag, the way LCM
+ /// loads an older .ldml file: brb-flex-2022's "km-Khmr-KH.ldml" has km plus KH for its
+ /// identity, so it is held as Id "km-Khmr-KH" with LanguageTag "km-KH". The repository
+ /// keeps an Id as given whenever it is equivalent to the LanguageTag.
+ ///
+ private CoreWritingSystemDefinition AddWritingSystemWithId(string id, string languageTag)
+ {
+ WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager;
+ CoreWritingSystemDefinition ws = wsManager.Create(languageTag);
+ ws.Id = id;
+ wsManager.Set(ws);
+ Assert.That(ws.Id, Is.EqualTo(id), "precondition: LCM should hold the writing system as {0}", id);
+ Assert.That(ws.LanguageTag, Is.EqualTo(languageTag));
+ return ws;
+ }
+
+ [Test]
+ public void WsIdFromLfTag_ExactIdWinsOverAnotherWritingSystemWithTheCanonicalId()
+ {
+ // The brb-flex-2022 layout: the real writing system under a non-canonical Id, and an
+ // unused second one whose Id is the canonical form. Text keyed by the real one's Id is
+ // the real one's text; resolving the canonical form first sent it to the other, and the
+ // clearing pass then deleted the real writing system's text as missing from LF.
+ CoreWritingSystemDefinition real = AddWritingSystemWithId("de-Latn-AT", "de-AT");
+ CoreWritingSystemDefinition stray = _cache.ServiceLocator.WritingSystemManager.Set("de-AT");
+ Assert.That(stray.Handle, Is.Not.EqualTo(real.Handle));
+
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "de-Latn-AT"), Is.EqualTo(real.Handle));
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "de-AT"), Is.EqualTo(stray.Handle));
+ }
+
+ [Test]
+ public void WsIdFromLfTag_CanonicalTagFindsAWritingSystemWhoseIdIsSpelledOtherwise()
+ {
+ // The spt-flex layout: LCM holds "hi-Deva-IN" and LF's input systems call it "hi-IN",
+ // which is neither the Id nor its canonical form spelled as an Id.
+ CoreWritingSystemDefinition real = AddWritingSystemWithId("fr-Latn-CA", "fr-CA");
+ Assert.That(_cache.WritingSystemFactory.GetWsFromStr("fr-CA"), Is.EqualTo(0),
+ "precondition: no writing system has the Id fr-CA");
+
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "fr-CA"), Is.EqualTo(real.Handle));
+ }
+
+ [Test]
+ public void LfWsToLcmWs_InputSystemSpelledAsTheLanguageTag_DoesNotDuplicateTheWritingSystem()
+ {
+ // The spt-flex layout: LCM holds "hi-Deva-IN", whose LanguageTag is "hi-IN", and LF's
+ // input systems -- which LfMerge used to export by LanguageTag -- call it "hi-IN". GetOrSet looks
+ // only at Ids, so it created a second writing system "hi-IN" and put it in the lists.
+ CoreWritingSystemDefinition real = AddWritingSystemWithId("pt-Latn-AO", "pt-AO");
+ Action restore = AddLfInputSystem("pt-AO");
+
+ try
+ {
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ Assert.That(LcmWritingSystemIds(), Does.Not.Contain("pt-AO"),
+ "no duplicate writing system should be created for the input system");
+ Assert.That(_cache.ServiceLocator.WritingSystemManager.WritingSystems
+ .Count(ws => ws.LanguageTag == "pt-AO"), Is.EqualTo(1));
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "pt-AO"), Is.EqualTo(real.Handle),
+ "LF text keyed \"pt-AO\" should go to the existing writing system");
+ }
+ finally
+ {
+ restore();
+ }
+ }
+
+ ///
+ /// An input system spelled by Id -- as every project LfMerge set up under FieldWorks 8 has
+ /// them -- for a writing system whose Id differs from its LanguageTag. Replace looked the
+ /// writing system up by LanguageTag, which either found nothing and re-registered it under a
+ /// fresh handle, or found the unused writing system with that LanguageTag as its Id and
+ /// evicted it, the real one taking over its handle.
+ ///
+ [TestCase("es-Latn-MX", "es-MX", false)]
+ [TestCase("it-Latn-CH", "it-CH", true)]
+ public void LfWsToLcmWs_ExistingWritingSystemWhoseIdIsNotItsLanguageTag_KeepsItsHandle(
+ string id, string languageTag, bool withUnusedNamesake)
+ {
+ WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager;
+ CoreWritingSystemDefinition real = AddWritingSystemWithId(id, languageTag);
+ int realHandle = real.Handle;
+ CoreWritingSystemDefinition namesake = withUnusedNamesake ? wsManager.Set(languageTag) : null;
+ int namesakeHandle = withUnusedNamesake ? namesake.Handle : 0;
+ Action restore = AddLfInputSystem(id);
+
+ try
+ {
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ Assert.That(real.Handle, Is.EqualTo(realHandle), "the writing system should keep its handle");
+ Assert.That(wsManager.GetWsFromStr(id), Is.EqualTo(realHandle));
+ Assert.That(real.Abbreviation, Is.EqualTo(id),
+ "the input system's properties should still have been applied");
+ if (withUnusedNamesake)
+ {
+ Assert.That(wsManager.GetWsFromStr(languageTag), Is.EqualTo(namesakeHandle),
+ "the other writing system with that LanguageTag should be left alone");
+ }
+ }
+ finally
+ {
+ restore();
+ }
+ }
+
+ ///
+ /// Two LF keys naming one writing system: the export's, spelled as its Id, and one spelled
+ /// as its LanguageTag. Only one can be written, and it is the one under the spelling LF's
+ /// config uses, wherever it comes, since that is the one LF's editor shows; with neither
+ /// spelling configured, the later one. The other is reported.
+ ///
+ /// The export's key comes first in an entry, since an LF user's new key is added after it.
+ /// Before the config is re-spelled by Id, the user's edit is under the LanguageTag and the
+ /// export's value is hidden and stale; afterwards, an Id-spelled edit is made in place, and a
+ /// LanguageTag key left from before is what is stale.
+ ///
+ // A pair each, since writing systems a test adds stay in the fixture's project
+ [TestCase("languageTag", true, "sv-Latn-FI", "sv-FI", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelledByLanguageTag_WritesTheEdit")]
+ [TestCase("id", true, "da-Latn-DE", "da-DE", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelledById_WritesTheEditMadeInPlace")]
+ [TestCase("languageTag", false, "nb-Latn-NO", "nb-NO", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_ConfigSpelling_WinsWhereverItComes")]
+ [TestCase("neither", true, "fi-Latn-SE", "fi-SE", TestName = "WriteToLcm_TwoKeysForOneWritingSystem_NeitherConfigured_WritesTheLaterKey")]
+ public void WriteToLcm_TwoKeysForOneWritingSystem(string configured, bool idFirst, string id, string languageTag)
+ {
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, languageTag);
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, languageTag), Is.EqualTo(ws.Handle),
+ "precondition: the LanguageTag spelling should resolve to the same writing system");
+ var multiText = new LfMultiText();
+ if (idFirst)
+ multiText.Add(id, LfStringField.FromString("under the Id"));
+ multiText.Add(languageTag, LfStringField.FromString("under the LanguageTag"));
+ if (!idFirst)
+ multiText.Add(id, LfStringField.FromString("under the Id"));
+ var configuredTags = new HashSet(StringComparer.Ordinal) { "en", "fr" };
+ if (configured == "id")
+ configuredTags.Add(id);
+ else if (configured == "languageTag")
+ configuredTags.Add(languageTag);
+ string expected = configured == "id" ? id : configured == "languageTag" ? languageTag : (idFirst ? languageTag : id);
+ var gloss = ((ILexEntry)_cache.ServiceLocator.GetObject(Guid.Parse(TestEntryGuidStr))).SensesOS[0].Gloss;
+ var reported = new List<(string NotWritten, string Written)>();
+
+ UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () =>
+ multiText.WriteToLcmMultiString(gloss, _cache.WritingSystemFactory, configuredTags,
+ onKeyNotWritten: (notWritten, written) => reported.Add((notWritten, written))));
+
+ Assert.That(gloss.get_String(ws.Handle).Text, Is.EqualTo(expected == id ? "under the Id" : "under the LanguageTag"));
+ Assert.That(reported, Is.EqualTo(new[] { (expected == id ? languageTag : id, expected) }));
+ }
+
+ ///
+ /// The config's spellings reach every writer that chooses between two keys naming one
+ /// writing system, through a whole sync: the built-in multitext fields, and picture captions
+ /// with them; the single-string fields; and the multitext and string custom fields. The
+ /// config's spelling comes first here, so a writer going on key order alone would write the
+ /// export's value instead.
+ ///
+ // A pair each, since writing systems a test adds stay in the fixture's project
+ [TestCase("gloss", "hu-Latn-SK", "hu-SK")]
+ [TestCase("scientificName", "sk-Latn-HU", "sk-HU")]
+ [TestCase("customMultiText", "sl-Latn-IT", "sl-IT")]
+ [TestCase("customString", "et-Latn-FI", "et-FI")]
+ public void MongoToLcm_TwoKeysForOneWritingSystem_WritesTheConfigsSpellingInEveryKindOfField(
+ string field, string id, string languageTag)
+ {
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, languageTag);
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, languageTag), Is.EqualTo(ws.Handle),
+ "precondition: the LanguageTag spelling should resolve to the same writing system");
+ var twoKeys = new BsonDocument {
+ { languageTag, new BsonDocument { { "value", "the user's edit" } } },
+ { id, new BsonDocument { { "value", "the export's value" } } } };
+ var data = new SampleData();
+ BsonDocument sense = data.bsonTestData["senses"][0].AsBsonDocument;
+ BsonDocument customFields = data.bsonTestData["customFields"].AsBsonDocument;
+ string[] configPath;
+ switch (field)
+ {
+ case "gloss":
+ sense["gloss"] = twoKeys;
+ configPath = new[] { "senses", "gloss" };
+ break;
+ case "scientificName":
+ sense["scientificName"] = twoKeys;
+ configPath = new[] { "senses", "scientificName" };
+ break;
+ case "customMultiText":
+ customFields["customField_entry_Cust_Single_Line_All"] = twoKeys;
+ configPath = new[] { "customField_entry_Cust_Single_Line_All" };
+ break;
+ default:
+ customFields["customField_entry_Cust_Single_Line"] = twoKeys;
+ configPath = new[] { "customField_entry_Cust_Single_Line" };
+ break;
+ }
+ data.bsonTestData["authorInfo"]["modifiedDate"] = DateTime.UtcNow;
+ _conn.UpdateMockLfLexEntry(data.bsonTestData);
+ Action restore = AddToConfigField(languageTag, configPath);
+
+ try
+ {
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ var entry = (ILexEntry)_cache.ServiceLocator.GetObject(Guid.Parse(data.bsonTestData["guid"].AsString));
+ ISilDataAccess sda = _cache.DomainDataByFlid;
+ string written;
+ switch (field)
+ {
+ case "gloss":
+ written = entry.SensesOS[0].Gloss.get_String(ws.Handle).Text;
+ break;
+ case "scientificName":
+ written = entry.SensesOS[0].ScientificName.Text;
+ break;
+ case "customMultiText":
+ written = sda.get_MultiStringAlt(entry.Hvo, EntryCustomFieldId("Cust Single Line All"), ws.Handle).Text;
+ break;
+ default:
+ written = sda.get_StringProp(entry.Hvo, EntryCustomFieldId("Cust Single Line")).Text;
+ break;
+ }
+ Assert.That(written, Is.EqualTo("the user's edit"));
+ }
+ finally
+ {
+ restore();
+ }
+ }
+
+ private int EntryCustomFieldId(string name)
+ {
+ int flid = _cache.MetaDataCacheAccessor.GetFieldId2(LexEntryTags.kClassId, name, false);
+ Assert.That(flid, Is.Not.EqualTo(0), "testlangproj should have the custom field {0}", name);
+ return flid;
+ }
+
+ ///
+ /// The xkk-flex-2022 layout, carried all the way round: one writing system whose Id is not
+ /// its LanguageTag, and a second whose Id IS that LanguageTag, each with text of its own.
+ /// LfMerge used to look the first up by LanguageTag when re-registering it, evict the second
+ /// and take its handle, so the two writing systems' text could end up under one of them.
+ /// Each must keep its own text, in LCM and in LF, through an export, an LF edit and an
+ /// import. The pair is one whose LanguageTag is the same under both SLDR datasets; xkk's own
+ /// is not, and the tests run on the older one.
+ ///
+ [Test]
+ public void RoundTrip_WritingSystemAndItsLanguageTagNamesake_EachKeepsItsOwnText()
+ {
+ // Setup: both writing systems in the analysis list, each with a gloss
+ const string id = "ca-Latn-AD", namesakeId = "ca-AD";
+ WritingSystemManager wsManager = _cache.ServiceLocator.WritingSystemManager;
+ CoreWritingSystemDefinition real = AddWritingSystemWithId(id, namesakeId);
+ CoreWritingSystemDefinition namesake = wsManager.Set(namesakeId);
+ int realHandle = real.Handle, namesakeHandle = namesake.Handle;
+ Assert.That(namesakeHandle, Is.Not.EqualTo(realHandle));
+ Guid entryGuid = Guid.Parse(TestEntryGuidStr);
+ var lcmEntry = (ILexEntry)_cache.ServiceLocator.GetObject(entryGuid);
+ UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () =>
+ {
+ _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(real);
+ _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(namesake);
+ lcmEntry.SensesOS[0].Gloss.set_String(realHandle, "glossa andorrana");
+ lcmEntry.SensesOS[0].Gloss.set_String(namesakeHandle, "glossa de l'homonima");
+ });
+
+ // Exercise: export, edit both in LF, import, export again
+ SutLcmToMongo.Run(_lfProj);
+ LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid);
+ Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo("glossa andorrana"));
+ Assert.That(lfEntry.Senses[0].Gloss[namesakeId].Value, Is.EqualTo("glossa de l'homonima"));
+ lfEntry.Senses[0].Gloss[id] = LfStringField.FromString("glossa andorrana, corregida a LF");
+ lfEntry.Senses[0].Gloss[namesakeId] = LfStringField.FromString("glossa de l'homonima, corregida a LF");
+ lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow;
+ _conn.UpdateMockLfLexEntry(lfEntry);
+ var restores = new[] { AddLfInputSystem(id), AddLfInputSystem(namesakeId) };
+ try
+ {
+ SutMongoToLcm.Run(_lfProj);
+ SutLcmToMongo.Run(_lfProj);
+
+ // Verify: neither writing system took the other's place or its text
+ Assert.That(real.Handle, Is.EqualTo(realHandle));
+ Assert.That(namesake.Handle, Is.EqualTo(namesakeHandle));
+ Assert.That(wsManager.GetWsFromStr(id), Is.EqualTo(realHandle));
+ Assert.That(wsManager.GetWsFromStr(namesakeId), Is.EqualTo(namesakeHandle));
+ Assert.That(lcmEntry.SensesOS[0].Gloss.get_String(realHandle).Text,
+ Is.EqualTo("glossa andorrana, corregida a LF"));
+ Assert.That(lcmEntry.SensesOS[0].Gloss.get_String(namesakeHandle).Text,
+ Is.EqualTo("glossa de l'homonima, corregida a LF"));
+ lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid);
+ Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo("glossa andorrana, corregida a LF"));
+ Assert.That(lfEntry.Senses[0].Gloss[namesakeId].Value, Is.EqualTo("glossa de l'homonima, corregida a LF"));
+ Assert.That(_conn.GetInputSystems(_lfProj).Keys, Is.SupersetOf(new[] { id, namesakeId }));
+ }
+ finally
+ {
+ foreach (Action restore in restores)
+ restore();
+ }
+ }
+
+ ///
+ /// LF's editor matches a field's input systems against an entry's keys exactly, so the two
+ /// must be spelled alike. The keys are spelled by Id; the input systems were spelled by
+ /// LanguageTag, so brb-flex-2022's Khmer text, keyed "km-Khmr-KH" under an input system
+ /// called "km-KH", showed as empty.
+ ///
+ [Test]
+ public void LcmToMongo_WritingSystemWhoseIdIsNotItsLanguageTag_IsSpelledByIdEverywhere()
+ {
+ // Setup: an analysis writing system held under a non-canonical Id, with a gloss in it.
+ const string id = "nl-Latn-BE";
+ const string gloss = "Nederlandse glos";
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId(id, "nl-BE");
+ Guid entryGuid = Guid.Parse(TestEntryGuidStr);
+ UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () =>
+ {
+ _cache.LanguageProject.AddToCurrentAnalysisWritingSystems(ws);
+ var lcmEntry = (ILexEntry)_cache.ServiceLocator.GetObject(entryGuid);
+ lcmEntry.SensesOS[0].Gloss.set_String(ws.Handle, gloss);
+ });
+
+ // Exercise
+ SutLcmToMongo.Run(_lfProj);
+
+ // Verify
+ Dictionary inputSystems = _conn.GetInputSystems(_lfProj);
+ Assert.That(inputSystems.Keys, Contains.Item(id));
+ Assert.That(inputSystems[id].Tag, Is.EqualTo(id));
+ Assert.That(inputSystems.Keys, Does.Not.Contain("nl-BE"));
+ LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(_lfProj, entryGuid);
+ Assert.That(lfEntry.Senses[0].Gloss.Keys, Contains.Item(id),
+ "the multitext key and the input system should be spelled alike");
+ Assert.That(lfEntry.Senses[0].Gloss[id].Value, Is.EqualTo(gloss));
+ Assert.That(_conn.LastAnalysisWss, Contains.Item(id));
+ Assert.That(_conn.LastAnalysisWss, Does.Not.Contain("nl-BE"));
+ var customFieldTags = _conn.GetCustomFieldConfig(_lfProj).Values.OfType()
+ .SelectMany(field => field.InputSystems).ToList();
+ Assert.That(customFieldTags, Contains.Item(id), "a custom field offering every analysis writing system");
+ Assert.That(customFieldTags, Does.Not.Contain("nl-BE"));
+ }
+
+ ///
+ /// testlangproj's current vernacular list names "qaa-Zxxx-x-kal-AUDIO", and LCM holds the
+ /// writing system as "qaa-Zxxx-x-kal-audio". It is in the vernacular list LF is given, so its
+ /// input system must be flagged vernacular too; the list's own Contains compares Ids exactly.
+ ///
+ [Test]
+ public void LcmToMongo_ListSpellingAnIdInAnotherCase_FlagsTheInputSystemAsInTheList()
+ {
+ Assert.That(_cache.LanguageProject.CurVernWss, Does.Contain(MixedCaseTag), "precondition");
+
+ SutLcmToMongo.Run(_lfProj);
+
+ Assert.That(_conn.LastVernacularWss, Contains.Item(MixedCaseId));
+ Assert.That(_conn.GetInputSystems(_lfProj)[MixedCaseId].VernacularWS, Is.True);
+ }
+
+ [Test]
+ public void LcmToMongo_PronunciationListNamingAWritingSystemLcmDoesNotHave_LeavesItOut()
+ {
+ // Setup: a pronunciation list naming an Id LCM has no writing system for. (The same in
+ // the vernacular list would still fail, inside liblcm's own GetWritingSystemList.)
+ string vernacularId = _cache.LanguageProject.DefaultVernacularWritingSystem.Id;
+ UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", _cache.ActionHandlerAccessor, () =>
+ _cache.LanguageProject.CurPronunWss = vernacularId + " zzz-x-nonesuch");
+
+ // Exercise
+ SutLcmToMongo.Run(_lfProj);
+
+ // Verify: the list LF is given keeps the writing system LCM has and leaves out the other
+ Assert.That(_conn.LastPronunciationWss, Is.EqualTo(new[] { vernacularId }));
+ }
+
+ [Test]
+ public void WsIdFromLfTag_UnknownOrEmptyTag_IsZero()
+ {
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, "zzz-x-nonesuch"), Is.EqualTo(0));
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, ""), Is.EqualTo(0));
+ Assert.That(LanguageTags.WsIdFromLfTag(_cache.WritingSystemFactory, null), Is.EqualTo(0));
+ }
+
+ [Test]
+ public void BestStringAndWsId_NonCanonicalKey_StillMatchesThePreferredWritingSystem()
+ {
+ // The lexicon's single-string fields compared LF's keys with ws.Id, so "fr-Latn" never
+ // matched French and the English value was chosen although French comes first.
+ int wsFr = _cache.WritingSystemFactory.GetWsFromStr("fr");
+ var multiText = new LfMultiText {
+ { "en", LfStringField.FromString("English") },
+ { "fr-Latn", LfStringField.FromString("Français") },
+ };
+
+ KeyValuePair best = multiText.BestStringAndWsId(new[] { wsFr, _wsEn }, _cache.WritingSystemFactory);
+
+ Assert.That(best.Key, Is.EqualTo(wsFr));
+ Assert.That(best.Value, Is.EqualTo("Français"));
+ }
+
+ [Test]
+ public void BestStringAndWsId_FallsBackToTheFirstValueInAKnownWritingSystem()
+ {
+ // An unidentified key comes first, but a value LCM has no writing system for is no use.
+ var multiText = new LfMultiText {
+ { "zzz-x-nonesuch", LfStringField.FromString("Unplaceable") },
+ { "en", LfStringField.FromString("English") },
+ };
+
+ KeyValuePair best = multiText.BestStringAndWsId(new int[0], _cache.WritingSystemFactory);
+
+ Assert.That(best.Key, Is.EqualTo(_wsEn));
+ Assert.That(best.Value, Is.EqualTo("English"));
+ }
+
+ ///
+ /// A single-string field chooses between two keys naming one writing system as a multitext
+ /// field does: the one spelled as the config spells it. Here the export's hidden value is
+ /// under the Id and the LF user's under the LanguageTag, which the config uses.
+ ///
+ [Test]
+ public void BestStringAndWsId_TwoKeysForOneWritingSystem_TakesTheConfigsSpelling()
+ {
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId("pl-Latn-PL", "pl-PL");
+ var multiText = new LfMultiText {
+ { "pl-Latn-PL", LfStringField.FromString("the export's value") },
+ { "pl-PL", LfStringField.FromString("the user's edit") },
+ };
+ var configuredTags = new HashSet(StringComparer.Ordinal) { "pl-PL" };
+
+ KeyValuePair best = multiText.BestStringAndWsId(new[] { ws.Handle }, _cache.WritingSystemFactory, configuredTags);
+
+ Assert.That(best.Key, Is.EqualTo(ws.Handle));
+ Assert.That(best.Value, Is.EqualTo("the user's edit"));
+ }
+
+ ///
+ /// The key is chosen before empty values are left out: a value the LF user has cleared,
+ /// under the config's spelling, is not replaced by the export's hidden one.
+ ///
+ [Test]
+ public void BestStringAndWsId_ValueClearedUnderTheConfigsSpelling_IsNotReplacedByTheHiddenOne()
+ {
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId("cs-Latn-CZ", "cs-CZ");
+ var multiText = new LfMultiText {
+ { "cs-Latn-CZ", LfStringField.FromString("the export's value") },
+ { "cs-CZ", LfStringField.FromString("") },
+ };
+ var configuredTags = new HashSet(StringComparer.Ordinal) { "cs-CZ" };
+
+ KeyValuePair best = multiText.BestStringAndWsId(new[] { ws.Handle }, _cache.WritingSystemFactory, configuredTags);
+
+ Assert.That(best.Value, Is.Null, "nothing should be placed for the cleared writing system");
+ }
+
+ [Test]
+ public void KeysPassedOver_NamesEachKeyWithTextLeftOutAndWhy()
+ {
+ CoreWritingSystemDefinition ws = AddWritingSystemWithId("nl-Latn-SR", "nl-SR");
+ var multiText = new LfMultiText {
+ { "nl-Latn-SR", LfStringField.FromString("the export's value") },
+ { "nl-SR", LfStringField.FromString("") },
+ { "zzz-x-nonesuch", LfStringField.FromString("unplaceable") },
+ { "en", LfStringField.FromString("placed") },
+ { "fr", LfStringField.FromString("") },
+ };
+ var configuredTags = new HashSet(StringComparer.Ordinal) { "nl-SR", "en", "fr" };
+
+ var passedOver = multiText.KeysPassedOver(_cache.WritingSystemFactory, configuredTags);
+
+ Assert.That(passedOver, Is.EquivalentTo(new[] {
+ new KeyValuePair("nl-Latn-SR", "nl-SR"),
+ new KeyValuePair("zzz-x-nonesuch", null) }));
+ }
+
+ [Test]
+ public void SpanStrToTsString_NonCanonicalSpanLang_UsesTheCanonicalWritingSystem()
+ {
+ int wsFr = _cache.WritingSystemFactory.GetWsFromStr("fr");
+
+ var tss = ConvertMongoToLcmTsStrings.SpanStrToTsString(
+ "English français", _wsEn, _cache.WritingSystemFactory);
+
+ Assert.That(tss.RunCount, Is.EqualTo(2));
+ Assert.That(tss.get_WritingSystem(1), Is.EqualTo(wsFr));
+ }
+
+ ///
+ /// A span whose lang names no writing system LCM has used to make the string build throw,
+ /// abandoning the entry half-written. Its text is now kept in the string's own writing
+ /// system, and the lang reported.
+ ///
+ [Test]
+ public void SpanStrToTsString_SpanLangNamingNoWritingSystem_KeepsTheTextInTheMainWritingSystem()
+ {
+ var tss = ConvertMongoToLcmTsStrings.SpanStrToTsString(
+ "English unplaceable", _wsEn, _cache.WritingSystemFactory);
+
+ Assert.That(tss.Text, Is.EqualTo("English unplaceable"));
+ for (int run = 0; run < tss.RunCount; run++)
+ Assert.That(tss.get_WritingSystem(run), Is.EqualTo(_wsEn));
+ Assert.That(_env.Logger.GetMessages(), Does.Contain("span lang \"zzz-x-nonesuch\" names no writing system LCM has"));
+ }
+
+ [Test]
+ public void LfWsToLcmWs_NewTagInBothAnchors_IsAddedAsBothVernacularAndAnalysis()
+ {
+ // Setup: a writing system LCM does not have yet, configured for the lexeme and for the
+ // definition, as "en" so often is. Being vernacular must not stop it being analysis too,
+ // or the definitions written in it would have no analysis writing system to go to.
+ const string tag = "qaa-x-both";
+ Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag);
+ var restores = new[] {
+ AddLfInputSystem(tag),
+ AddToConfigField(tag, "lexeme"),
+ AddToConfigField(tag, "senses", "definition"),
+ };
+
+ try
+ {
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ ILangProject langProj = _cache.LanguageProject;
+ Assert.That(langProj.CurrentVernacularWritingSystems.Select(ws => ws.Id), Contains.Item(tag));
+ Assert.That(langProj.CurrentAnalysisWritingSystems.Select(ws => ws.Id), Contains.Item(tag));
+ }
+ finally
+ {
+ foreach (Action restore in restores)
+ restore();
+ }
+ }
+
+ ///
+ /// An input system LF spells non-canonically, while the config's fields spell it
+ /// canonically: the lexeme field says "qaa-x-fresh" for the input system "qaa-x-qaa-fresh". The
+ /// writing system is created under the canonical form, and must still get the vernacular
+ /// role the lexeme field gives it; looked up under the input system's spelling only, it
+ /// was not found, and was made analysis.
+ ///
+ [Test]
+ public void LfWsToLcmWs_NewInputSystemSpelledOtherwiseThanInTheConfig_GetsTheConfigsRole()
+ {
+ const string inputSystemTag = "qaa-x-qaa-fresh", configTag = "qaa-x-fresh";
+ Assert.That(LcmWritingSystemIds(), Does.Not.Contain(configTag), "testlangproj should not contain {0}", configTag);
+ var restores = new[] {
+ AddLfInputSystem(inputSystemTag),
+ AddToConfigField(configTag, "lexeme"),
+ };
+
+ try
+ {
+ // Exercise
+ SutMongoToLcm.Run(_lfProj);
+
+ // Verify
+ ILangProject langProj = _cache.LanguageProject;
+ Assert.That(LcmWritingSystemIds(), Contains.Item(configTag));
+ Assert.That(langProj.CurrentVernacularWritingSystems.Select(ws => ws.Id), Contains.Item(configTag));
+ Assert.That(langProj.CurrentAnalysisWritingSystems.Select(ws => ws.Id), Does.Not.Contain(configTag));
+ }
+ finally
+ {
+ foreach (Action restore in restores)
+ restore();
+ }
+ }
+
+ ///
+ /// Classifying a writing system counts the text in the whole lexicon, a second pass over
+ /// Mongo. Only a writing system new to LCM is classified, so with none the lexicon is read
+ /// once, for the entries themselves.
+ ///
+ [Test]
+ public void MongoToLcm_NoInputSystemNewToLcm_ReadsTheLexiconOnce()
+ {
+ int readsBefore = _conn.LexiconReads;
+
+ SutMongoToLcm.Run(_lfProj);
+
+ Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(1));
+ }
+
+ ///
+ /// With writing systems new to LCM, the lexicon is counted once more, and only once however
+ /// many there are: the classification is worked out for the first and kept for the rest.
+ ///
+ [Test]
+ public void MongoToLcm_InputSystemsNewToLcm_CountTheLexiconOnceMore()
+ {
+ var tags = new[] { "qaa-x-new", "qaa-x-newtoo" };
+ var restores = new List();
+ foreach (string tag in tags)
+ {
+ Assert.That(LcmWritingSystemIds(), Does.Not.Contain(tag), "testlangproj should not contain {0}", tag);
+ restores.Add(AddLfInputSystem(tag));
+ }
+ try
+ {
+ int readsBefore = _conn.LexiconReads;
+
+ SutMongoToLcm.Run(_lfProj);
+
+ Assert.That(LcmWritingSystemIds(), Is.SupersetOf(tags), "precondition: both should have been created");
+ Assert.That(_conn.LexiconReads - readsBefore, Is.EqualTo(2));
+ }
+ finally
+ {
+ foreach (Action restore in restores)
+ restore();
+ }
+ }
+ }
+}
diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs
new file mode 100644
index 00000000..d5da98e1
--- /dev/null
+++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/LfWritingSystemUsageTests.cs
@@ -0,0 +1,150 @@
+// Copyright (c) 2016-2018 SIL International
+// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
+
+using System.Collections.Generic;
+using LfMerge.Core.DataConverters;
+using LfMerge.Core.LanguageForge.Model;
+using NUnit.Framework;
+
+namespace LfMerge.Core.Tests.Lcm.DataConverters
+{
+ ///
+ /// Counting the text a writing system actually holds, field by field. This is a pure function
+ /// over the Language Forge model, so the fixture needs neither Mongo nor an LCM project. It
+ /// must not call MongoConnectionDouble.Initialize, which throws once another fixture in this
+ /// assembly has registered the conventions.
+ ///
+ public class LfWritingSystemUsageTests
+ {
+ private static LfMultiText Text(params string[] tagThenValue)
+ {
+ var result = new LfMultiText();
+ for (int i = 0; i < tagThenValue.Length; i += 2)
+ result[tagThenValue[i]] = new LfStringField { Value = tagThenValue[i + 1] };
+ return result;
+ }
+
+ private static LfLexEntry Entry(LfMultiText lexeme, params LfSense[] senses)
+ {
+ return new LfLexEntry { Lexeme = lexeme, Senses = new List(senses) };
+ }
+
+ private static LfSense Sense(LfMultiText gloss, params LfExample[] examples)
+ {
+ return new LfSense { Gloss = gloss, Examples = new List(examples) };
+ }
+
+ [Test]
+ public void FromLexicon_CountsOneEntryPerNonEmptyAlternative()
+ {
+ var lexicon = new[] {
+ Entry(Text("qaa-x-kal", "first", "en", "one")),
+ Entry(Text("qaa-x-kal", "second")),
+ };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.EntriesCounted, Is.EqualTo(2));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(2));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(1));
+ }
+
+ [Test]
+ public void FromLexicon_DoesNotCountEmptyOrWhitespaceOrMissingText()
+ {
+ // A field Language Forge offers but nobody has filled in is exactly what the counts are
+ // meant to tell apart from a field that is really in use.
+ var lexicon = new[] {
+ Entry(Text("qaa-x-kal", "", "en", " ", "fr", "real")),
+ new LfLexEntry { Lexeme = null },
+ };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.EntriesCounted, Is.EqualTo(2));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(0));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(0));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "fr"), Is.EqualTo(1));
+ }
+
+ [Test]
+ public void FromLexicon_SkipsDeletedEntries()
+ {
+ // Deleted entries are tombstones kept so the deletion can be replayed; their text is
+ // not evidence that anyone is using that writing system.
+ var lexicon = new[] {
+ Entry(Text("qaa-x-kal", "kept")),
+ new LfLexEntry { Lexeme = Text("qaa-x-kal", "gone"), IsDeleted = true },
+ };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.EntriesCounted, Is.EqualTo(1));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-kal"), Is.EqualTo(1));
+ }
+
+ [Test]
+ public void FromLexicon_CountsNestedSenseAndExampleFields()
+ {
+ var lexicon = new[] {
+ Entry(Text("kal", "word"),
+ Sense(Text("en", "meaning"),
+ new LfExample { Sentence = Text("kal", "a sentence"), Translation = Text("en", "a translation") },
+ new LfExample { Sentence = Text("kal", "another") })),
+ };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Gloss, "en"), Is.EqualTo(1));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.ExampleSentence, "kal"), Is.EqualTo(2));
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.ExampleTranslation, "en"), Is.EqualTo(1));
+ }
+
+ [Test]
+ public void NonEmptyCount_SumsOverSeveralFields()
+ {
+ var lexicon = new[] {
+ Entry(Text("kal", "word"),
+ Sense(null, new LfExample { Sentence = Text("kal", "a sentence") })),
+ };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.NonEmptyCount(
+ new[] { LfWritingSystemUsage.Lexeme, LfWritingSystemUsage.ExampleSentence }, "kal"),
+ Is.EqualTo(2));
+ }
+
+ [Test]
+ public void NonEmptyCount_MatchesTagsCaseInsensitively()
+ {
+ // Every LCM writing system lookup is case-insensitive, and the config and the lexicon do
+ // not always agree on the case of a tag.
+ var lexicon = new[] { Entry(Text("qaa-x-IPA-dupl1", "text")) };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.NonEmptyCount(LfWritingSystemUsage.Lexeme, "qaa-x-ipa-dupl1"), Is.EqualTo(1));
+ }
+
+ [Test]
+ public void FromLexicon_ToleratesAnEmptyOrNullLexicon()
+ {
+ Assert.That(LfWritingSystemUsage.FromLexicon(null).EntriesCounted, Is.EqualTo(0));
+ Assert.That(LfWritingSystemUsage.FromLexicon(new LfLexEntry[0]).EntriesCounted, Is.EqualTo(0));
+ Assert.That(LfWritingSystemUsage.FromLexicon(new LfLexEntry[0])
+ .NonEmptyCount(LfWritingSystemUsage.Lexeme, "en"), Is.EqualTo(0));
+ }
+
+ [Test]
+ public void TagsWithText_ListsOnlyTheWritingSystemsThatHoldText()
+ {
+ var lexicon = new[] { Entry(Text("kal", "word", "en", "", "fr", "mot")) };
+
+ LfWritingSystemUsage usage = LfWritingSystemUsage.FromLexicon(lexicon);
+
+ Assert.That(usage.TagsWithText(LfWritingSystemUsage.Lexeme), Is.EquivalentTo(new[] { "kal", "fr" }));
+ Assert.That(usage.TagsWithText(LfWritingSystemUsage.Gloss), Is.Empty);
+ }
+ }
+}
diff --git a/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs
new file mode 100644
index 00000000..3494b1a0
--- /dev/null
+++ b/src/LfMerge.Core.Tests/Lcm/DataConverters/VernacularClassificationTests.cs
@@ -0,0 +1,790 @@
+using System;
+using System.Collections.Generic;
+using LfMerge.Core.DataConverters;
+using LfMerge.Core.LanguageForge.Config;
+using LfMerge.Core.LanguageForge.Model;
+using NUnit.Framework;
+
+namespace LfMerge.Core.Tests.Lcm.DataConverters
+{
+ ///
+ /// LF has no vernacular/analysis flag on a writing system: which ones are vernacular is derived
+ /// from the project's own config, not from languageCode. Configs are built by hand here, since
+ /// this is a pure function needing no Mongo and no LCM project. The fixture must not call
+ /// MongoConnectionDouble.Initialize -- it throws if another fixture in this assembly has already
+ /// registered the conventions, and nothing here needs it.
+ ///
+ /// Most cases are named after a real project in the 2026-07-06 corpus that has that shape.
+ ///
+ public class VernacularClassificationTests
+ {
+ private static LfConfigMultiText Field(params string[] inputSystems)
+ {
+ return new LfConfigMultiText { InputSystems = new List(inputSystems) };
+ }
+
+ ///
+ /// Builds a config from dotted paths spelled as Mongo spells them, so nesting goes through
+ /// "fields" ("senses.fields.examples.fields.sentence").
+ ///
+ private static LfProjectConfig Config(params (string Path, LfConfigFieldBase Field)[] fields)
+ {
+ var entry = new LfConfigFieldList();
+ var senses = new LfConfigFieldList();
+ var examples = new LfConfigFieldList();
+ foreach ((string path, LfConfigFieldBase field) in fields)
+ {
+ if (path.StartsWith("senses.fields.examples.fields."))
+ examples.Fields[path.Substring("senses.fields.examples.fields.".Length)] = field;
+ else if (path.StartsWith("senses.fields."))
+ senses.Fields[path.Substring("senses.fields.".Length)] = field;
+ else
+ entry.Fields[path] = field;
+ }
+ if (examples.Fields.Count > 0) senses.Fields["examples"] = examples;
+ if (senses.Fields.Count > 0) entry.Fields["senses"] = senses;
+ return new LfProjectConfig { Entry = entry };
+ }
+
+ ///
+ /// One entry holding text at the given field path, so a lexicon of them gives a writing
+ /// system a known amount of text in a field of known role.
+ ///
+ private static LfLexEntry EntryWithTextAt(string fieldPath, string tag)
+ {
+ var text = new LfMultiText { { tag, LfStringField.FromString("text") } };
+ var entry = new LfLexEntry();
+ switch (fieldPath)
+ {
+ case LfWritingSystemUsage.Lexeme: entry.Lexeme = text; break;
+ case LfWritingSystemUsage.CitationForm: entry.CitationForm = text; break;
+ case LfWritingSystemUsage.Definition:
+ entry.Senses = new List { new LfSense { Definition = text } }; break;
+ case LfWritingSystemUsage.Gloss:
+ entry.Senses = new List { new LfSense { Gloss = text } }; break;
+ case LfWritingSystemUsage.ExampleSentence:
+ entry.Senses = new List {
+ new LfSense { Examples = new List { new LfExample { Sentence = text } } } };
+ break;
+ case LfWritingSystemUsage.ExampleTranslation:
+ entry.Senses = new List {
+ new LfSense { Examples = new List { new LfExample { Translation = text } } } };
+ break;
+ case LfWritingSystemUsage.Note: entry.Note = text; break;
+ case LfWritingSystemUsage.LiteralMeaning: entry.LiteralMeaning = text; break;
+ case LfWritingSystemUsage.Pronunciation: entry.Pronunciation = text; break;
+ case "etymology": entry.Etymology = text; break;
+ default: throw new ArgumentException("no test entry shape for " + fieldPath);
+ }
+ return entry;
+ }
+
+ private static LfWritingSystemUsage Usage(params (string Path, string Tag, int Count)[] spec)
+ {
+ var lexicon = new List();
+ foreach ((string path, string tag, int count) in spec)
+ for (int i = 0; i < count; i++)
+ lexicon.Add(EntryWithTextAt(path, tag));
+ return LfWritingSystemUsage.FromLexicon(lexicon);
+ }
+
+ ///
+ /// A null or empty tag in a config field's input systems names no writing system. The
+ /// 2026-10-01 corpus has none, but a null one made counting the text throw, which would
+ /// have ended the whole transfer.
+ ///
+ [Test]
+ public void ANullOrEmptyTagInTheConfigIsIgnored()
+ {
+ var config = Config(("lexeme", Field("kal", null)), ("senses.fields.gloss", Field("en", "")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" }));
+ }
+
+ [Test]
+ public void LexemeAndCitationFormAreAlwaysVernacular()
+ {
+ var config = Config(("lexeme", Field("seh")), ("citationForm", Field("seh-fonipa-x-etic")),
+ ("senses.fields.definition", Field("en", "pt")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" }));
+ Assert.That(result.Unresolved, Is.Empty);
+ }
+
+ [Test]
+ public void UnionsLexemeAndCitationFormWithoutDuplicating()
+ {
+ var config = Config(
+ ("lexeme", Field("qaa-x-kal", "qaa-fonipa-x-kal")),
+ ("citationForm", Field("qaa-x-kal", "seh")),
+ ("senses.fields.definition", Field("en")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal");
+ Assert.That(result.Vernacular,
+ Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "seh" }));
+ }
+
+ [Test]
+ public void IgnoresTheAnalysisAnchorFields()
+ {
+ var config = Config(("lexeme", Field("qaa-x-kal")),
+ ("senses.fields.definition", Field("en", "fr")), ("senses.fields.gloss", Field("en")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal" }));
+ Assert.That(result.Vernacular, Does.Not.Contain("en"));
+ Assert.That(result.Vernacular, Does.Not.Contain("fr"));
+ }
+
+ ///
+ /// apltw2016, the case that started this: its lexeme field uses a phonetic writing system
+ /// alongside the project language, and the rule this replaces -- vernacular is whatever
+ /// equals languageCode -- filed that phonetic one as analysis.
+ ///
+ [Test]
+ public void TreatsASecondLexemeWritingSystemAsVernacular()
+ {
+ var config = Config(("lexeme", Field("qaa-x-IPA-dupl1", "th")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th");
+ Assert.That(result.Vernacular, Contains.Item("th"));
+ Assert.That(result.Vernacular, Contains.Item("qaa-x-IPA-dupl1"),
+ "a writing system the lexeme field uses is vernacular even though it is not the language code");
+ }
+
+ [Test]
+ public void MatchesTagsCaseInsensitively()
+ {
+ var config = Config(("lexeme", Field("qaa-x-IPA-dupl1")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th");
+ Assert.That(result.Vernacular.Contains("qaa-x-ipa-dupl1"), Is.True,
+ "LCM's own writing system lookups are case-insensitive, so this should be too");
+ }
+
+ /// The 741-project shape: etymology written in the analysis language.
+ [Test]
+ public void EtymologyInAnAnalysisWritingSystemStaysAnalysis()
+ {
+ var config = Config(("lexeme", Field("yog")), ("senses.fields.gloss", Field("en")),
+ ("etymology", Field("en")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "yog");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "yog" }));
+ Assert.That(result.Vernacular, Does.Not.Contain("en"));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" }));
+ }
+
+ /// The 378-project shape, e.g. test-india-sena-01.
+ [Test]
+ public void EtymologyInTheVernacularIsVernacular()
+ {
+ var config = Config(("lexeme", Field("seh", "seh-fonipa-x-etic")),
+ ("senses.fields.gloss", Field("en", "pt")),
+ ("etymology", Field("seh", "seh-fonipa-x-etic")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "seh-fonipa-x-etic" }));
+ }
+
+ ///
+ /// A vernacular field is evidence about the field, not about every writing system in it:
+ /// Portuguese is exclusive to the analysis anchors, so an etymology that also carries the
+ /// vernacular must not make it vernacular -- a new Portuguese writing system would then be
+ /// created vernacular-only, and the glosses in it would have no analysis writing system.
+ ///
+ [Test]
+ public void AVernacularFieldDoesNotMakeAnAnalysisOnlyWritingSystemVernacular()
+ {
+ var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("en", "pt")),
+ ("etymology", Field("seh", "pt")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "pt" }));
+ }
+
+ ///
+ /// A field offering a vernacular-only and an analysis-only writing system side by side says
+ /// nothing about the role of the others it offers. It used to count as vernacular, because
+ /// that was checked first, which made kam-flex's Spanish and French vernacular; FLEx has
+ /// both as analysis.
+ ///
+ [Test]
+ public void AFieldOfferingBothRolesIsNotEvidenceOfEither()
+ {
+ var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("en")),
+ ("customField_senses_Wordlist", Field("seh", "en", "fr")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh");
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "fr" }));
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "seh", "fr" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "fr" }));
+ }
+
+ ///
+ /// grc-vie-flex: etymologies in Hebrew and Aramaic, neither vernacular nor analysis. With no
+ /// evidence either way they are both, so neither a vernacular-typed nor an analysis-typed
+ /// field is left without them.
+ ///
+ [Test]
+ public void WritingSystemsInNeitherAnchorAreBothRolesAndReported()
+ {
+ var config = Config(("lexeme", Field("grc")), ("senses.fields.gloss", Field("en", "vi")),
+ ("etymology", Field("hbo", "arc")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "grc");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "grc", "hbo", "arc" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "vi", "hbo", "arc" }));
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo", "arc" }));
+ }
+
+ ///
+ /// A French-only notes field overlaps neither anchor. French must reach the analysis list, or
+ /// an analysis-typed FieldWorks custom field would not offer it.
+ ///
+ [Test]
+ public void AnUnresolvedCustomFieldWritingSystemIsAnalysisToo()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("senses.fields.customField_senses_note", Field("fr")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal");
+ Assert.That(result.Analysis, Contains.Item("fr"));
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "fr" }));
+ }
+
+ ///
+ /// With no analysis anchor there is nothing to rule a field out, so every other field's
+ /// writing systems are unresolved -- and therefore both vernacular and analysis. Unless
+ /// their language has already been spoken for: qaa-fonipa-x-kal is a phonetic spelling of
+ /// the same language as the lexeme's qaa-x-kal, so it is vernacular like its relative,
+ /// while Hebrew keeps no such company and stays in doubt.
+ ///
+ [Test]
+ public void WithNoAnalysisAnchorEveryOtherFieldIsUnresolved()
+ {
+ var config = Config(("lexeme", Field("qaa-x-kal")),
+ ("etymology", Field("qaa-fonipa-x-kal", "hbo")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal");
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" }));
+ Assert.That(result.Vernacular,
+ Is.EquivalentTo(new[] { "qaa-x-kal", "qaa-fonipa-x-kal", "hbo" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "hbo" }));
+ }
+
+ /// flh-flex: languageCode names a writing system only etymology uses.
+ [Test]
+ public void LexemeWinsWhenLanguageCodeDisagrees()
+ {
+ var config = Config(("lexeme", Field("flh-x-cm")), ("senses.fields.gloss", Field("en", "id")),
+ ("etymology", Field("flh-x-ortho")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "flh-x-ortho");
+ Assert.That(result.Vernacular, Contains.Item("flh-x-cm"));
+ }
+
+ /// odo: languageCode "th" is not among the project's writing systems at all.
+ [Test]
+ public void LexemeIsVernacularEvenWhenLanguageCodeIsAbsent()
+ {
+ var config = Config(("lexeme", Field("en")),
+ ("senses.fields.definition", Field("qaa-Syrc-IQ-x-syr", "en", "acm")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "th");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en" }));
+ }
+
+ ///
+ /// A writing system in both anchors is no evidence of any other field's role, so the Hebrew
+ /// sharing the etymology with it is in doubt. "en" itself is not: the anchors placed it.
+ ///
+ [Test]
+ public void AWritingSystemInBothAnchorsIsNotEvidence()
+ {
+ var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("en")),
+ ("etymology", Field("en", "hbo")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "en", "hbo" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "hbo" }));
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" }),
+ "etymology overlaps neither anchor exclusively, so the writing system it alone places is in doubt");
+ }
+
+ ///
+ /// The "en" in lexeme and gloss alike: it is vernacular because the lexeme uses it, and it is
+ /// analysis because the gloss does. Being one must not stop it being the other.
+ ///
+ [Test]
+ public void AWritingSystemInBothAnchorsIsBothVernacularAndAnalysis()
+ {
+ var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal", "en" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" }));
+ }
+
+ ///
+ /// The example sentence and translation are anchors nested in a field list, so this covers
+ /// the recursion into one: the sentence is vernacular and the translation analysis.
+ ///
+ [Test]
+ public void TheNestedExampleAnchorsAreFound()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.definition", Field("fr")),
+ ("senses.fields.gloss", Field("en")),
+ ("senses.fields.examples.fields.sentence", Field("kal")),
+ ("senses.fields.examples.fields.translation", Field("en")));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" }));
+ Assert.That(result.Vernacular, Does.Not.Contain("en"),
+ "the example translation is not a vernacular field");
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en", "fr" }));
+ Assert.That(result.Unresolved, Is.Empty);
+ }
+
+ [Test]
+ public void ToleratesAPartiallyMissingNestedPath()
+ {
+ // senses exists but has no children, so the walk has to give up cleanly.
+ var config = Config(("lexeme", Field("qaa-x-kal")), ("senses", new LfConfigFieldList()));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "qaa-x-kal" }));
+ }
+
+ [Test]
+ public void FallsBackToLanguageCodeWhenTheConfigSaysNothing()
+ {
+ Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(null, "fr").Vernacular,
+ Is.EquivalentTo(new[] { "fr" }));
+ Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(new LfProjectConfig(), "fr").Vernacular,
+ Is.EquivalentTo(new[] { "fr" }));
+ }
+
+ [Test]
+ public void FallsBackToLanguageCodeWhenLexemeListsNoInputSystems()
+ {
+ var config = Config(("lexeme", Field()));
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "fr");
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "fr" }));
+ }
+
+ ///
+ /// The case that motivates counting at all: the config offers "en" for the lexeme field, so
+ /// the config alone calls it vernacular, but no headword is written in it.
+ ///
+ [Test]
+ public void AWritingSystemConfiguredForTheLexemeButUnusedThereIsNotVernacular()
+ {
+ var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 100),
+ (LfWritingSystemUsage.Gloss, "en", 100));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" }));
+ Assert.That(result.Vernacular, Does.Not.Contain("en"),
+ "nothing is written in en in the lexeme field, so it is no evidence of vernacular");
+ }
+
+ ///
+ /// brb-flex-2022: the phonetic writing system has 4,030 headwords and one stray string in an
+ /// example reference, and FieldWorks has it as vernacular only.
+ ///
+ [Test]
+ public void AFewStrayStringsDoNotMakeAVernacularWritingSystemAnalysisAsWell()
+ {
+ var config = Config(("lexeme", Field("seh-fonipa-x-etic")), ("senses.fields.gloss", Field("en", "seh-fonipa-x-etic")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "seh-fonipa-x-etic", 4030),
+ (LfWritingSystemUsage.Gloss, "seh-fonipa-x-etic", 1),
+ (LfWritingSystemUsage.Gloss, "en", 4000));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh", usage);
+
+ Assert.That(result.Vernacular, Contains.Item("seh-fonipa-x-etic"));
+ Assert.That(result.Analysis, Does.Not.Contain("seh-fonipa-x-etic"));
+ Assert.That(result.Analysis, Contains.Item("en"));
+ }
+
+ ///
+ /// brb-flex-2022 again: the vernacular is also glossed in, 72/28, and FieldWorks really does
+ /// list it in both.
+ ///
+ [Test]
+ public void AWritingSystemUsedSubstantiallyForBothRolesGetsBoth()
+ {
+ var config = Config(("lexeme", Field("brb")), ("senses.fields.gloss", Field("brb", "en")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "brb", 7256),
+ (LfWritingSystemUsage.Gloss, "brb", 2801));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "brb", usage);
+
+ Assert.That(result.Vernacular, Contains.Item("brb"));
+ Assert.That(result.Analysis, Contains.Item("brb"));
+ Assert.That(result.Unresolved, Is.Empty, "28% is a real second role, not an unknown one");
+ }
+
+ [Test]
+ public void TextOutweighsTheConfigInBothDirections()
+ {
+ // Configured the wrong way round: the lexeme offers "en" and the gloss offers "kal",
+ // but every headword is in kal and every gloss in en.
+ var config = Config(("lexeme", Field("en")), ("senses.fields.gloss", Field("kal")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 50),
+ (LfWritingSystemUsage.Gloss, "en", 50));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "en", usage);
+
+ Assert.That(result.Vernacular, Is.EquivalentTo(new[] { "kal" }));
+ Assert.That(result.Analysis, Is.EquivalentTo(new[] { "en" }));
+ }
+
+ [Test]
+ public void AWritingSystemUsedOnlyInAFieldOfUnknownRoleIsStillUnresolved()
+ {
+ // Text in the etymology says nothing: the corpus has 33 projects writing etymologies in
+ // the vernacular and 50 in an analysis language.
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("etymology", Field("hbo")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10), ("etymology", "hbo", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "hbo" }));
+ }
+
+ [Test]
+ public void AWritingSystemTheConfigForgotIsClassifiedFromItsTextAnyway()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Lexeme, "kal-fonipa", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Vernacular, Contains.Item("kal-fonipa"));
+ }
+
+ [Test]
+ public void AnEmptyLexiconLeavesTheConfigInCharge()
+ {
+ var config = Config(("lexeme", Field("kal", "en")), ("senses.fields.gloss", Field("en")));
+
+ var withoutData = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal",
+ LfWritingSystemUsage.FromLexicon(new LfLexEntry[0]));
+ var configOnly = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal");
+
+ Assert.That(withoutData.Vernacular, Is.EquivalentTo(configOnly.Vernacular));
+ Assert.That(withoutData.Analysis, Is.EquivalentTo(configOnly.Analysis));
+ Assert.That(withoutData.Vernacular, Contains.Item("en"));
+ }
+
+ ///
+ /// FieldWorks fixes the role of these fields, and the corpus agrees: counting the projects
+ /// whose text in each is exclusively one role, the example sentence runs 332 vernacular to 5
+ /// analysis, and the translation, entry note and literal meaning run the other way.
+ ///
+ [TestCase("senses.fields.examples.fields.sentence", true)]
+ [TestCase("senses.fields.examples.fields.translation", false)]
+ [TestCase("note", false)]
+ [TestCase("literalMeaning", false)]
+ public void TheFieldsFieldWorksFixesTheRoleOfAreEvidenceToo(string fieldPath, bool isVernacular)
+ {
+ // Only the field under test carries any text for "xyz", so nothing else can decide it.
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ (fieldPath, Field("xyz")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10), (fieldPath, "xyz", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Does.Not.Contain("xyz"));
+ if (isVernacular)
+ {
+ Assert.That(result.Vernacular, Contains.Item("xyz"));
+ Assert.That(result.Analysis, Does.Not.Contain("xyz"));
+ }
+ else
+ {
+ Assert.That(result.Analysis, Contains.Item("xyz"));
+ Assert.That(result.Vernacular, Does.Not.Contain("xyz"));
+ }
+ }
+
+ ///
+ /// The pronunciation field is not an anchor. It looks vernacular in the corpus at 114
+ /// projects to 16, but 12% is above the share a single writing system may hold in the other
+ /// role and still count as one thing, so projects evidently use it for more than one. Being
+ /// offered there, with nothing written in it, therefore settles nothing.
+ ///
+ [Test]
+ public void ThePronunciationFieldIsNotAnAnchor()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("pronunciation", Field("xyz")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Contains.Item("xyz"));
+ }
+
+ ///
+ /// But text in it makes a writing system vernacular, since FieldWorks draws its
+ /// pronunciation writing systems from the vernacular ones. xin-flex's "xin" holds 226
+ /// pronunciations and nothing else.
+ ///
+ [Test]
+ public void PronunciationTextMakesAWritingSystemVernacular()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("pronunciation", Field("xyz")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "xyz", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Does.Not.Contain("xyz"));
+ Assert.That(result.Vernacular, Contains.Item("xyz"));
+ Assert.That(result.Analysis, Does.Not.Contain("xyz"));
+ }
+
+ ///
+ /// A writing system offered only in a custom field, with pronunciation text and nothing
+ /// else, is vernacular only, by design. Without the pronunciation text the custom field
+ /// would say nothing about it and it would be both; with it, it counts as placed, so that
+ /// field cannot put it in doubt. In the 2026-10-01 corpus every writing system in that
+ /// position is a pronunciation writing system -- a phonetic, IPA or audio one -- for which
+ /// vernacular only is the right answer.
+ ///
+ /// The writing system's language is one the project has nowhere else: a phonetic spelling of
+ /// the vernacular, "kal-fonipa", would be made vernacular only by its language whether or not
+ /// it had pronunciation text, and the test would not show the rule at work.
+ ///
+ [TestCase(true)]
+ [TestCase(false)]
+ public void PronunciationTextAloneMakesAWritingSystemOfferedOnlyInACustomFieldVernacularOnly(bool hasPronunciationText)
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_senses_Phonetic", Field("xyz-fonipa")));
+ var counts = new List<(string, string, int)> {
+ (LfWritingSystemUsage.Lexeme, "kal", 10), (LfWritingSystemUsage.Gloss, "en", 10) };
+ if (hasPronunciationText)
+ counts.Add((LfWritingSystemUsage.Pronunciation, "xyz-fonipa", 1));
+ var usage = Usage(counts.ToArray());
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Vernacular, Contains.Item("xyz-fonipa"));
+ Assert.That(result.Analysis.Contains("xyz-fonipa"), Is.EqualTo(!hasPronunciationText),
+ "with pronunciation text, vernacular only; without, both");
+ Assert.That(result.Unresolved.Contains("xyz-fonipa"), Is.EqualTo(!hasPronunciationText));
+ }
+
+ ///
+ /// Pronunciation text only adds the vernacular role; it does not take away an analysis role
+ /// the config gives. In dtlat-flex a single pronunciation string in Swedish would otherwise
+ /// make Swedish, configured for the gloss, vernacular only.
+ ///
+ [Test]
+ public void PronunciationTextDoesNotTakeAwayAnAnalysisRole()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en", "sv")),
+ ("pronunciation", Field("kal", "sv")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10), (LfWritingSystemUsage.Pronunciation, "sv", 1));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Vernacular, Contains.Item("sv"));
+ Assert.That(result.Analysis, Contains.Item("sv"));
+ }
+
+ ///
+ /// A writing system offered both in a field that says nothing about its role and in one
+ /// whose company settles it is settled by the latter. The field of no role is listed first
+ /// on purpose: placing as each field came, it used to put the writing system in doubt.
+ ///
+ [TestCase("kal", true)]
+ [TestCase("en", false)]
+ public void AWritingSystemInDoubtIsSettledByTheCompanyItKeeps(string companion, bool isVernacular)
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Doubtful", Field("abc", "xyz")),
+ ("customField_entry_Settled", Field(companion, "abc")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Does.Not.Contain("abc"),
+ "abc shares a field with {0}, which is settled, so abc is settled too", companion);
+ Assert.That(result.Vernacular.Contains("abc"), Is.EqualTo(isVernacular));
+ Assert.That(result.Analysis.Contains("abc"), Is.EqualTo(!isVernacular));
+ // xyz is offered only in the field of no role, so the field loop puts it in doubt; only
+ // the company it keeps there, abc, settles it
+ Assert.That(result.Unresolved, Does.Not.Contain("xyz"), "xyz is settled by the company it keeps");
+ Assert.That(result.Vernacular.Contains("xyz"), Is.EqualTo(isVernacular));
+ Assert.That(result.Analysis.Contains("xyz"), Is.EqualTo(!isVernacular));
+ }
+
+ ///
+ /// The answer must not depend on the order the config lists its fields in. Here en-fonipa is
+ /// offered in a vernacular field and an analysis field, so it plays both roles. Placing as
+ /// each field came, a field of no role listed before them put it in doubt, and language
+ /// affinity with "en" then stripped its vernacular role; listed after them, it did not.
+ ///
+ [TestCase(true, TestName = "TheFieldOrderDoesNotChangeTheAnswerWithTheNoRoleFieldFirst")]
+ [TestCase(false, TestName = "TheFieldOrderDoesNotChangeTheAnswerWithTheNoRoleFieldLast")]
+ public void TheFieldOrderDoesNotChangeTheAnswer(bool noRoleFieldFirst)
+ {
+ var noRole = ("customField_entry_NoRole", (LfConfigFieldBase)Field("en-fonipa"));
+ var fields = new List<(string, LfConfigFieldBase)> {
+ ("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Vernacular", Field("kal", "en-fonipa")),
+ ("customField_entry_Analysis", Field("en", "en-fonipa")),
+ };
+ fields.Insert(noRoleFieldFirst ? 2 : fields.Count, noRole);
+ var config = Config(fields.ToArray());
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal");
+
+ Assert.That(result.Vernacular, Contains.Item("en-fonipa"));
+ Assert.That(result.Analysis, Contains.Item("en-fonipa"));
+ Assert.That(result.Unresolved, Does.Not.Contain("en-fonipa"));
+ }
+
+ [Test]
+ public void WritingSystemsWhoseOnlyCompanyIsEachOtherStayUnresolved()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Doubtful", Field("abc", "xyz")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "abc", "xyz" }));
+ }
+
+ ///
+ /// "en" is in both anchors, so it plays both roles. Turning up alone in a sense's general
+ /// note, a field that gives no evidence, must not put it in doubt: if it did, the company it keeps in the
+ /// lexeme field (the vernacular) would then strip it of its analysis role, and a new "en"
+ /// writing system would be created vernacular only.
+ ///
+ [Test]
+ public void AWritingSystemAlreadyPlacedIsNotPutInDoubtByAFieldWithNoEvidence()
+ {
+ var config = Config(("lexeme", Field("seh", "en")), ("senses.fields.gloss", Field("en")),
+ ("senses.fields.definition", Field("en")), ("senses.fields.generalNote", Field("en")));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh");
+
+ Assert.That(result.Unresolved, Is.Empty);
+ Assert.That(result.Vernacular, Contains.Item("en"));
+ Assert.That(result.Analysis, Contains.Item("en"));
+ }
+
+ [Test]
+ public void AWritingSystemWithNoCompanyAtAllStaysUnresolved()
+ {
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Note", Field("xyz")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "xyz" }));
+ }
+
+ ///
+ /// A writing system with nothing written in it and no company still has a language, and the
+ /// other writing systems of that language have already been placed.
+ ///
+ [TestCase("seh", "seh-fonipa-x-etic", true, TestName = "AffinityPhoneticVariantOfTheVernacular")]
+ [TestCase("qaa-x-kal", "qaa-Zxxx-x-kal-audio", true, TestName = "AffinityAudioVariantOfTheVernacular")]
+ [TestCase("qaa-x-kal", "qaa-x-kal-dupl1", true, TestName = "AffinityDuplicateOfTheVernacular")]
+ [TestCase("x-kal", "x-kal-dupl1", true, TestName = "AffinityDuplicateOfAWhollyPrivateUseVernacular")]
+ public void AWritingSystemOfTheSameLanguageAsTheVernacularIsVernacular(
+ string vernacularTag, string relative, bool isVernacular)
+ {
+ var config = Config(("lexeme", Field(vernacularTag)), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Extra", Field(relative)));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, vernacularTag, 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, vernacularTag, usage);
+
+ Assert.That(result.Unresolved, Is.Empty);
+ Assert.That(result.Vernacular.Contains(relative), Is.EqualTo(isVernacular));
+ Assert.That(result.Analysis.Contains(relative), Is.EqualTo(!isVernacular));
+ }
+
+ [Test]
+ public void AWritingSystemOfTheSameLanguageAsTheAnalysisLanguageIsAnalysis()
+ {
+ // Script and region are not the language: "en-GB" is English.
+ var config = Config(("lexeme", Field("kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Extra", Field("en-GB")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "kal", usage);
+
+ Assert.That(result.Unresolved, Is.Empty);
+ Assert.That(result.Analysis, Contains.Item("en-GB"));
+ Assert.That(result.Vernacular, Does.Not.Contain("en-GB"));
+ }
+
+ ///
+ /// Under "qaa" the private-use subtags are the language, so two of them are two languages.
+ ///
+ [Test]
+ public void ADifferentPrivateUseLanguageIsNotTheSameLanguage()
+ {
+ var config = Config(("lexeme", Field("qaa-x-kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Extra", Field("qaa-x-hbo")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "qaa-x-kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "qaa-x-kal", usage);
+
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "qaa-x-hbo" }));
+ }
+
+ ///
+ /// A tag can be private-use from its first subtag (the corpus has "x-IPA" and "x-v"), and
+ /// then too the private-use subtags are the language.
+ ///
+ [Test]
+ public void AWhollyPrivateUseTagIsNotTheSameLanguageAsAnother()
+ {
+ var config = Config(("lexeme", Field("x-kal")), ("senses.fields.gloss", Field("en")),
+ ("customField_entry_Extra", Field("x-hbo")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "x-kal", 10),
+ (LfWritingSystemUsage.Gloss, "en", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "x-kal", usage);
+
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "x-hbo" }));
+ }
+
+ [Test]
+ public void ALanguageUsedForBothRolesSettlesNothing()
+ {
+ // seh is the vernacular and seh-fonipa is glossed in, so the language says nothing.
+ var config = Config(("lexeme", Field("seh")), ("senses.fields.gloss", Field("seh-fonipa")),
+ ("customField_entry_Extra", Field("seh-Latn")));
+ var usage = Usage((LfWritingSystemUsage.Lexeme, "seh", 10),
+ (LfWritingSystemUsage.Gloss, "seh-fonipa", 10));
+
+ var result = ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(config, "seh", usage);
+
+ Assert.That(result.Unresolved, Is.EquivalentTo(new[] { "seh-Latn" }));
+ }
+
+ [Test]
+ public void ReturnsNothingWhenThereIsNoConfigAndNoLanguageCode()
+ {
+ Assert.That(ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems(null, null).Vernacular,
+ Is.Empty);
+ }
+ }
+}
diff --git a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs
index 3a7d8b55..1f425769 100644
--- a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs
+++ b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016 SIL International
+// Copyright (c) 2016 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
@@ -58,18 +58,14 @@ public void Action_IsInitialClone_ShouldPopulateMongoInputSystems()
// Verify
const int expectedNumVernacularWS = 3;
const int expectedNumAnalysisWS = 2;
- // TODO: Investigate why qaa-Zxxx-x-kal-audio is in CurrentVernacularWritingSystems, but somehow in
- // UpdateMongoDbFromLcm.LcmWsToLfWs() is not contained in _cache.LangProject.CurrentVernacularWritingSystems
- const string notVernacularWs = "qaa-Zxxx-x-kal-audio";
lfWsList = _conn.GetInputSystems(lfProject);
var languageProj = lfProject.FieldWorksProject.Cache.LangProject;
+ // Every one, qaa-Zxxx-x-kal-audio included: the list spells it qaa-Zxxx-x-kal-AUDIO,
+ // which the list's own Contains, comparing Ids exactly, did not match
foreach (var lcmVernacularWs in languageProj.CurrentVernacularWritingSystems)
- {
- if (lcmVernacularWs.Id != notVernacularWs)
- Assert.That(lfWsList[lcmVernacularWs.Id].VernacularWS);
- }
+ Assert.That(lfWsList[lcmVernacularWs.Id].VernacularWS, "{0} should be flagged vernacular", lcmVernacularWs.Id);
Assert.That(languageProj.CurrentVernacularWritingSystems.Count, Is.EqualTo(expectedNumVernacularWS));
foreach (var lcmAnalysisWs in languageProj.CurrentAnalysisWritingSystems)
diff --git a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs
index 0013eca5..86dc7aa1 100644
--- a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs
+++ b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs
@@ -745,6 +745,39 @@ public void Action_NeverSyncedProjectWhoseEntriesLfMergeWrote_KeepsFieldWorksVer
Assert.That(_counts.Modified, Is.EqualTo(0));
}
+ [Test]
+ public void Action_NeverSyncedProjectWithEntryImportedIntoLf_SendsItToFieldWorks()
+ {
+ // Setup: an LF project's first Send/Receive, as when one is migrated. A LIFT import into
+ // LF leaves no user references on its entries, the same as LfMerge's own writes do, so
+ // the entry is stored with UpdateRecord rather than as an LF user's edit.
+ var lfProj = _lfProj;
+ _conn.SetLastSyncedDate(lfProj, MagicValues.UnixEpoch);
+ FwProject fwProject = lfProj.FieldWorksProject;
+ string vernacularWS = fwProject.Cache.LanguageProject.DefaultVernacularWritingSystem.Id;
+ Guid importedGuid = Guid.NewGuid();
+ var imported = new LfLexEntry();
+ imported.Guid = importedGuid;
+ imported.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, "imported into LF");
+ imported.AuthorInfo = new LfAuthorInfo();
+ imported.AuthorInfo.CreatedDate = new DateTime(2016, 4, 10, 0, 0, 0, DateTimeKind.Utc);
+ imported.AuthorInfo.ModifiedDate = imported.AuthorInfo.CreatedDate;
+ imported.DateModified = new DateTime(2016, 7, 25, 0, 0, 0, DateTimeKind.Utc);
+ _conn.UpdateRecord(lfProj, imported);
+
+ // Exercise
+ SutMongoToLcm.Run(lfProj);
+
+ // Verify
+ ILexEntry sent;
+ Assert.That(fwProject.ServiceLocator.GetInstance().TryGetObject(importedGuid, out sent), Is.True,
+ "an entry only LF has must reach FieldWorks, or the transfer back to Mongo marks it deleted");
+ Assert.That(sent.LexemeFormOA.Form.VernacularDefaultWritingSystem.Text, Is.EqualTo("imported into LF"));
+ Assert.That(_counts.Added, Is.EqualTo(1));
+ Assert.That(_counts.Modified, Is.EqualTo(0));
+ Assert.That(_counts.Deleted, Is.EqualTo(0));
+ }
+
// TODO: Move custom field tests to their own test class, and move these helper functions with them
public void AddCustomFieldToSense(LfSense sense, string fieldName, BsonValue fieldValue)
{
diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs
index e8667743..9584c5e2 100644
--- a/src/LfMerge.Core.Tests/TestDoubles.cs
+++ b/src/LfMerge.Core.Tests/TestDoubles.cs
@@ -116,7 +116,9 @@ public static void Initialize()
}
private readonly Dictionary _storedInputSystems = new Dictionary();
- private readonly Dictionary _storedLfLexEntries = new Dictionary();
+ // Held as documents, as Mongo holds them: an entry LF wrote keeps what LfMerge's own
+ // serializer would leave out, such as a multitext holding an empty value
+ private readonly Dictionary _storedLfLexEntries = new Dictionary();
private readonly Dictionary _storedLfOptionLists = new Dictionary();
private Dictionary _storedCustomFieldConfig = new Dictionary();
private Dictionary _storedLastSyncDate = new Dictionary();
@@ -154,11 +156,19 @@ public Dictionary GetInputSystems(ILfProject projec
return _storedInputSystems;
}
+ /// The current vernacular, analysis and pronunciation lists the last export gave.
+ public List LastVernacularWss { get; private set; }
+ public List LastAnalysisWss { get; private set; }
+ public List LastPronunciationWss { get; private set; }
+
public bool SetInputSystems(ILfProject project, Dictionary inputSystems,
List vernacularWss, List analysisWss, List pronunciationWss)
{
foreach (var ws in inputSystems.Keys)
_storedInputSystems[ws] = inputSystems[ws];
+ LastVernacularWss = vernacularWss;
+ LastAnalysisWss = analysisWss;
+ LastPronunciationWss = pronunciationWss;
if (project.IsInitialClone)
{
@@ -182,30 +192,36 @@ public Dictionary GetCustomFieldConfig(ILfProject pro
return _storedCustomFieldConfig;
}
- public void UpdateMockLfLexEntry(BsonDocument mockData)
- {
- LfLexEntry data = BsonSerializer.Deserialize(mockData);
- UpdateMockLfLexEntry(data);
- }
-
// The user SampleData's entries were last modified by
private static readonly ObjectId LfUserRef = ObjectId.Parse("561b666c0f87096a35c3cf2d");
///
/// Store an entry the way an LF user's edit does: LF's MapperModel::write stamps DateModified,
/// and LexEntryCommands::updateEntry records who made the edit. LfMerge's own writes go
- /// through UpdateRecord, which does neither.
+ /// through UpdateRecord, which does neither. The document is stored as given, empty values
+ /// and all, since LF writes what its editor sends.
///
- public void UpdateMockLfLexEntry(LfLexEntry mockData)
+ public void UpdateMockLfLexEntry(BsonDocument mockData)
{
- Guid guid = mockData.Guid ?? Guid.Empty;
- var stored = DeepCopy(mockData);
- stored.DateModified = DateTime.UtcNow;
- if (stored.AuthorInfo != null && stored.AuthorInfo.ModifiedByUserRef == null)
- stored.AuthorInfo.ModifiedByUserRef = LfUserRef;
+ var stored = (BsonDocument)mockData.DeepClone();
+ Guid guid = BsonSerializer.Deserialize(stored).Guid ?? Guid.Empty;
+ stored["dateModified"] = new BsonDateTime(DateTime.UtcNow);
+ BsonValue authorInfo;
+ if (stored.TryGetValue("authorInfo", out authorInfo) && authorInfo.IsBsonDocument)
+ {
+ BsonValue userRef;
+ if (!authorInfo.AsBsonDocument.TryGetValue("modifiedByUserRef", out userRef) || userRef.IsBsonNull)
+ authorInfo.AsBsonDocument["modifiedByUserRef"] = LfUserRef;
+ }
_storedLfLexEntries[guid] = stored;
}
+ /// As UpdateMockLfLexEntry(BsonDocument), for an entry built as an object.
+ public void UpdateMockLfLexEntry(LfLexEntry mockData)
+ {
+ UpdateMockLfLexEntry(mockData.ToBsonDocument());
+ }
+
public void UpdateMockOptionList(BsonDocument mockData)
{
LfOptionList data = BsonSerializer.Deserialize(mockData);
@@ -220,14 +236,14 @@ public void UpdateMockOptionList(LfOptionList mockData)
public IEnumerable GetLfLexEntries()
{
- return new List(_storedLfLexEntries.Values.Select(entry => DeepCopy(entry)));
+ return new List(_storedLfLexEntries.Values.Select(entry => BsonSerializer.Deserialize(entry)));
}
public LfLexEntry GetLfLexEntryByGuid(ILfProject _project, Guid key)
{
- LfLexEntry result;
+ BsonDocument result;
if (_storedLfLexEntries.TryGetValue(key, out result))
- return result;
+ return BsonSerializer.Deserialize(result);
return null;
}
@@ -238,10 +254,26 @@ public IEnumerable GetLfOptionLists()
public IEnumerable GetRecords(ILfProject project, string collectionName, Expression> filter)
{
- return GetRecords(project, collectionName).Where(filter.Compile());
+ // A filtered request finds a few records, not a pass over the collection, so it is not
+ // counted in LexiconReads
+ return Records(collectionName).Where(filter.Compile());
}
+ ///
+ /// How many times the whole lexicon has been asked for, each request being a pass over
+ /// Mongo. Filtered requests, such as the comment converter's one per comment, are not
+ /// counted.
+ ///
+ public int LexiconReads { get; private set; }
+
public IEnumerable GetRecords(ILfProject project, string collectionName)
+ {
+ if (collectionName == MagicStrings.LfCollectionNameForLexicon)
+ LexiconReads++;
+ return Records(collectionName);
+ }
+
+ private IEnumerable Records(string collectionName)
{
switch (collectionName)
{
@@ -257,7 +289,8 @@ public IEnumerable GetRecords(ILfProject project, string c
public Dictionary GetAllModifiedDatesForEntries(ILfProject project)
{
- return _storedLfLexEntries.ToDictionary(kv => kv.Key, kv => kv.Value.AuthorInfo.ModifiedDate);
+ return _storedLfLexEntries.ToDictionary(kv => kv.Key,
+ kv => BsonSerializer.Deserialize(kv.Value).AuthorInfo.ModifiedDate);
}
public LfOptionList GetLfOptionListByCode(ILfProject project, string listCode)
@@ -284,7 +317,7 @@ public IMongoDatabase GetMainDatabase()
public bool UpdateRecord(ILfProject project, LfLexEntry data)
{
- _storedLfLexEntries[data.Guid ?? Guid.Empty] = DeepCopy(data);
+ _storedLfLexEntries[data.Guid ?? Guid.Empty] = data.ToBsonDocument();
return true;
}
@@ -302,7 +335,7 @@ public bool RemoveRecord(ILfProject project, Guid guid)
public bool SetLastSyncedDate(ILfProject project, DateTime? newSyncedDate)
{
- // Mongo keeps dates to the millisecond, as DeepCopy does for the entries' dates
+ // Mongo keeps dates to the millisecond, as the entries' dates are kept here in BSON
if (newSyncedDate != null)
newSyncedDate = newSyncedDate.Value.AddTicks(-(newSyncedDate.Value.Ticks % TimeSpan.TicksPerMillisecond));
_storedLastSyncDate[project.ProjectCode] = newSyncedDate;
diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs
index 3a736398..f560084f 100644
--- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs
+++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.Collections.Generic;
@@ -156,11 +156,7 @@ Dictionary lfCustomFieldTypes
// Single line or MultiText fields need writing systems
int fieldWs = LcmMetaData.GetFieldWs(flid);
// That's a "magic" ws, which we need to expand into a (list of) real writing system(s).
-#if FW8_COMPAT
- var wsesForThisField = new List();
-#else
var wsesForThisField = new List();
-#endif
// GetWritingSystemList() in FW 8.3 is buggy and doesn't properly handle the kwsAnal and kwsVern cases, so we handle them here instead.
switch (fieldWs) {
case WritingSystemServices.kwsAnal:
@@ -173,11 +169,10 @@ Dictionary lfCustomFieldTypes
wsesForThisField = WritingSystemServices.GetWritingSystemList(cache, fieldWs, forceIncludeEnglish: false);
break;
}
-#if FW8_COMPAT
- IEnumerable inputSystems = wsesForThisField.Select(LcmWs => LcmWs.Id);
-#else
- IEnumerable inputSystems = wsesForThisField.Select(LcmWs => LcmWs.LanguageTag);
-#endif
+ // Spelled as every multitext key is, by GetStrFromWs: LF matches a field's input
+ // systems against an entry's keys exactly. See ConvertLcmToMongoLexicon.LfTagOf.
+ IEnumerable inputSystems = wsesForThisField.Select(
+ LcmWs => servLoc.WritingSystemManager.GetStrFromWs(LcmWs.Handle));
// GetWritingSystemList returns all analysis WSes even when asked for just one, so if this
// is a single-line custom field, trim the WSes down to just the first one
if (lfCustomFieldType.StartsWith("Single"))
diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs
index 413bdd9a..0a74f7be 100644
--- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs
+++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.Collections.Generic;
@@ -14,6 +14,7 @@
using MongoDB.Bson;
using SIL.LCModel;
using SIL.LCModel.Core.KernelInterfaces;
+using SIL.LCModel.Core.WritingSystems;
using SIL.Progress;
namespace LfMerge.Core.DataConverters
@@ -65,15 +66,9 @@ public ConvertLcmToMongoLexicon(ILfProject lfProject, ILogger logger, IMongoConn
// Reconcile writing systems from Lcm and Mongo
Dictionary lfWsList = LcmWsToLfWs();
- #if FW8_COMPAT
- List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(ws => ws.Id).ToList();
- List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(ws => ws.Id).ToList();
- List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(ws => ws.Id).ToList();
- #else
- List VernacularWss = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems.Select(ws => ws.LanguageTag).ToList();
- List AnalysisWss = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems.Select(ws => ws.LanguageTag).ToList();
- List PronunciationWss = ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems.Select(ws => ws.LanguageTag).ToList();
- #endif
+ List VernacularWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentVernacularWritingSystems, "vernacular");
+ List AnalysisWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems, "analysis");
+ List PronunciationWss = LfTagsOf(ServiceLocator.LanguageProject.CurrentPronunciationWritingSystems, "pronunciation");
Connection.SetInputSystems(LfProject, lfWsList, VernacularWss, AnalysisWss, PronunciationWss);
ListConverters = new Dictionary();
@@ -585,15 +580,77 @@ private static string LcmPictureFilenameToLfPictureFilename(string LcmInternalFi
return Regex.Replace(LcmInternalFilename.Normalize(System.Text.NormalizationForm.FormC), @"^Pictures[/\\]", "");
}
+ ///
+ /// How Language Forge spells an LCM writing system: exactly as every multitext key LfMerge
+ /// writes is spelled, since those come from GetStrFromWs too (see LfMultiText). That is the
+ /// writing system's Id, falling back to its LanguageTag should it have no Id.
+ ///
+ /// Input systems and the config's field lists must be spelled the same way, because LF's
+ /// editor matches a field's input systems against an entry's keys exactly and
+ /// case-sensitively. They were spelled by LanguageTag, which in older projects is not the
+ /// Id: brb-flex-2022's Khmer is Id "km-Khmr-KH" with LanguageTag "km-KH", and che-flex's
+ /// vernacular is Id "ce-la" with LanguageTag "ce-LA", so LF showed all of that text as empty.
+ ///
+ private string LfTagOf(CoreWritingSystemDefinition ws)
+ {
+ return ServiceLocator.WritingSystemManager.GetStrFromWs(ws.Handle);
+ }
+
+ ///
+ /// The LF tags of one of the project's current writing-system lists, leaving out any Id the
+ /// list names that LCM has no writing system for, rather than ending the transfer. That
+ /// protects the pronunciation list. A missing Id in a current vernacular or analysis list is
+ /// almost always in the full list too, and LcmWsToLfWs, which reads the full lists through
+ /// AllWritingSystems, still fails on it first -- as FieldWorks itself would.
+ ///
+ /// The list is read with foreach on purpose. liblcm's enumerator yields null for such an
+ /// Id, but its indexer and CopyTo throw KeyNotFoundException, and LINQ's Select and ToList
+ /// use those on an IList.
+ ///
+ private List LfTagsOf(IEnumerable wss, string listName)
+ {
+ var tags = new List();
+ int missing = 0;
+ foreach (CoreWritingSystemDefinition ws in wss)
+ {
+ if (ws == null)
+ missing++;
+ else
+ tags.Add(LfTagOf(ws));
+ }
+ if (missing > 0)
+ Logger.Warning("LcmToMongo: the current {0} writing systems list names {1} writing system(s) that LCM does not have; leaving them out",
+ listName, missing);
+ return tags;
+ }
+
+ ///
+ /// The handles of the writing systems in one of the project's current lists, leaving out any
+ /// Id it names that LCM has no writing system for; see LfTagsOf.
+ ///
+ private static HashSet Handles(IEnumerable wss)
+ {
+ var handles = new HashSet();
+ foreach (CoreWritingSystemDefinition ws in wss)
+ {
+ if (ws != null)
+ handles.Add(ws.Handle);
+ }
+ return handles;
+ }
+
///
/// Converts Lcm writing systems to LF input systems
///
/// The list of LF input systems.
private Dictionary LcmWsToLfWs()
{
- // Using var here so that we'll stay compatible with both FW 8 and 9 (the type of these two lists changed between 8 and 9).
- var vernacularWSList = ServiceLocator.LanguageProject.CurrentVernacularWritingSystems;
- var analysisWSList = ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems;
+ // By handle, from the same enumerator LfTagsOf reads the lists with. The lists' own
+ // Contains compares Ids exactly, while LCM resolves them ignoring case, so a list
+ // naming "qaa-Zxxx-x-kal-AUDIO" for the writing system "qaa-Zxxx-x-kal-audio" put it in
+ // the vernacular list LF is given but flagged it as not vernacular.
+ var vernacularHandles = Handles(ServiceLocator.LanguageProject.CurrentVernacularWritingSystems);
+ var analysisHandles = Handles(ServiceLocator.LanguageProject.CurrentAnalysisWritingSystems);
var lfWsList = new Dictionary();
foreach (var LcmWs in ServiceLocator.LanguageProject.AllWritingSystems)
@@ -605,20 +662,12 @@ private Dictionary LcmWsToLfWs()
Abbreviation = LcmWs.Abbreviation,
IsRightToLeft = LcmWs.RightToLeftScript,
LanguageName = LcmWs.LanguageName,
- #if FW8_COMPAT
- Tag = LcmWs.Id,
- #else
- Tag = LcmWs.LanguageTag,
- #endif
- VernacularWS = vernacularWSList.Contains(LcmWs),
- AnalysisWS = analysisWSList.Contains(LcmWs)
+ Tag = LfTagOf(LcmWs),
+ VernacularWS = vernacularHandles.Contains(LcmWs.Handle),
+ AnalysisWS = analysisHandles.Contains(LcmWs.Handle)
};
- #if FW8_COMPAT
- lfWsList[LcmWs.Id] = lfWs;
- #else
- lfWsList[LcmWs.LanguageTag] = lfWs;
- #endif
+ lfWsList[lfWs.Tag] = lfWs;
}
return lfWsList;
}
diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs
index a8de39d0..b301b2b3 100644
--- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs
+++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs
@@ -25,14 +25,21 @@ public class ConvertMongoToLcmCustomField
private IFwMetaDataCacheManaged lcmMetaData;
private ILogger logger;
private int wsEn;
+ // Fields already reported as not writable, so each is reported once per sync rather than
+ // once for every entry, sense or example that has it
+ private readonly HashSet unwritableFieldsReported = new HashSet();
+ // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm
+ private readonly ISet configuredTags;
- public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn)
+ public ConvertMongoToLcmCustomField(LcmCache cache, FwServiceLocatorCache serviceLocator, ILogger logger, int wsEn,
+ ISet configuredTags = null)
{
this.cache = cache;
this.servLoc = serviceLocator;
this.lcmMetaData = (IFwMetaDataCacheManaged)cache.MetaDataCacheAccessor;
this.logger = logger;
this.wsEn = wsEn;
+ this.configuredTags = configuredTags;
}
public Guid ParseGuidOrDefault(string input)
@@ -89,7 +96,9 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui
if (fieldName == null)
return false;
- // Valid field types in LCM are GenDate, Integer, String, OwningAtomic, ReferenceAtomic, and ReferenceCollection, so that's all we implement.
+ // Valid field types in LCM are GenDate, Integer, String, MultiUnicode, OwningAtomic,
+ // ReferenceAtomic, and ReferenceCollection, so that's all we implement. MultiUnicode is the
+ // one every text custom field created in LF has, since LF has no single-string custom field.
switch (fieldType)
{
case CellarPropertyType.GenDate:
@@ -176,7 +185,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui
if (multiPara.InputSystem == null)
wsId = lcmMetaData.GetFieldWs(flid);
else
- wsId = servLoc.WritingSystemFactory.GetWsFromStr(multiPara.InputSystem);
+ wsId = LanguageTags.WsIdFromLfTag(servLoc.WritingSystemFactory, multiPara.InputSystem);
ConvertUtilities.SetCustomStTextValues(text, multiPara.Paragraphs, wsId);
return true;
@@ -356,17 +365,53 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui
return true;
}
+ case CellarPropertyType.MultiUnicode:
+ {
+ var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument);
+
+ var existingWsIds = new List();
+ ITsMultiString oldValues = data.get_MultiStringProp(hvo, flid);
+ if (oldValues != null)
+ {
+ for (int index = 0; index < oldValues.StringCount; index++)
+ {
+ int oldWsId;
+ oldValues.GetStringFromIndex(index, out oldWsId);
+ existingWsIds.Add(oldWsId);
+ }
+ }
+
+ // The same writer as the built-in multitext fields, reaching LCM through
+ // ISilDataAccess because a custom field has no IMultiAccessorBase.
+ return valueAsMultiText.WriteToLcm(existingWsIds,
+ wsId => data.get_MultiStringAlt(hvo, flid, wsId),
+ (wsId, tss) => data.SetMultiStringAlt(hvo, flid, wsId, tss),
+ servLoc.WritingSystemFactory,
+ configuredTags,
+ tag => logger.Warning("Custom field {0}: skipping unidentified writing system {1}",
+ fieldName, tag),
+ (notWritten, written) => logger.Warning(
+ "Custom field {0}: keys \"{1}\" and \"{2}\" name the same writing system; wrote \"{2}\" ({3}), not \"{1}\" ({4})",
+ fieldName, notWritten, written, valueAsMultiText.Excerpt(written),
+ valueAsMultiText.Excerpt(notWritten)));
+ }
+
case CellarPropertyType.String:
{
var valueAsMultiText = BsonSerializer.Deserialize(value.AsBsonDocument);
int wsIdForField = lcmMetaData.GetFieldWs(flid);
- string wsStrForField = servLoc.WritingSystemFactory.GetStrFromWs(wsIdForField);
- KeyValuePair kv = valueAsMultiText.BestStringAndWs(new string[] { wsStrForField });
- string foundWs = kv.Key ?? string.Empty;
+ KeyValuePair kv = valueAsMultiText.BestStringAndWsId(
+ new[] { wsIdForField }, servLoc.WritingSystemFactory, configuredTags);
+ int foundWsId = kv.Key;
string foundData = kv.Value ?? string.Empty;
- int foundWsId = servLoc.WritingSystemFactory.GetWsFromStr(foundWs);
if (foundWsId == 0)
- return false; // Skip any unidentified writing systems
+ {
+ // Nothing placeable, so the field is left as it was; if LF holds text, say why
+ if (!valueAsMultiText.IsEmpty)
+ ConvertMongoToLcmLexicon.LogTextNotPlaced(valueAsMultiText, "Custom field " + fieldName,
+ logger, servLoc.WritingSystemFactory, configuredTags);
+ return false;
+ }
ITsString oldValue = data.get_StringProp(hvo, flid);
ITsString newValue = ConvertMongoToLcmTsStrings.SpanStrToTsString(foundData, foundWsId, servLoc.WritingSystemFactory);
if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) // GetDiffsInTsStrings() returns null when there are no changes
@@ -379,8 +424,12 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui
}
default:
+ if (unwritableFieldsReported.Add(flid))
+ logger.Warning(
+ "Custom field {0} not written to LCM: CellarPropertyType.{1} is not implemented "
+ + "for the LF to LCM direction, so any data LF holds in it is being dropped",
+ fieldName, fieldType.ToString());
return false;
- // TODO: Maybe issue a proper warning (or error) log message for "field type not recognized"?
}
}
@@ -421,4 +470,4 @@ public void SetCustomFieldsForThisCmObject(ICmObject cmObj, string objectType, B
}
}
}
-}
\ No newline at end of file
+}
diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs
index 70d172fb..c86a1065 100644
--- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs
+++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs
@@ -6,6 +6,7 @@
using System.Text.RegularExpressions;
using LfMerge.Core.DataConverters.CanonicalSources;
using LfMerge.Core.FieldWorks;
+using LfMerge.Core.LanguageForge.Config;
using LfMerge.Core.LanguageForge.Model;
using LfMergeBridge.LfMergeModel;
using LfMerge.Core.Logging;
@@ -39,9 +40,13 @@ public class ConvertMongoToLcmLexicon
private int _wsEn;
private ConvertMongoToLcmCustomField _convertCustomField;
+ // Every writing-system spelling the project's config uses; see LfMultiText.WriteToLcm
+ private ISet _configuredTags;
- // Entries where LF's copy differs from FieldWorks but no LF user has touched it since the
- // last sync, so FieldWorks holds the newer version (see EditedInLfSinceLastSync)
+ // Entries where LF's copy differs from FieldWorks, or FieldWorks lacks it, but no LF user has
+ // touched it since the last sync, so FieldWorks holds the newer version (see
+ // EditedInLfSinceLastSync). An entry FieldWorks lacks in a project that has never synced is
+ // not counted: it is LF's own, and is sent (see LfLexEntryToLcmLexEntry).
private int _changedOnlyInFieldWorks;
private int _absentOnlyFromFieldWorks;
@@ -138,14 +143,18 @@ public ConversionError RunConversion()
// Logger.Debug("Running \"fake\" MtFComments, should see comments show up below:");
var entryObjectIdToGuidMappings = Connection.GetGuidsByObjectIdForCollection(LfProject, MagicStrings.LfCollectionNameForLexicon);
EntryCounts.Reset();
- // Update writing systems from project config input systems. Won't commit till the end
+ // Update writing systems from project config input systems. Won't commit till the end.
+ // Which list a new writing system belongs in depends on what it is actually used for, so
+ // the lexicon is counted -- a second streaming pass over Mongo, which GetLexicon yields
+ // from a cursor -- but only if some input system is new to LCM.
UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", Cache.ActionHandlerAccessor, () =>
- LfWsToLcmWs(ProjectRecord.InputSystems));
+ LfWsToLcmWs(ProjectRecord.InputSystems, () => LfWritingSystemUsage.FromLexicon(GetLexicon(LfProject))));
// Set English ws handle again in case it changed
_wsEn = ServiceLocator.WritingSystemFactory.GetWsFromStr("en");
- _convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn);
+ _configuredTags = ConfiguredTags(ProjectRecord.Config);
+ _convertCustomField = new ConvertMongoToLcmCustomField(Cache, ServiceLocator, Logger, _wsEn, _configuredTags);
IEnumerable lexicon = GetLexicon(LfProject);
_changedOnlyInFieldWorks = 0;
@@ -201,45 +210,446 @@ private IEnumerable GetLexicon(ILfProject project)
return Connection.GetRecords(project, MagicStrings.LfCollectionNameForLexicon);
}
+ // The field groups whose vernacular/analysis role is never in doubt, because FieldWorks fixes
+ // it: a headword and an example sentence are vernacular, a gloss and a translation are not.
+ // Everything else is classified per project by which of these its writing systems appear in
+ // -- a fixed list cannot be right for every project: in the 2026-07-06 corpus 741 projects
+ // configure etymology with analysis writing systems and 378 with vernacular ones.
+ //
+ // Counting that corpus, for each field, the projects whose text there is exclusively
+ // vernacular against exclusively analysis: the example sentence is 332 to 5, the example
+ // translation 1 to 320, the entry note 0 to 255 and the literal meaning 4 to 136.
+ //
+ // The pronunciation field is deliberately NOT here. It looks vernacular at 114 to 16, but
+ // that is 12% analysis, above the MinorityShare a single writing system would have to stay
+ // under to count as one role alone, so projects evidently use it for more than one thing.
+ // Its text still counts for something in ClassifyByText, though: see there.
+ private static readonly string[] VernacularAnchorFields = {
+ LfWritingSystemUsage.Lexeme,
+ LfWritingSystemUsage.CitationForm,
+ LfWritingSystemUsage.ExampleSentence,
+ };
+ private static readonly string[] AnalysisAnchorFields = {
+ LfWritingSystemUsage.Definition,
+ LfWritingSystemUsage.Gloss,
+ LfWritingSystemUsage.ExampleTranslation,
+ LfWritingSystemUsage.Note,
+ LfWritingSystemUsage.LiteralMeaning,
+ };
+
+ ///
+ /// Work out which writing systems this project treats as vernacular and which as analysis,
+ /// from its own config and the text in its lexicon.
+ ///
+ /// The anchor fields have a role FieldWorks fixes: the lexeme, citation form and example
+ /// sentence are vernacular; the definition, gloss, example translation, note and literal
+ /// meaning are analysis (see VernacularAnchorFields).
+ ///
+ /// Where the lexicon has text in the anchors for a writing system, that text decides: one
+ /// holding nearly all of it in the vernacular anchors is vernacular whatever the config
+ /// offers, and vice versa, and one whose text is split is both. See
+ /// . A writing system whose only text is pronunciation text is
+ /// vernacular; see .
+ ///
+ /// The rest go by the config. A writing system the anchors offer is placed by them, and one
+ /// in both anchor groups -- "en" very often is -- is both. Every other field carrying input
+ /// systems (etymology, custom fields) is assigned by overlap: a writing system exclusive to
+ /// the vernacular role is evidence the field is vernacular, and vice versa. Evidence decides
+ /// the role of the field, not of every writing system in it, and it has to point one way: a
+ /// field offering writing systems exclusive to each role, such as an etymology configured
+ /// as [seh, pt], is evidence of neither, so it does not make Portuguese vernacular, nor
+ /// anything else it offers that the anchors have not placed. All the fields' evidence is
+ /// weighed before any writing system is placed by it, so the order of the config's fields
+ /// does not matter. What is left is settled, where it can be, by the company a writing
+ /// system keeps and then by its language.
+ ///
+ /// This replaces comparing each tag against ProjectRecord.LanguageCode, which named exactly one
+ /// vernacular writing system and got it wrong whenever languageCode disagreed with the lexeme
+ /// field: flh-flex has languageCode "flh-x-ortho" (used only by etymology) while its lexeme
+ /// uses "flh-x-cm", and odo has languageCode "th" which is not among its writing systems at all,
+ /// leaving its 24,460 "en" headwords classified as analysis.
+ ///
+ ///
+ /// How much text each writing system holds in each field, or null to go on the config alone.
+ ///
+ ///
+ /// The vernacular tags; the analysis tags; and the tags that could not be resolved. The first
+ /// two sets overlap wherever a writing system plays both roles. A tag in neither set -- one
+ /// no config field uses -- is analysis, as it always has been.
+ ///
+ /// Unresolved writing systems are treated as BOTH vernacular and analysis, since nothing says
+ /// which they are and a writing system missing from the list a field draws on leaves that
+ /// field's data with nowhere to go. The FieldWorks fields such a writing system most often
+ /// feeds (etymology form, example sentence) are vernacular-typed, which is why being only
+ /// analysis will not do; but a custom field can be analysis-typed, as a French-only notes
+ /// field would be, which is why being only vernacular will not do either. 7 projects in the
+ /// 2026-10-01 corpus need this, e.g. kam-flex, whose Spanish and French are offered only in
+ /// a wordlist field that offers its vernacular and its analysis language as well.
+ ///
+ public static (ISet Vernacular, ISet Analysis, ISet Unresolved)
+ ClassifyVernacularWritingSystems(LfProjectConfig config, string languageCode,
+ LfWritingSystemUsage usage = null)
+ {
+ var byPath = new Dictionary>();
+ CollectInputSystems((config == null) ? null : config.Entry, "", byPath);
+
+ var vernacular = new HashSet(StringComparer.OrdinalIgnoreCase);
+ var analysis = new HashSet(StringComparer.OrdinalIgnoreCase);
+ var unresolved = new HashSet(StringComparer.OrdinalIgnoreCase);
+
+ // Writing systems whose text has already answered for them. They keep that answer: the
+ // config below may offer them for fields nobody writes in, and must not overrule it.
+ var decidedByText = ClassifyByText(usage, byPath, vernacular, analysis);
+
+ var vernacularAnchor = Union(byPath, VernacularAnchorFields);
+ var analysisAnchor = Union(byPath, AnalysisAnchorFields);
+ vernacular.UnionWith(vernacularAnchor.Where(tag => !decidedByText.Contains(tag)));
+ analysis.UnionWith(analysisAnchor.Where(tag => !decidedByText.Contains(tag)));
+
+ // Only a writing system exclusive to one role is evidence of a field's role.
+ var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis);
+
+ // Every field's evidence is gathered before any writing system is placed by it, so that
+ // the answer does not depend on the order the config lists its fields in. Placing as
+ // each field came used to let a field with no evidence, listed before the one that
+ // placed a writing system, put that writing system in doubt, and the passes below could
+ // then strip one of the roles that field gave it.
+ var anchors = new HashSet(VernacularAnchorFields.Concat(AnalysisAnchorFields));
+ var inVernacularField = new HashSet(StringComparer.OrdinalIgnoreCase);
+ var inAnalysisField = new HashSet(StringComparer.OrdinalIgnoreCase);
+ var inFieldOfNoRole = new HashSet(StringComparer.OrdinalIgnoreCase);
+ foreach (var field in byPath)
+ {
+ if (anchors.Contains(field.Key) || field.Value.Count == 0)
+ continue;
+ // A writing system its own text has spoken for is still evidence of what this field
+ // is for, but nothing here may change the answer it was given.
+ var undecided = field.Value.Where(tag => !decidedByText.Contains(tag)).ToList();
+ bool withVernacular = field.Value.Any(vernacularOnly.Contains);
+ bool withAnalysis = field.Value.Any(analysisOnly.Contains);
+ if (withVernacular && !withAnalysis)
+ inVernacularField.UnionWith(undecided);
+ else if (withAnalysis && !withVernacular)
+ inAnalysisField.UnionWith(undecided);
+ else
+ {
+ // Overlaps neither role, or both: nothing says which role this field plays. A field
+ // offering a vernacular-only and an analysis-only writing system side by side is
+ // no more evidence for one role than the other, just as only unanimous company
+ // counts in ResolveByCompanions.
+ inFieldOfNoRole.UnionWith(undecided);
+ }
+ }
+ // A field of no role puts in doubt only the writing systems nothing else places, neither
+ // the anchors nor a field that has a role. One already placed keeps its place: "en" in
+ // both the lexeme and the gloss is not made doubtful by also turning up in a sense's
+ // general note, and if it were, the passes below could strip one of its roles on the
+ // strength of the company it keeps there. (The entry's note is an anchor, so never comes
+ // this far.)
+ var doubtful = inFieldOfNoRole
+ .Where(tag => !vernacular.Contains(tag) && !analysis.Contains(tag)
+ && !inVernacularField.Contains(tag) && !inAnalysisField.Contains(tag))
+ .ToList();
+ vernacular.UnionWith(inVernacularField);
+ analysis.UnionWith(inAnalysisField);
+ vernacular.UnionWith(doubtful);
+ analysis.UnionWith(doubtful);
+ unresolved.UnionWith(doubtful);
+
+ ResolveByCompanions(byPath, vernacular, analysis, unresolved);
+ ResolveByLanguageAffinity(vernacular, analysis, unresolved);
+
+ // Safety net for a config LfMerge cannot read: without this such a project would get no
+ // vernacular writing system at all, which is worse than the old rule.
+ if (vernacular.Count == 0 && !string.IsNullOrEmpty(languageCode))
+ vernacular.Add(languageCode);
+
+ return (vernacular, analysis, unresolved);
+ }
+
+ ///
+ /// The writing systems that play one role only: those vernacular and not analysis, and those
+ /// analysis and not vernacular. Each pass that weighs evidence recomputes them, since the
+ /// passes before it change the two sets.
+ ///
+ private static (HashSet VernacularOnly, HashSet AnalysisOnly) ExclusiveRoles(
+ ISet vernacular, ISet analysis)
+ {
+ var vernacularOnly = new HashSet(vernacular, StringComparer.OrdinalIgnoreCase);
+ vernacularOnly.ExceptWith(analysis);
+ var analysisOnly = new HashSet(analysis, StringComparer.OrdinalIgnoreCase);
+ analysisOnly.ExceptWith(vernacular);
+ return (vernacularOnly, analysisOnly);
+ }
+
+ ///
+ /// Settles what is left by the company a writing system keeps. What is left is offered only
+ /// in fields that said nothing of their role -- a field with a writing system of one role
+ /// alone has already placed the rest -- but the company it keeps there may since have been
+ /// placed by another field: in a custom field offering "abc" and "xyz" and nothing else of
+ /// known role, "xyz" is vernacular if another custom field, offering "abc" beside the
+ /// vernacular, has made "abc" vernacular.
+ ///
+ /// Only unanimous company counts. A field holding writing systems of both roles says
+ /// nothing, which is the same reason a writing system in both anchors is no evidence of a
+ /// field's role. One pass is made, not one until nothing changes: a writing system whose
+ /// only company is another left in doubt stays in doubt, and keeps both roles, even if that
+ /// other is settled here.
+ ///
+ private static void ResolveByCompanions(IDictionary> byPath,
+ ISet vernacular, ISet analysis, ISet unresolved)
+ {
+ if (unresolved.Count == 0)
+ return;
+
+ // Unresolved writing systems sit in both sets, so they are in neither of these and
+ // cannot vote for each other.
+ var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis);
+
+ foreach (string tag in unresolved.ToList())
+ {
+ var companions = new HashSet(StringComparer.OrdinalIgnoreCase);
+ foreach (var field in byPath)
+ {
+ if (field.Value.Contains(tag))
+ companions.UnionWith(field.Value);
+ }
+ companions.Remove(tag);
+ bool withVernacular = companions.Any(vernacularOnly.Contains);
+ bool withAnalysis = companions.Any(analysisOnly.Contains);
+ if (withVernacular == withAnalysis)
+ continue; // No company, or company of both roles: still nothing to go on.
+ if (withVernacular)
+ analysis.Remove(tag);
+ else
+ vernacular.Remove(tag);
+ unresolved.Remove(tag);
+ }
+ }
+
+ ///
+ /// Settles what is left by language. A writing system plays the same role as the others
+ /// that spell the same language: a phonetic or audio spelling of the vernacular is
+ /// vernacular too, and a regional spelling of the analysis language is analysis.
+ ///
+ private static void ResolveByLanguageAffinity(ISet vernacular, ISet analysis,
+ ISet unresolved)
+ {
+ if (unresolved.Count == 0)
+ return;
+
+ // Unresolved writing systems sit in both sets, so they are in neither of these and
+ // cannot vote for each other.
+ var (vernacularOnly, analysisOnly) = ExclusiveRoles(vernacular, analysis);
+ var vernacularLanguages = new HashSet(vernacularOnly.Select(LanguageOf), StringComparer.OrdinalIgnoreCase);
+ var analysisLanguages = new HashSet(analysisOnly.Select(LanguageOf), StringComparer.OrdinalIgnoreCase);
+
+ foreach (string tag in unresolved.ToList())
+ {
+ string language = LanguageOf(tag);
+ bool withVernacular = vernacularLanguages.Contains(language);
+ bool withAnalysis = analysisLanguages.Contains(language);
+ if (withVernacular == withAnalysis)
+ continue; // The language is spoken for by both roles, or by neither.
+ if (withVernacular)
+ analysis.Remove(tag);
+ else
+ vernacular.Remove(tag);
+ unresolved.Remove(tag);
+ }
+ }
+
+ // Subtags that mark a variant of a language rather than a language of its own. FieldWorks
+ // also appends "dupl1", "dupl2" and so on when a project needs a second writing system for
+ // the same thing.
+ private static readonly HashSet VariantMarkers = new HashSet(
+ new[] { "fonipa", "etic", "emic", "audio" }, StringComparer.OrdinalIgnoreCase);
+
+ ///
+ /// The language a tag names, as far as telling one project's writing systems apart goes:
+ /// the language subtag plus any private-use subtags that name the language rather than a
+ /// variant of it. Script and region are left out, so "en" and "en-GB" are one language.
+ ///
+ /// The private-use subtags have to stay, because under "qaa" -- the code for a language
+ /// with no code of its own, which Language Forge projects lean on heavily -- they ARE the
+ /// language: "qaa-x-kal" and "qaa-x-hbo" are no more the same language than "fr" and "de".
+ /// So "qaa-x-kal", "qaa-fonipa-x-kal" and "qaa-Zxxx-x-kal-audio" all come out "qaa-kal",
+ /// while "seh" and "seh-fonipa-x-etic" both come out "seh". The same goes for a tag that is
+ /// private-use from the start: "x-kal" comes out "x-kal", not "x".
+ ///
+ private static string LanguageOf(string tag)
+ {
+ if (string.IsNullOrEmpty(tag))
+ return tag;
+ string[] parts = tag.Split('-');
+ var language = new List { parts[0] };
+ int privateUse = Array.FindIndex(parts,
+ part => part.Equals("x", StringComparison.OrdinalIgnoreCase));
+ if (privateUse >= 0)
+ {
+ for (int i = privateUse + 1; i < parts.Length; i++)
+ {
+ if (VariantMarkers.Contains(parts[i]) ||
+ parts[i].StartsWith("dupl", StringComparison.OrdinalIgnoreCase))
+ continue;
+ language.Add(parts[i]);
+ }
+ }
+ return string.Join("-", language);
+ }
+
+ ///
+ /// A writing system holding at most this share of its text in the fields of one role is
+ /// taken to belong to the other role alone; in between, it plays both.
+ ///
+ /// A few strays are not evidence of a second role. In brb-flex-2022 the phonetic writing
+ /// system has 4,030 headwords and one stray string in an example reference, and FieldWorks
+ /// has it as vernacular only; in cmo-khm-flex the vernacular has 7,815 vernacular strings
+ /// against 300 in glosses, and is likewise vernacular only. The one writing system the four
+ /// FieldWorks projects on hand really do list as BOTH sits at 28%, so anything between 4%
+ /// and 28% separates the two, and 10% is the round number in the middle.
+ ///
+ public const double MinorityShare = 0.10;
+
+ ///
+ /// Classifies every writing system the lexicon has text for, by where that text sits.
+ /// Adds them to and , and returns
+ /// the ones it answered for.
+ ///
+ /// A writing system whose only text is in the pronunciation field is made vernacular,
+ /// because FieldWorks draws its pronunciation writing systems from the vernacular ones
+ /// (InitializePronunciationWritingSystems considers no others). It is not returned as
+ /// answered for, so the anchor fields can still add analysis: Swedish offered for the
+ /// gloss, with a stray pronunciation string, is both. But from here on it counts as
+ /// vernacular, as any placed writing system does: a field of no role does not put it in
+ /// doubt, and a custom field it alone is offered in is taken for a vernacular field. So
+ /// a writing system offered only in custom fields, with pronunciation text and nothing
+ /// else, is vernacular only. In the 2026-10-01 corpus all 57 writing systems with
+ /// pronunciation text and no anchor text are used that way -- phonetic, IPA and audio
+ /// writing systems, and xin-flex's "xin", which holds 226 pronunciations and nothing else
+ /// -- and FLEx has as vernacular every one of them whose role LfMerge's export records.
+ /// The pronunciation field is still not an anchor; see VernacularAnchorFields.
+ ///
+ private static ISet ClassifyByText(LfWritingSystemUsage usage,
+ IDictionary> byPath, ISet vernacular, ISet analysis)
+ {
+ var decided = new HashSet(StringComparer.OrdinalIgnoreCase);
+ if (usage == null || usage.EntriesCounted == 0)
+ return decided;
+
+ // Every writing system the config mentions, plus any the config forgot but the lexicon
+ // uses anyway -- those need classifying just as much.
+ var candidates = new HashSet(StringComparer.OrdinalIgnoreCase);
+ foreach (ISet tags in byPath.Values)
+ candidates.UnionWith(tags);
+ foreach (string path in VernacularAnchorFields.Concat(AnalysisAnchorFields))
+ candidates.UnionWith(usage.TagsWithText(path));
+ candidates.UnionWith(usage.TagsWithText(LfWritingSystemUsage.Pronunciation));
+
+ foreach (string tag in candidates)
+ {
+ int vernacularText = usage.NonEmptyCount(VernacularAnchorFields, tag);
+ int analysisText = usage.NonEmptyCount(AnalysisAnchorFields, tag);
+ int total = vernacularText + analysisText;
+ if (total == 0)
+ {
+ // No text in a field of known role, so its text decides nothing.
+ if (usage.NonEmptyCount(LfWritingSystemUsage.Pronunciation, tag) > 0)
+ vernacular.Add(tag);
+ continue;
+ }
+ double analysisShare = (double)analysisText / total;
+ if (analysisShare <= MinorityShare)
+ vernacular.Add(tag);
+ else if (analysisShare >= 1.0 - MinorityShare)
+ analysis.Add(tag);
+ else
+ {
+ vernacular.Add(tag);
+ analysis.Add(tag);
+ }
+ decided.Add(tag);
+ }
+ return decided;
+ }
+
+ private static ISet Union(IDictionary> byPath, IEnumerable paths)
+ {
+ var result = new HashSet(StringComparer.OrdinalIgnoreCase);
+ foreach (var path in paths)
+ {
+ ISet tags;
+ if (byPath.TryGetValue(path, out tags))
+ result.UnionWith(tags);
+ }
+ return result;
+ }
+
+ ///
+ /// Every writing-system spelling the config's fields use, exactly as spelled: LF's editor
+ /// matches them against an entry's keys exactly.
+ ///
+ private static ISet ConfiguredTags(LfProjectConfig config)
+ {
+ var byPath = new Dictionary>();
+ CollectInputSystems((config == null) ? null : config.Entry, "", byPath);
+ var tags = new HashSet(StringComparer.Ordinal);
+ foreach (ISet fieldTags in byPath.Values)
+ tags.UnionWith(fieldTags);
+ return tags;
+ }
+
+ ///
+ /// Record every config field's input systems, keyed by its dotted path as Mongo spells it, so
+ /// nesting goes through "fields" ("senses.fields.examples.fields.sentence").
+ ///
+ private static void CollectInputSystems(LfConfigFieldList fieldList, string path,
+ IDictionary> byPath)
+ {
+ if (fieldList == null || fieldList.Fields == null)
+ return;
+ foreach (var child in fieldList.Fields)
+ {
+ string childPath = string.IsNullOrEmpty(path) ? child.Key : path + "." + child.Key;
+ var multiText = child.Value as LfConfigMultiText;
+ // A null or empty tag names no writing system, and a null one would make the text
+ // counts throw, ending the whole transfer
+ if (multiText != null && multiText.InputSystems != null)
+ byPath[childPath] = new HashSet(multiText.InputSystems.Where(tag => !string.IsNullOrEmpty(tag)),
+ StringComparer.OrdinalIgnoreCase);
+ var nested = child.Value as LfConfigFieldList;
+ if (nested != null)
+ CollectInputSystems(nested, childPath + ".fields", byPath);
+ }
+ }
+
///
/// Converts the list of LF input systems and adds them to Lcm writing systems
///
/// List of LF input systems.
- private void LfWsToLcmWs(Dictionary lfWsList)
- {
- // Between FW 8.2 and 9, a few classes and interfaces were renamed. The ones most relevant here are
- // IWritingSystemManager (interface was removed and replaced with the WritingSystemManager concrete class),
- // and PalasoWritingSystem which was replaced with CoreWritingSystemDefinition. Since their internals
- // didn't change much (and the only changes were in areas we don't access), we can use a simple compiler
- // define to choose the type of the wsm variable here (and the ws variable later) and we're fine.
- // HOWEVER, if the code inside #if...#endif blocks starts to grow, this is not an ideal solution. A better
- // solution if the code grows complex will be to write several classes to the same interface, each of which
- // can deal with one particular version of FW or Lcm. Register them all with Autofac with a way to choose among them
- // (http://docs.autofac.org/en/stable/register/registration.html#selection-of-an-implementation-by-parameter-value)
- // and then, at runtime, we can instantiate the particular class that's needed for dealing with *this* FW project.
- //
- // But for now, these #if...#endif blocks are enough. - 2016-03 RM
-#if FW8_COMPAT
- // Note that we can't use ILgWritingSystemFactory here, because it doesn't have some methods we need later on.
- IWritingSystemManager wsManager = ServiceLocator.WritingSystemManager;
-#else
+ ///
+ /// Counts the text in the lexicon, for classifying a writing system new to LCM. Called only
+ /// if there is one, since it reads the whole lexicon.
+ ///
+ private void LfWsToLcmWs(Dictionary lfWsList,
+ Func countUsage = null)
+ {
WritingSystemManager wsManager = ServiceLocator.WritingSystemManager;
-#endif
if (wsManager == null)
{
Logger.Error("Failed to find the writing system manager");
return;
}
- string vernacularLanguageCode = ProjectRecord.LanguageCode;
+ // Worked out when the first writing system new to LCM turns up, and not at all if none
+ // does, which after a project's first sync is nearly always: only a new writing system is
+ // put in a list, and working out which ones counts the whole lexicon. It depends only on
+ // LF's config and text, so the writing systems created before then cannot change it.
+ (ISet Vernacular, ISet Analysis, ISet Unresolved)? classification = null;
// TODO: Split the inside of this foreach() out into its own function
foreach (var lfWs in lfWsList.Values)
{
-#if FW8_COMPAT
- IWritingSystem ws;
-#else
CoreWritingSystemDefinition ws;
-#endif
// It would be nice to call this to add to both analysis and vernacular WS.
// But we need the flexibility of bringing in LF WS properties.
@@ -253,46 +663,112 @@ private void LfWsToLcmWs(Dictionary lfWsList)
}
*/
- // TODO: It might be possible to rewrite this code to NOT rely on TryGet() after all, in which case we could
- // use the ILgWritingSystemFactory interface and remove one point of FW 8-to-9 API incompatibility.
-
- if (wsManager.TryGet(lfWs.Tag, out ws))
+ // Find the writing system the way every other LF tag is found before creating one.
+ // LfMerge used to export input systems by LanguageTag, while LCM knows a writing
+ // system by its Id, and in older projects the two differ: spt-flex's input system "hi-IN" is
+ // the writing system LCM holds as "hi-Deva-IN". GetOrSet looks a tag up only as an
+ // Id, so it did not find it, created a duplicate "hi-IN" and put it in the current
+ // lists; WsIdFromLfTag, which every multitext key goes through, then matched the
+ // duplicate exactly, and text typed in LF went to it rather than to hi-Deva-IN.
+ int existingHandle = LanguageTags.WsIdFromLfTag(wsManager, lfWs.Tag);
+ bool wsAlreadyExisted;
+ if (existingHandle != 0)
{
- // The WS does check that a property has a different value before setting it
- // (and thus setting IsChanged flag), but for Abbreviation the WS returns
- // Language if not set, and it fails to check that.
- if (ws.Abbreviation != lfWs.Abbreviation)
- ws.Abbreviation = lfWs.Abbreviation;
- ws.RightToLeftScript = lfWs.IsRightToLeft;
- wsManager.Replace(ws);
+ ws = wsManager.Get(existingHandle);
+ wsAlreadyExisted = true;
}
else
{
- ws = wsManager.Create(lfWs.Tag);
- ws.Abbreviation = lfWs.Abbreviation;
- ws.RightToLeftScript = lfWs.IsRightToLeft;
- wsManager.Set(ws);
-
- // LF doesn't distinguish between vernacular/analysis WS, so we'll
- // only assign the project language code to vernacular.
- // All other WS assigned to analysis.
+ // Nothing to find, so create it. GetOrSet creates under the canonical form of the
+ // tag -- "qaa-x-qaa-v" becomes "qaa-x-v", "th-Thai" becomes "th" -- and adds the
+ // writing system to the manager, so no Set() call is needed here.
+ wsAlreadyExisted = wsManager.GetOrSet(lfWs.Tag, out ws);
+ }
- // TODO: What if our vernacular was Thai, but we added th-ipa? This logic needs to be a bit "fuzzier", really.
- if (lfWs.Tag.Equals(vernacularLanguageCode))
+ // The WS does check that a property has a different value before setting it
+ // (and thus setting IsChanged flag), but for Abbreviation the WS returns
+ // Language if not set, and it fails to check that.
+ if (ws.Abbreviation != lfWs.Abbreviation)
+ {
+ ws.Abbreviation = lfWs.Abbreviation;
+ }
+ ws.RightToLeftScript = lfWs.IsRightToLeft;
+ // No wsManager.Replace(ws): ws is the manager's own object, so the edits above are
+ // already in place. Replace looks the writing system up by LanguageTag, and where
+ // the Id is spelled otherwise that lookup does harm. If it finds nothing, Replace
+ // re-registers ws under a fresh handle mid-sync; if it finds another writing system
+ // with that LanguageTag -- brb-flex-2022's unused "km-KH" beside "km-Khmr-KH" --
+ // Replace evicts that one and gives ws its handle.
+
+ if (!wsAlreadyExisted)
+ {
+ // LF doesn't distinguish between vernacular/analysis WS, so the roles are worked
+ // out per project from its config and text -- see ClassifyVernacularWritingSystems. A
+ // writing system can play both, "en" in lexeme and gloss alike being the usual
+ // case, and one no field claims is analysis. The input system, the config and
+ // the lexicon need not spell the writing system alike, so the roles are looked
+ // up under any spelling of it; see HasRole.
+ if (classification == null)
+ classification = ClassifyForNewWritingSystems(countUsage);
+ bool isVernacular = HasRole(classification.Value.Vernacular, lfWs.Tag);
+ if (isVernacular)
ServiceLocator.LanguageProject.AddToCurrentVernacularWritingSystems(ws);
- else
+ if (!isVernacular || HasRole(classification.Value.Analysis, lfWs.Tag))
ServiceLocator.LanguageProject.AddToCurrentAnalysisWritingSystems(ws);
-
}
}
}
+ ///
+ /// Whether a writing system new to LCM plays this role, under any spelling of its tag. The
+ /// classification spells each writing system as the config and the lexicon do, and the
+ /// input system can spell it otherwise: an LF user who adds "qaa-x-qaa-new" may configure
+ /// fields with "qaa-x-new". Spellings sharing a canonical form can be taken for one writing
+ /// system here, because this one is new: had LCM held any writing system with that canonical
+ /// form, WsIdFromLfTag would have found it. A role found under one spelling therefore counts
+ /// for all of them, even where the text decided otherwise under another: a writing system
+ /// whose text makes it vernacular only, offered by the config in the gloss under another
+ /// spelling, is both. That can only add a role, never take one away.
+ ///
+ private static bool HasRole(ISet role, string tag)
+ {
+ if (role.Contains(tag))
+ return true;
+ string canonical = LanguageTags.Canonical(tag);
+ return role.Any(other => string.Equals(LanguageTags.Canonical(other), canonical, StringComparison.OrdinalIgnoreCase));
+ }
+
+ ///
+ /// Which writing systems this project treats as vernacular and which as analysis, derived
+ /// from its own config and text rather than from languageCode alone; see
+ /// ClassifyVernacularWritingSystems.
+ ///
+ private (ISet Vernacular, ISet Analysis, ISet Unresolved)
+ ClassifyForNewWritingSystems(Func countUsage)
+ {
+ var classification = ClassifyVernacularWritingSystems(ProjectRecord.Config,
+ ProjectRecord.LanguageCode, countUsage?.Invoke());
+ if (classification.Unresolved.Count > 0)
+ {
+ Logger.Notice("MongoToLcm: writing system(s) {0} appear only in config fields whose "
+ + "vernacular/analysis role could not be determined; treating them as both",
+ string.Join(", ", classification.Unresolved));
+ }
+ return classification;
+ }
+
+ ///
+ /// The best string for a single-string LCM field, and the writing system it is in. Null when
+ /// LF holds no text in any writing system LCM has -- which is not the same as holding no text
+ /// at all; see .
+ ///
private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool isAnalysisField = true)
{
if (input == null) return null;
if (input.Count == 0)
{
- Logger.Warning("BestStringAndWsFromMultiText got a non-null multitext, but it was empty. Empty LF MultiText objects should be nulls in Mongo. Unfortunately, at this point in the code it's hard to know which multitext it was.");
+ // Some Language Forge fields, like scientificName on senses, have empty but
+ // non-null multitexts; those should also be empty in LCM.
return null;
}
@@ -303,38 +779,79 @@ private Tuple BestStringAndWsFromMultiText(LfMultiText input, bool
// _analysisWsIdsAndNamesInSearchOrder :
// _vernacularWsIdsAndNamesInSearchOrder;
- foreach (ILgWritingSystem ws in wsesToSearch)
- {
- LfStringField field;
- if (input.TryGetValue(ws.Id, out field) && field != null && !field.IsEmpty)
- {
-// Logger.Debug("Returning TsString from {0} for writing system {1}", field.Value, ws.Id);
- return new Tuple(field.Value, ws.Handle);
- }
- }
-
- // Last-ditch option: just grab the first non-empty string we can find
- KeyValuePair kv = input.WsIdAndFirstNonEmptyString(Cache);
- if (kv.Value == null) return null;
-// Logger.Debug("Returning first non-empty TsString from {0} for writing system with ID {1}",
-// kv.Value, kv.Key);
- return new Tuple(kv.Value, kv.Key);
+ // By handle, not by comparing LF's keys with ws.Id, which would miss a key LF spells
+ // differently from LCM. Falls back to the first value in any writing system LCM knows.
+ KeyValuePair best = input.BestStringAndWsId(
+ wsesToSearch.Select(ws => ws.Handle), ServiceLocator.WritingSystemFactory, _configuredTags);
+ if (best.Value == null)
+ return null;
+ return new Tuple(best.Value, best.Key);
}
- private ITsString BestTsStringFromMultiText(LfMultiText input, bool isAnalysisField = true)
+ ///
+ /// The value to give a single-string LCM field, whose current value is
+ /// .
+ ///
+ /// Null, clearing the field, when LF holds no text for it. But when LF holds text and every
+ /// alternative is in a writing system LCM does not have, there is nowhere to put it, and the
+ /// field keeps its current value. Neither alternative will do: building the string in
+ /// writing system 0 throws, which abandons the rest of the entry half-written, and clearing
+ /// the field would delete what LCM holds on account of text that could not be placed.
+ ///
+ /// The field also keeps its value when LF's text is all under keys passed over for an empty
+ /// key naming the same writing system. That empty key may be a clear made under the config's
+ /// spelling, or the empty value LF's editor fills in for every configured input system an
+ /// entry lacks, beside the export's value under the Id; nothing here tells the two apart, and
+ /// keeping what LCM holds loses nothing that LF can show.
+ ///
+ private ITsString BestTsStringFromMultiText(LfMultiText input, ITsString current, bool isAnalysisField = true)
{
Tuple stringAndWsId = BestStringAndWsFromMultiText(input, isAnalysisField);
- if (stringAndWsId == null)
- return null;
- return ConvertMongoToLcmTsStrings.SpanStrToTsString(stringAndWsId.Item1, stringAndWsId.Item2, ServiceLocator.WritingSystemFactory);
+ if (stringAndWsId != null)
+ return ConvertMongoToLcmTsStrings.SpanStrToTsString(stringAndWsId.Item1, stringAndWsId.Item2, ServiceLocator.WritingSystemFactory);
+ return NothingToPlace(input) ? null : current;
}
- private string BestStringFromMultiText(LfMultiText input, bool isAnalysisField = true)
+ /// As , for a plain string field.
+ private string BestStringFromMultiText(LfMultiText input, string current, bool isAnalysisField = true)
{
Tuple stringAndWsId = BestStringAndWsFromMultiText(input, isAnalysisField);
- if (stringAndWsId == null)
- return null;
- return stringAndWsId.Item1;
+ if (stringAndWsId != null)
+ return stringAndWsId.Item1;
+ return NothingToPlace(input) ? null : current;
+ }
+
+ ///
+ /// Whether LF holds no text at all here. When it does hold text that nevertheless could not
+ /// be placed, says why in the log, since that text is not reaching FieldWorks.
+ ///
+ private bool NothingToPlace(LfMultiText input)
+ {
+ if (input == null || input.IsEmpty)
+ return true;
+ LogTextNotPlaced(input, "MongoToLcm", Logger, ServiceLocator.WritingSystemFactory, _configuredTags);
+ return false;
+ }
+
+ ///
+ /// Logs why each key holding text in a single-string field's multitext was not written
+ /// there. Only for when nothing was: every key with text then either names no writing system
+ /// LCM has or was passed over for an empty key, and the messages say so. Called after a
+ /// value has been placed, it would call the key written empty.
+ ///
+ internal static void LogTextNotPlaced(LfMultiText input, string context, ILogger logger,
+ ILgWritingSystemFactory wsManager, ISet configuredTags)
+ {
+ foreach (KeyValuePair passedOver in input.KeysPassedOver(wsManager, configuredTags))
+ {
+ if (passedOver.Value == null)
+ logger.Warning("{0}: skipping text under \"{1}\" ({2}), which names no writing system LCM has; " +
+ "leaving the field as it was", context, passedOver.Key, input.Excerpt(passedOver.Key));
+ else
+ logger.Warning("{0}: keys \"{1}\" and \"{2}\" name the same writing system, and \"{2}\", the one " +
+ "to write, is empty; leaving the field as it was rather than write \"{1}\" ({3})",
+ context, passedOver.Key, passedOver.Value, input.Excerpt(passedOver.Key));
+ }
}
// This GetOrCreate() function takes an extra out parameter so we can correctly update
@@ -543,6 +1060,9 @@ private DateTime? LastSyncedDate
/// LfMerge's: the initial clone's transfer to Mongo does not always get as far as recording a
/// sync. LfMerge clears the user references on every entry it writes, and LF sets one on every
/// edit and creation; a deletion sets none, but deleting what FieldWorks lacks is harmless.
+ /// A LIFT import into LF sets none either, so this says "not edited" of every imported entry
+ /// in a never-synced LF project; LfLexEntryToLcmLexEntry sends such an entry anyway when
+ /// FieldWorks does not have it.
///
private bool EditedInLfSinceLastSync(LfLexEntry lfEntry)
{
@@ -564,12 +1084,25 @@ private void LfLexEntryToLcmLexEntry(LfLexEntry lfEntry)
if (lfEntry.IsDeleted)
{
// An LF deletion that an earlier sync has already sent
+ return;
}
- else if (!GetInstance().TryGetObject(guid, out existing))
+ if (GetInstance().TryGetObject(guid, out existing))
+ {
+ if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified)
+ _changedOnlyInFieldWorks++;
+ return;
+ }
+ if (LastSyncedDate != null)
+ {
_absentOnlyFromFieldWorks++;
- else if (lfEntry.AuthorInfo == null || lfEntry.AuthorInfo.ModifiedDate.ToLocalTime() != existing.DateModified)
- _changedOnlyInFieldWorks++;
- return;
+ return;
+ }
+ // Never synced, and FieldWorks does not have it: an LF project's own entry, going to
+ // FieldWorks for the first time. A LIFT import into LF sets no user references either,
+ // so EditedInLfSinceLastSync cannot tell it from one LfMerge wrote; but FieldWorks
+ // cannot hold a newer version of an entry it does not have, so there is nothing to
+ // keep. Leaving it out would also lose it: the transfer back to Mongo marks every
+ // entry FieldWorks lacks as deleted.
}
bool createdEntry = false;
bool wantCreation = !lfEntry.IsDeleted;
@@ -693,7 +1226,7 @@ private void LfExampleToLcmExample(LfExample lfExample, ILexSense owner)
// LcmExample.Guid,
// LcmExample.Hvo
// );
- LcmExample.Reference = BestTsStringFromMultiText(lfExample.Reference);
+ LcmExample.Reference = BestTsStringFromMultiText(lfExample.Reference, LcmExample.Reference);
ICmTranslation t = FindOrCreateTranslationByGuid(lfExample.TranslationGuid, LcmExample,
_freeTranslationType);
SetMultiStringFrom(t.Translation, lfExample.Translation);
@@ -719,9 +1252,17 @@ private void LfPictureToLcmPicture(LfPicture lfPicture, ILexSense owner)
string caption = "";
if (lfPicture.Caption != null)
{
- KeyValuePair kv = lfPicture.Caption.WsIdAndFirstNonEmptyString(Cache);
- captionWs = kv.Key;
- caption = kv.Value;
+ // A caption in no writing system LCM has would come back as writing system 0, which
+ // cannot be built into a string; the picture then starts with an empty caption, and
+ // SetMultiStringFrom below fills in every alternative that can be placed.
+ KeyValuePair kv = lfPicture.Caption.BestStringAndWsId(
+ ServiceLocator.LanguageProject.AnalysisWritingSystems.Select(ws => ws.Handle),
+ ServiceLocator.WritingSystemFactory, _configuredTags);
+ if (kv.Value != null)
+ {
+ captionWs = kv.Key;
+ caption = kv.Value;
+ }
}
// Lcm expects internal pictures in a certain path. If an external path already
@@ -789,19 +1330,19 @@ private void LfSenseToLcmSense(LfSense lfSense, ILexEntry owner)
SetMultiStringFrom(LcmSense.PhonologyNote, lfSense.PhonologyNote);
// LcmSense.ReversalEntriesRC = lfSense.ReversalEntries; // TODO: More complex than
// that. Handle it correctly. Maybe.
- LcmSense.ScientificName = BestTsStringFromMultiText(lfSense.ScientificName);
+ LcmSense.ScientificName = BestTsStringFromMultiText(lfSense.ScientificName, LcmSense.ScientificName);
ListConverters[SemDomListCode].UpdatePossibilitiesFromStringArray(LcmSense.SemanticDomainsRC,
lfSense.SemanticDomain);
SetMultiStringFrom(LcmSense.SemanticsNote, lfSense.SemanticsNote);
SetMultiStringFrom(LcmSense.Bibliography, lfSense.SenseBibliography);
// lfSense.SenseId; // TODO: What do I do with this one?
- LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue);
+ LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue, LcmSense.ImportResidue);
SetMultiStringFrom(LcmSense.Restrictions, lfSense.SenseRestrictions);
LcmSense.SenseTypeRA = ListConverters[SenseTypeListCode].FromStringField(lfSense.SenseType);
SetMultiStringFrom(LcmSense.SocioLinguisticsNote, lfSense.SociolinguisticsNote);
- LcmSense.Source = BestTsStringFromMultiText(lfSense.Source);
+ LcmSense.Source = BestTsStringFromMultiText(lfSense.Source, LcmSense.Source);
LcmSense.StatusRA = ListConverters[StatusListCode].FromStringArrayFieldWithOneCase(lfSense.Status);
ListConverters[UsageTypeListCode].UpdatePossibilitiesFromStringArray(LcmSense.UsageTypesRC,
lfSense.Usages);
@@ -903,7 +1444,15 @@ private void SetMultiStringFrom(IMultiStringAccessor dest, LfMultiText source)
if (source == null)
ClearMultiString(dest);
else
- source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager);
+ source.WriteToLcmMultiString(dest, ServiceLocator.WritingSystemManager, _configuredTags,
+ // Text under a key that names no writing system LCM has never reaches FieldWorks,
+ // and leaves LF too once FieldWorks next changes the entry and it is exported again
+ tag => Logger.Warning(
+ "MongoToLcm: skipping text under \"{0}\" ({1}), which names no writing system LCM has",
+ tag, source.Excerpt(tag)),
+ (notWritten, written) => Logger.Warning(
+ "MongoToLcm: keys \"{0}\" and \"{1}\" name the same writing system; wrote \"{1}\" ({2}), not \"{0}\" ({3})",
+ notWritten, written, source.Excerpt(written), source.Excerpt(notWritten)));
}
///
@@ -952,7 +1501,7 @@ private void SetEtymologyFields(ILexEntry LcmEntry, LfLexEntry lfEntry)
SetMultiStringFrom(LcmEtymology.Gloss, lfEntry.EtymologyGloss);
if (lfEntry.EtymologySource != null)
#if DBVERSION_7000068
- LcmEtymology.Source = BestStringFromMultiText(lfEntry.EtymologySource);
+ LcmEtymology.Source = BestStringFromMultiText(lfEntry.EtymologySource, LcmEtymology.Source);
#else
SetMultiStringFrom(LcmEtymology.LanguageNotes, lfEntry.EtymologySource);
#endif
@@ -994,8 +1543,8 @@ private void SetPronunciation(ILexEntry LcmEntry, LfLexEntry lfEntry)
LcmPronunciation = GetInstance().Create();
LcmEntry.PronunciationsOS.Add(LcmPronunciation);
}
- LcmPronunciation.CVPattern = BestTsStringFromMultiText(lfEntry.CvPattern);
- LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone);
+ LcmPronunciation.CVPattern = BestTsStringFromMultiText(lfEntry.CvPattern, LcmPronunciation.CVPattern);
+ LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone, LcmPronunciation.Tone);
SetMultiStringFrom(LcmPronunciation.Form, lfEntry.Pronunciation);
LcmPronunciation.LocationRA =
(ICmLocation)ListConverters[LocationListCode].FromStringField(lfEntry.Location);
diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs
index e33a4f96..1edbc320 100644
--- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs
+++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.Collections.Generic;
@@ -289,7 +289,7 @@ public void PopulateCmPossibilityFromOptionListItem(ICmPossibility poss, LfOptio
// Should only be called when NO canonical item can be found
if (wsStr == null)
wsStr = "en"; // TODO: Set this from LF user's writing system rather than English, once LF allows setting interface languages
- int wsId = poss.Cache.WritingSystemFactory.GetWsFromStr(wsStr);
+ int wsId = LanguageTags.WsIdFromLfTag(poss.Cache.WritingSystemFactory, wsStr);
poss.Abbreviation.set_String(wsId, item.Abbreviation);
poss.Name.set_String(wsId, item.Value);
}
diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs
index cd1b1b7f..56d8242f 100644
--- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs
+++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmTsStrings.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.Collections.Generic;
@@ -89,7 +89,16 @@ public static ITsString SpanStrToTsString(string source, int mainWs, ILgWritingS
builder.ClearProps(); // Make sure there aren't leftover properties from previous run
// To remove a string property, you set it to null, so we can just use StyleName directly whether or not it's null.
builder.SetStrPropValue((int)FwTextPropType.ktptNamedStyle, run.StyleName);
- int runWs = (run.Lang == null) ? mainWs : wsf.GetWsFromStr(run.Lang);
+ int runWs = (run.Lang == null) ? mainWs : LanguageTags.WsIdFromLfTag(wsf, run.Lang);
+ if (runWs == 0)
+ {
+ // A lang naming no writing system LCM has cannot be built into the string, and the
+ // exception that follows abandons the whole entry half-written. The run's text is
+ // kept in the string's own writing system; only its language mark is lost.
+ MainClass.Logger?.Warning("MongoToLcm: span lang \"{0}\" names no writing system LCM has; " +
+ "keeping \"{1}\" in the field's writing system", run.Lang, run.Content);
+ runWs = mainWs;
+ }
builder.SetIntPropValues((int)FwTextPropType.ktptWs, (int)FwTextPropVar.ktpvDefault, runWs);
// We don't care about Guids in this function, so run.Guid is ignored
// But we do need to set any other int or string properties that were in the original
diff --git a/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs
new file mode 100644
index 00000000..62d213a6
--- /dev/null
+++ b/src/LfMerge.Core/DataConverters/LfWritingSystemUsage.cs
@@ -0,0 +1,142 @@
+// Copyright (c) 2016-2018 SIL International
+// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
+using System.Collections.Generic;
+using LfMerge.Core.LanguageForge.Model;
+
+namespace LfMerge.Core.DataConverters
+{
+ ///
+ /// How much text each writing system actually holds in each Language Forge field, counted over
+ /// a project's lexicon.
+ ///
+ /// The project config says which writing systems a field OFFERS, which is a much weaker claim
+ /// than it looks: Language Forge routinely offers every input system in every field, so a tag
+ /// configured for the lexeme field may well have no headword in it anywhere. Counting the text
+ /// tells us what a writing system is really being used for, which is what
+ /// ConvertMongoToLcmLexicon.ClassifyVernacularWritingSystems needs to tell a vernacular writing
+ /// system from an analysis one.
+ ///
+ /// Fields are keyed by the dotted path the config spells them with, nesting through "fields"
+ /// ("senses.fields.examples.fields.sentence"), so the counts line up with the paths that
+ /// classification already works in.
+ ///
+ public class LfWritingSystemUsage
+ {
+ // Field paths, spelled as the config spells them.
+ public const string Lexeme = "lexeme";
+ public const string CitationForm = "citationForm";
+ public const string Definition = "senses.fields.definition";
+ public const string Gloss = "senses.fields.gloss";
+ public const string ExampleSentence = "senses.fields.examples.fields.sentence";
+ public const string ExampleTranslation = "senses.fields.examples.fields.translation";
+ public const string Note = "note";
+ public const string LiteralMeaning = "literalMeaning";
+ public const string Pronunciation = "pronunciation";
+
+ private readonly Dictionary> _countsByPath =
+ new Dictionary>();
+
+ /// How many entries were counted; zero means we have no evidence to go on.
+ public int EntriesCounted { get; private set; }
+
+ /// The number of non-empty strings this writing system holds in this field.
+ public int NonEmptyCount(string fieldPath, string tag)
+ {
+ Dictionary byTag;
+ int count;
+ if (_countsByPath.TryGetValue(fieldPath, out byTag) && byTag.TryGetValue(tag, out count))
+ return count;
+ return 0;
+ }
+
+ ///
+ /// The total number of non-empty strings this writing system holds across the given fields.
+ ///
+ public int NonEmptyCount(IEnumerable fieldPaths, string tag)
+ {
+ int total = 0;
+ foreach (string path in fieldPaths)
+ total += NonEmptyCount(path, tag);
+ return total;
+ }
+
+ /// The writing systems that hold any text in this field.
+ public IEnumerable TagsWithText(string fieldPath)
+ {
+ Dictionary byTag;
+ if (!_countsByPath.TryGetValue(fieldPath, out byTag))
+ return new string[0];
+ return byTag.Keys;
+ }
+
+ ///
+ /// Counts the text in the fields classification consults: the vernacular and analysis
+ /// anchors and the pronunciation field. Counting every other field too would only lengthen
+ /// a pass over the whole lexicon for numbers nothing reads. Deleted entries are skipped:
+ /// they are tombstones Language Forge keeps so the deletion can be replayed, and their text
+ /// is not evidence of anything.
+ ///
+ public static LfWritingSystemUsage FromLexicon(IEnumerable lexicon)
+ {
+ var usage = new LfWritingSystemUsage();
+ if (lexicon == null)
+ return usage;
+ foreach (LfLexEntry entry in lexicon)
+ {
+ if (entry == null || entry.IsDeleted)
+ continue;
+ usage.EntriesCounted++;
+ usage.Add(Lexeme, entry.Lexeme);
+ usage.Add(CitationForm, entry.CitationForm);
+ usage.Add(Note, entry.Note);
+ usage.Add(LiteralMeaning, entry.LiteralMeaning);
+ usage.Add(Pronunciation, entry.Pronunciation);
+ if (entry.Senses == null)
+ continue;
+ foreach (LfSense sense in entry.Senses)
+ {
+ if (sense == null)
+ continue;
+ usage.Add(Definition, sense.Definition);
+ usage.Add(Gloss, sense.Gloss);
+ if (sense.Examples == null)
+ continue;
+ foreach (LfExample example in sense.Examples)
+ {
+ if (example == null)
+ continue;
+ usage.Add(ExampleSentence, example.Sentence);
+ usage.Add(ExampleTranslation, example.Translation);
+ }
+ }
+ }
+ return usage;
+ }
+
+ ///
+ /// Counts one field's non-empty alternatives. Whitespace counts as empty; span markup is
+ /// not stripped first, since a string made only of markup carries no text to begin with.
+ ///
+ private void Add(string fieldPath, LfMultiText text)
+ {
+ if (text == null)
+ return;
+ foreach (KeyValuePair kv in text)
+ {
+ if (kv.Value == null || string.IsNullOrWhiteSpace(kv.Value.Value))
+ continue;
+ Dictionary byTag;
+ if (!_countsByPath.TryGetValue(fieldPath, out byTag))
+ {
+ // Case-insensitively, as every LCM writing system lookup is: the config and the
+ // lexicon do not always agree on the case of a tag, and they mean the same one.
+ byTag = new Dictionary(System.StringComparer.OrdinalIgnoreCase);
+ _countsByPath[fieldPath] = byTag;
+ }
+ int count;
+ byTag.TryGetValue(kv.Key, out count);
+ byTag[kv.Key] = count + 1;
+ }
+ }
+ }
+}
diff --git a/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs b/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs
new file mode 100644
index 00000000..21c50e88
--- /dev/null
+++ b/src/LfMerge.Core/LanguageForge/Model/CustomMongoSerializerForInt32.cs
@@ -0,0 +1,24 @@
+// Copyright (c) 2026 SIL Global
+// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
+
+using MongoDB.Bson;
+using MongoDB.Bson.IO;
+using MongoDB.Bson.Serialization;
+using MongoDB.Bson.Serialization.Serializers;
+
+namespace LfMerge.Core.LanguageForge.Model
+{
+ public class CustomMongoSerializerForInt32 : Int32Serializer
+ {
+ public override int Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
+ {
+ IBsonReader bsonReader = context.Reader;
+ if (bsonReader.GetCurrentBsonType() == BsonType.Null)
+ {
+ bsonReader.ReadNull();
+ return 0;
+ }
+ return base.Deserialize(context, args);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs
index 28fb02a6..92ea82e1 100644
--- a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs
+++ b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016 SIL International
+// Copyright (c) 2016 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.Collections.Generic;
@@ -19,6 +19,9 @@ public class LfLexEntry : LfFieldBase, IHasNullableGuid
public string MercurialSha { get; set; }
public DateTime DateCreated { get; set; }
public DateTime DateModified { get; set; }
+ [BsonDefaultValue(0)]
+ // CustomMongoSerializerForInt32 converts null to 0
+ [BsonSerializer(typeof(CustomMongoSerializerForInt32))]
public int DirtySR { get; set ; }
// Data properties
diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs
index 96dd29dd..02e6b5d9 100644
--- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs
+++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs
@@ -1,4 +1,4 @@
-// Copyright (c) 2016-2018 SIL International
+// Copyright (c) 2016-2018 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.Linq;
@@ -112,13 +112,40 @@ public string FirstNonEmptyString()
return (result == null) ? null : result.Value;
}
- public KeyValuePair WsIdAndFirstNonEmptyString(LcmCache cache)
+ ///
+ /// The value for the first writing system in wsSearchOrder that has a non-empty one, falling
+ /// back to the first non-empty value in any writing system LCM knows, together with that
+ /// writing system's handle. Keys are resolved to handles before they are compared, so a key
+ /// LF spells differently from LCM's id for the writing system still matches it.
+ ///
+ /// Where two keys name one writing system, the value is the one WriteToLcm would write: the
+ /// key spelled as the config spells it, or else the later one. That is chosen before empty
+ /// values are left out, so a value an LF user has cleared under the config's spelling is
+ /// not replaced by the export's hidden, stale one.
+ ///
+ ///
+ /// Every writing-system spelling the project's config uses; see WriteToLcm.
+ ///
+ /// The handle and the value, or (0, null) when no non-empty value resolves.
+ public KeyValuePair BestStringAndWsId(IEnumerable wsSearchOrder, ILgWritingSystemFactory wsManager,
+ ISet configuredTags = null)
{
- KeyValuePair kv = FirstNonEmptyKeyValue();
- if (kv.Key == null) return new KeyValuePair();
- ILgWritingSystemFactory wsManager = cache.ServiceLocator.WritingSystemManager;
- int wsId = wsManager.GetWsFromStr(kv.Key);
- return new KeyValuePair(wsId, kv.Value);
+ var resolved = new List>();
+ foreach (KeyValuePair chosen in ChooseKeys(wsManager, configuredTags))
+ {
+ LfStringField field = this[chosen.Value];
+ if (field != null && !field.IsEmpty)
+ resolved.Add(new KeyValuePair(chosen.Key, field.Value));
+ }
+ foreach (int wsId in wsSearchOrder)
+ {
+ foreach (KeyValuePair candidate in resolved)
+ {
+ if (candidate.Key == wsId)
+ return candidate;
+ }
+ }
+ return resolved.FirstOrDefault();
}
public KeyValuePair FirstNonEmptyKeyValue()
@@ -129,24 +156,171 @@ public KeyValuePair FirstNonEmptyKeyValue()
new KeyValuePair(result.Key, result.Value.Value);
}
- public void WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager)
+ public bool WriteToLcmMultiString(IMultiAccessorBase dest, ILgWritingSystemFactory wsManager,
+ ISet configuredTags = null, Action onUnidentifiedTag = null,
+ Action onKeyNotWritten = null)
{
if (dest == null)
- return;
- HashSet destWsIdsToClear = new HashSet(dest.AvailableWritingSystemIds);
+ return false;
+ return WriteToLcm(dest.AvailableWritingSystemIds, dest.get_String, dest.set_String, wsManager,
+ configuredTags, onUnidentifiedTag, onKeyNotWritten);
+ }
+
+ ///
+ /// Writes every alternative into an LCM multistring, then clears each alternative LCM holds
+ /// that has no key here. The built-in multitext fields and the MultiUnicode custom fields
+ /// both come through here, reaching LCM through an IMultiAccessorBase and through
+ /// ISilDataAccess respectively, which is why LCM is reached through delegates.
+ ///
+ /// Clearing is right because the LCM-to-Mongo direction exports every alternative LCM holds:
+ /// one missing from LF is one a user removed there, not one LF never saw.
+ ///
+ /// Each key is resolved with LanguageTags.WsIdFromLfTag. Resolving it any other way would
+ /// skip a non-canonical key as unidentified, and the clearing pass would then blank that
+ /// writing system's text in LCM: the data would not merely be dropped but deleted.
+ ///
+ /// An alternative whose value has not changed is left alone, so it stays out of the .fwdata
+ /// XML and out of the Mercurial commit.
+ ///
+ /// Two keys can name the same writing system, spelled two ways that both resolve to it, and
+ /// only one can be written. That happens when LF's config spells a writing system otherwise
+ /// than LfMerge's export: the export keys every value by the writing system's Id, an LF user
+ /// edits under the config's spelling, and the export's value stays in the entry, hidden
+ /// from the editor, until the entry is next exported -- which, the edit once written to LCM,
+ /// may be a long time. In spt-flex, whose config says "hi-IN" for the Id "hi-Deva-IN", an
+ /// edited entry holds both. The value to write is the one under the spelling the config
+ /// uses, since that is what LF's editor shows and so the one a user can have edited; the
+ /// other is the hidden export, or, after the config has been re-spelled, an edit made under
+ /// the old spelling that has already been written. Where both spellings or neither are in
+ /// the config, it is the later key, as it always was: LF adds a new key after the existing
+ /// ones.
+ ///
+ /// The writing systems LCM currently holds alternatives for.
+ /// Reads LCM's alternative for a writing system.
+ /// Writes LCM's alternative for a writing system.
+ /// Resolves LF's keys to LCM writing systems.
+ ///
+ /// Every writing-system spelling the project's config uses, to choose between two keys
+ /// naming one writing system; null to go on key order alone.
+ ///
+ /// Told each key that resolves to no writing system.
+ ///
+ /// Told each key left unwritten because another key names the same writing system, and the
+ /// key written instead.
+ ///
+ /// Whether anything in LCM changed.
+ public bool WriteToLcm(IEnumerable existingWsIds, Func getAlternative,
+ Action setAlternative, ILgWritingSystemFactory wsManager,
+ ISet configuredTags = null, Action onUnidentifiedTag = null,
+ Action onKeyNotWritten = null)
+ {
+ // Each writing system's key, chosen before anything is written
+ List> keyFor = ChooseKeys(wsManager, configuredTags, onUnidentifiedTag, onKeyNotWritten);
+
+ var wsIdsToClear = new HashSet(existingWsIds);
+ bool changed = false;
+ foreach (KeyValuePair chosen in keyFor)
+ {
+ int wsId = chosen.Key;
+ LfStringField field = this[chosen.Value];
+ wsIdsToClear.Remove(wsId);
+ string text = (field == null) ? string.Empty : (field.Value ?? string.Empty);
+ ITsString newValue = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(text, wsId, wsManager);
+ ITsString oldValue = getAlternative(wsId);
+ // GetDiffsInTsStrings() returns null when there are no changes
+ if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null)
+ continue;
+ setAlternative(wsId, newValue);
+ changed = true;
+ }
+ foreach (int wsId in wsIdsToClear)
+ {
+ ITsString oldValue = getAlternative(wsId);
+ if (oldValue == null || string.IsNullOrEmpty(oldValue.Text))
+ continue;
+ setAlternative(wsId, TsStringUtils.EmptyString(wsId));
+ changed = true;
+ }
+ return changed;
+ }
+
+ /// The start of one key's text, enough to find the entry it is in.
+ public string Excerpt(string key)
+ {
+ LfStringField field;
+ string text = (TryGetValue(key, out field) && field != null) ? (field.Value ?? string.Empty) : string.Empty;
+ return text.Length <= 40 ? text : text.Substring(0, 40) + "...";
+ }
+
+ ///
+ /// The keys holding text that passes over, each with the
+ /// reason: null where the key names no writing system LCM has, or else the key naming the
+ /// same writing system that is chosen over it.
+ ///
+ public List> KeysPassedOver(ILgWritingSystemFactory wsManager,
+ ISet configuredTags = null)
+ {
+ var chosenFor = ChooseKeys(wsManager, configuredTags).ToDictionary(kv => kv.Key, kv => kv.Value);
+ var passedOver = new List>();
foreach (KeyValuePair kv in this)
{
- int wsId = wsManager.GetWsFromStr(kv.Key);
- if (wsId == 0) continue; // Skip any unidentified writing systems
- string value = kv.Value.Value;
- ITsString tss = LfMerge.Core.DataConverters.ConvertMongoToLcmTsStrings.SpanStrToTsString(value, wsId, wsManager);
- dest.set_String(wsId, tss);
- destWsIdsToClear.Remove(wsId);
+ if (kv.Value == null || kv.Value.IsEmpty)
+ continue;
+ int wsId = LanguageTags.WsIdFromLfTag(wsManager, kv.Key);
+ if (wsId == 0)
+ passedOver.Add(new KeyValuePair(kv.Key, null));
+ else if (chosenFor[wsId] != kv.Key)
+ passedOver.Add(new KeyValuePair(kv.Key, chosenFor[wsId]));
}
- foreach (int wsId in destWsIdsToClear)
+ return passedOver;
+ }
+
+ ///
+ /// Each writing system's key, in the order the writing systems first appear: the only key
+ /// naming it, or of two, the one picks.
+ ///
+ private List> ChooseKeys(ILgWritingSystemFactory wsManager, ISet configuredTags,
+ Action onUnidentifiedTag = null, Action onKeyNotWritten = null)
+ {
+ var keyFor = new Dictionary();
+ var order = new List();
+ foreach (string key in Keys)
+ {
+ int wsId = LanguageTags.WsIdFromLfTag(wsManager, key);
+ if (wsId == 0)
+ {
+ onUnidentifiedTag?.Invoke(key);
+ continue;
+ }
+ string other;
+ if (keyFor.TryGetValue(wsId, out other))
+ {
+ string kept = PreferredKey(other, key, configuredTags);
+ onKeyNotWritten?.Invoke(kept == key ? other : key, kept);
+ keyFor[wsId] = kept;
+ }
+ else
+ {
+ keyFor[wsId] = key;
+ order.Add(wsId);
+ }
+ }
+ return order.Select(wsId => new KeyValuePair(wsId, keyFor[wsId])).ToList();
+ }
+
+ ///
+ /// Of two keys naming one writing system, the one to write: the one spelled as the config
+ /// spells it, or else the later one. LF's editor matches spellings exactly, so this does too.
+ ///
+ private static string PreferredKey(string earlier, string later, ISet configuredTags)
+ {
+ if (configuredTags != null)
{
- dest.set_String(wsId, string.Empty);
+ bool earlierConfigured = configuredTags.Contains(earlier);
+ if (earlierConfigured != configuredTags.Contains(later))
+ return earlierConfigured ? earlier : later;
}
+ return later;
}
}
diff --git a/src/LfMerge.Core/LanguageTags.cs b/src/LfMerge.Core/LanguageTags.cs
new file mode 100644
index 00000000..620c47b0
--- /dev/null
+++ b/src/LfMerge.Core/LanguageTags.cs
@@ -0,0 +1,99 @@
+using System;
+using System.Linq;
+using SIL.LCModel.Core.KernelInterfaces;
+using SIL.LCModel.Core.WritingSystems;
+using SIL.WritingSystems;
+
+namespace LfMerge.Core
+{
+ public static class LanguageTags
+ {
+ ///
+ /// The canonical form of a language tag: the id LCM knows a writing system by, once it has
+ /// created one from this tag.
+ ///
+ /// Language Forge stores a tag exactly as it was typed. LCM canonicalizes it when the writing
+ /// system is created -- WritingSystemManager.GetOrSet and Create both do -- and thereafter
+ /// GetWsFromStr matches that canonical id literally. It is case-insensitive, so a difference
+ /// of case resolves on its own, but it is not structure-aware, so a tag that loses or absorbs
+ /// a subtag is never found:
+ ///
+ /// qaa-x-qaa-v -> qaa-x-v the private-use section repeats the "qaa" language subtag,
+ /// and canonicalizing absorbs it
+ /// th-Thai -> th Thai's suppress-script is dropped
+ ///
+ /// Every lookup of an LF tag, LfWsToLcmWs's included, should go through
+ /// rather than calling this and GetWsFromStr itself. GetOrSet is
+ /// no substitute: it looks a tag up only as an Id.
+ ///
+ ///
+ /// The canonical tag, or the tag unchanged when it is empty or cannot be parsed --
+ /// IetfLanguageTag.Canonicalize throws on those, and an unparseable tag should go on failing
+ /// where it failed before rather than here.
+ ///
+ public static string Canonical(string tag)
+ {
+ if (string.IsNullOrEmpty(tag) || !IetfLanguageTag.IsValid(tag))
+ {
+ return tag;
+ }
+ return IetfLanguageTag.Canonicalize(tag);
+ }
+
+ ///
+ /// The handle of the LCM writing system a Language Forge tag names, or 0 when there is none.
+ ///
+ /// Every place that resolves a tag LF supplied -- a multitext key, a span's lang attribute,
+ /// a multi-paragraph's input system -- goes through here, so that a non-canonical spelling
+ /// resolves the same way everywhere.
+ ///
+ /// LCM knows a writing system by its Id, which is whatever its .ldml file is named, and also
+ /// has its LanguageTag, which is always canonical. The two differ in older projects:
+ /// brb-flex-2022 holds "km-Khmr-KH.ldml", whose identity is km plus KH, so its Id is
+ /// "km-Khmr-KH" and its LanguageTag "km-KH". LfMerge writes LF's multitext keys by Id, but it
+ /// used to write the project's input systems by LanguageTag, and LF users type under the
+ /// spelling the config gives, so LF can hand back either spelling. Tried in turn:
+ ///
+ /// 1. The tag as LF spells it, as an Id. This must come first. brb-flex-2022 also holds a
+ /// second, unused writing system whose Id is "km-KH"; trying the canonical form first
+ /// sent the text under "km-Khmr-KH" there, and WriteToLcm then cleared the real
+ /// writing system's text as missing from LF.
+ /// 2. Its canonical form, as an Id: the Id LCM gives any writing system it creates.
+ /// 3. Its canonical form as a LanguageTag, for an Id spelled some other way:
+ /// spt-flex's "hi-IN" is the writing system LCM holds as "hi-Deva-IN". Should several
+ /// writing systems share the LanguageTag, the one with the lowest handle wins: LCM
+ /// numbers writing systems in the order it loads them, not the order they were made.
+ ///
+ public static int WsIdFromLfTag(ILgWritingSystemFactory wsf, string tag)
+ {
+ if (string.IsNullOrEmpty(tag))
+ {
+ return 0;
+ }
+ int wsId = wsf.GetWsFromStr(tag);
+ if (wsId != 0)
+ {
+ return wsId;
+ }
+ string canonical = Canonical(tag);
+ if (!string.Equals(canonical, tag, StringComparison.OrdinalIgnoreCase))
+ {
+ wsId = wsf.GetWsFromStr(canonical);
+ if (wsId != 0)
+ {
+ return wsId;
+ }
+ }
+ var wsManager = wsf as WritingSystemManager;
+ if (wsManager == null)
+ {
+ return 0;
+ }
+ CoreWritingSystemDefinition byLanguageTag = wsManager.WritingSystems
+ .Where(ws => string.Equals(ws.LanguageTag, canonical, StringComparison.OrdinalIgnoreCase))
+ .OrderBy(ws => ws.Handle)
+ .FirstOrDefault();
+ return (byLanguageTag == null) ? 0 : byLanguageTag.Handle;
+ }
+ }
+}