Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
6e83222
Add failing tests for non-canonical writing system tags
rmunn Aug 13, 2026
dee46fd
Resolve lexicon multitext keys by canonical language tag
rmunn Aug 14, 2026
2703944
Use GetOrSet instead of TryGet in LfWsToLcmWs
rmunn Aug 14, 2026
2088096
Identify vernacular writing systems from the vernacular config fields
rmunn Aug 14, 2026
ba1ac3d
Treat missing or null DirtySR value as 0
rmunn Aug 18, 2026
5039fbc
Classify vernacular writing systems per project, not by a fixed list
rmunn Aug 24, 2026
6c45cfd
Write MultiUnicode custom fields from LF into LCM
rmunn Aug 25, 2026
d90c0b9
Silence spammy log warning we no longer need
rmunn Aug 25, 2026
e111c60
Resolve MultiUnicode custom field keys by canonical language tag
rmunn Sep 28, 2026
920dc44
Classify analysis writing systems too, and let one play both roles
rmunn Sep 28, 2026
8900058
Resolve every LF writing system tag through one helper
rmunn Sep 28, 2026
e69494b
Count the text each writing system holds, field by field
rmunn Sep 29, 2026
9badbf2
Classify a writing system by where its text actually is
rmunn Sep 29, 2026
a6a5810
Take the example sentence, translation, note and literal meaning as e…
rmunn Sep 29, 2026
99315f0
Settle a writing system by the company it keeps
rmunn Sep 29, 2026
323391a
Do not put an already-placed writing system in doubt
rmunn Sep 30, 2026
c66141f
Leave a single-string field alone when LF's text cannot be placed
rmunn Sep 30, 2026
af04bb8
Resolve an LF tag by its exact Id first, then by LanguageTag
rmunn Sep 30, 2026
7d64415
Find an input system's writing system before creating one
rmunn Sep 30, 2026
9491fac
Stop re-registering existing writing systems on every sync
rmunn Sep 30, 2026
2377f89
Spell input systems the way multitext keys are spelled
rmunn Sep 30, 2026
3bde15d
Leave out a listed writing system that LCM does not have
rmunn Oct 2, 2026
c373e03
Keep the language of a tag that is private-use from the start
rmunn Oct 2, 2026
d618544
Report a custom field LfMerge cannot write once per sync
rmunn Oct 2, 2026
7b76f8f
Do not count a field offering both roles as vernacular
rmunn Oct 5, 2026
26aaee8
Make a writing system with pronunciation text vernacular
rmunn Oct 5, 2026
7ae3016
Count the lexicon only when a writing system is new to LCM
rmunn Oct 5, 2026
f0da10e
Classify config fields without regard to their order
rmunn Oct 5, 2026
ef2c7fc
Test a writing system and its LanguageTag namesake round trip
rmunn Oct 5, 2026
5392d82
Write the config's spelling when two keys name one writing system
rmunn Oct 5, 2026
9b914f5
Look a new writing system's roles up under any spelling of its tag
rmunn Oct 5, 2026
d4fc7ac
Derive the writing systems of one role only in one place
rmunn Oct 5, 2026
ce389f3
Warn when a built-in field's text names no writing system LCM has
rmunn Oct 5, 2026
7dde0d8
Say what the pronunciation rule does to a custom-field writing system
rmunn Oct 5, 2026
969b7ff
Bring comments the code review found out of date back in line
rmunn Oct 5, 2026
3540fd9
Make three tests test what they say they test
rmunn Oct 5, 2026
88d426a
Ignore a null or empty tag in a config field's input systems
rmunn Oct 5, 2026
fa97a16
Choose between two spellings in single-string fields as multitexts do
rmunn Oct 5, 2026
f77493c
Keep a span's text when its lang names no writing system LCM has
rmunn Oct 5, 2026
28e2cc5
Flag an input system as in a list whatever case the list spells it in
rmunn Oct 5, 2026
f7f7f16
Bring two more comments the code review found out of date in line
rmunn Oct 5, 2026
be87e86
Say why a single-string field's text was not written
rmunn Oct 5, 2026
26c2bb3
Test that the config's spellings reach every writer, through a sync
rmunn Oct 5, 2026
ac41521
Make the pronunciation custom-field test fail without the rule it tests
rmunn Oct 5, 2026
54ef816
Test that the lexicon is counted once however many writing systems ar…
rmunn Oct 5, 2026
6760be9
Bring the last comments the code review found out of date in line
rmunn Oct 6, 2026
96f584d
Delete the dead FW8_COMPAT blocks
rmunn Oct 6, 2026
499b2ea
Add a mongosh script to re-spell LF input systems as LCM Ids
rmunn Oct 6, 2026
b91108e
Remove the migration script from the repository
rmunn Oct 6, 2026
1a174b5
Handle any hg versions in create-tarball script
rmunn Nov 12, 2026
2e748f8
Send a never-synced LF project's entries FieldWorks lacks
rmunn Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 5 additions & 47 deletions docker/scripts/create-installation-tarball.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,53 +52,11 @@ install -m 755 output/${BUILD}/${FRAMEWORK}/LfMerge ${DBDESTDIR}/${LIB} 2>/dev/n
install -m 755 output/${BUILD}/${FRAMEWORK}/LfMergeQueueManager ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/LfMergeQueueManager ${DBDESTDIR}/${LIB}
install -m 755 output/${BUILD}/${FRAMEWORK}/chorusmerge ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/chorusmerge ${DBDESTDIR}/${LIB}
chmod 755 ${DBDESTDIR}/${LIB}/FixFwData.exe
install -d ${DBDESTDIR}/${LIB}/Mercurial
install -d ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport
install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext
install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert
install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight
install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles
install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils
install -d ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils
install -d ${DBDESTDIR}/${LIB}/MercurialExtensions
install -d ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8
install -m 755 Mercurial/chg ${DBDESTDIR}/${LIB}/Mercurial
install -m 755 Mercurial/hg ${DBDESTDIR}/${LIB}/Mercurial
install -m 644 Mercurial/mercurial.ini ${DBDESTDIR}/${LIB}/Mercurial
install -m 644 Mercurial/hgdemandimport/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgdemandimport
install -m 644 Mercurial/hgext/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext
install -m 644 Mercurial/hgext/convert/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert
install -m 644 Mercurial/hgext/highlight/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/highlight
install -m 644 Mercurial/hgext/largefiles/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/largefiles
install -m 644 Mercurial/hgext/zeroconf/*.* ${DBDESTDIR}/${LIB}/Mercurial/hgext/zeroconf
install -m 644 Mercurial/mercurial/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial
install -m 644 Mercurial/mercurial/defaultrc/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/defaultrc
install -m 644 Mercurial/mercurial/dirstateutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/dirstateutils
install -m 644 Mercurial/mercurial/hgweb/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/hgweb
install -m 644 Mercurial/mercurial/httpclient/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/httpclient || true
install -m 644 Mercurial/mercurial/interfaces/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/interfaces
install -m 644 Mercurial/mercurial/pure/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/pure
install -m 644 Mercurial/mercurial/revlogutils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/revlogutils
install -m 644 Mercurial/mercurial/stabletailgraph/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/stabletailgraph
install -m 644 Mercurial/mercurial/testing/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/testing
install -m 644 Mercurial/mercurial/thirdparty/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty
install -m 644 Mercurial/mercurial/thirdparty/attr/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/thirdparty/attr
install -m 644 Mercurial/mercurial/utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/utils
install -m 644 Mercurial/mercurial/upgrade_utils/*.* ${DBDESTDIR}/${LIB}/Mercurial/mercurial/upgrade_utils
install -m 644 MercurialExtensions/fixutf8/*.* ${DBDESTDIR}/${LIB}/MercurialExtensions/fixutf8
cp -a Mercurial ${DBDESTDIR}/${LIB}/
cp -a MercurialExtensions ${DBDESTDIR}/${LIB}/
# Ensure hg and chg are world-executable
chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/hg
chmod 755 ${DBDESTDIR}/${LIB}/Mercurial/chg
install -d ${DBDESTDIR}/${LIB}/runtimes
install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64
install -d ${DBDESTDIR}/${LIB}/runtimes/linux-x64/lib
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
// Copyright (c) 2016-2018 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)

using System;
using System.Linq;
using LfMerge.Core.DataConverters;
using MongoDB.Bson;
using NUnit.Framework;
using SIL.LCModel;

namespace LfMerge.Core.Tests.Lcm.DataConverters
{
/// <summary>
/// Writing custom fields from LF into LCM.
///
/// Cust_Single_Line_All in the test project is a MultiUnicode field with English and French
/// alternatives. That is the shape every text custom field an LF project owns takes: LF has
/// never offered a single-string custom field, so a field created there is always multilingual
/// even when only one writing system is filled in.
/// </summary>
public class ConvertMongoToLcmCustomFieldTests : LcmTestBase
{
private const string MultiTextField = "customField_entry_Cust_Single_Line_All";
// A String field, whose writing system is the magic "first analysis" selector.
private const string SingleStringField = "customField_entry_Cust_Single_Line";

private static BsonDocument MultiText(params string[] wsThenValue)
{
var multiText = new BsonDocument();
for (int i = 0; i < wsThenValue.Length; i += 2)
multiText.Add(wsThenValue[i], new BsonDocument("value", wsThenValue[i + 1]));
return multiText;
}

private ILexEntry TestEntry()
{
var entry = _servLoc.GetInstance<ILexEntryRepository>().GetObject(Guid.Parse(TestEntryGuidStr));
Assert.That(entry, Is.Not.Null);
return entry;
}

private ConvertMongoToLcmCustomField Converter()
{
return new ConvertMongoToLcmCustomField(_cache, _servLoc,
new TestLogger(TestContext.CurrentContext.Test.Name), _wsEn);
}

/// <summary>Reads the entry's custom fields back out through the converter going the other way.</summary>
private BsonDocument ReadBack(ILexEntry entry)
{
var reader = new ConvertLcmToMongoCustomField(_cache, _servLoc,
new TestLogger(TestContext.CurrentContext.Test.Name));
return reader.GetCustomFieldsForThisCmObject(entry, "entry", _listConverters)[0].AsBsonDocument;
}

[Test]
public void SetCustomFieldsForThisCmObject_ShouldWriteEveryMultiUnicodeAlternative()
{
// Setup
ILexEntry entry = TestEntry();
var customFields = new BsonDocument(MultiTextField,
MultiText("en", "Updated English", "fr", "Updated French"));

// Exercise
Converter().SetCustomFieldsForThisCmObject(entry, "entry", customFields, null);

// Verify
BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
Assert.That(written["en"]["value"].AsString, Is.EqualTo("Updated English"));
Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Updated French"));
}

[Test]
public void SetCustomFieldsForThisCmObject_ShouldWriteAFieldThatWasEmptyInLcm()
{
// The migration case: LF holds the only copy, and LCM has nothing in the field yet.
// Emptying it leaves the alternatives in place holding empty strings rather than removing
// them, so the precondition checks the contents rather than the presence of the key.
ILexEntry entry = TestEntry();
Converter().SetCustomFieldsForThisCmObject(entry, "entry",
new BsonDocument(MultiTextField, MultiText("en", "", "fr", "")), null);
foreach (BsonElement alternative in ReadBack(entry)[MultiTextField].AsBsonDocument)
Assert.That(alternative.Value["value"].AsString, Is.Empty,
$"precondition: {alternative.Name} should have been emptied");

// Exercise
Converter().SetCustomFieldsForThisCmObject(entry, "entry",
new BsonDocument(MultiTextField, MultiText("en", "Brand new")), null);

// Verify
Assert.That(ReadBack(entry)[MultiTextField]["en"]["value"].AsString, Is.EqualTo("Brand new"));
}

[Test]
public void SetCustomFieldsForThisCmObject_ShouldClearAnAlternativeLfNoLongerHas()
{
// The reverse converter exports every alternative LCM holds, so one missing from LF is
// one the user deleted there -- not one LF never knew about.
ILexEntry entry = TestEntry();
Assert.That(ReadBack(entry)[MultiTextField].AsBsonDocument.Contains("fr"), Is.True,
"precondition: the test project's field starts with a French alternative");

// Exercise
Converter().SetCustomFieldsForThisCmObject(entry, "entry",
new BsonDocument(MultiTextField, MultiText("en", "English only")), null);

// Verify
BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
Assert.That(written["en"]["value"].AsString, Is.EqualTo("English only"));
Assert.That(written.Contains("fr"), Is.False, "French should have been cleared, not left stale");
}

[Test]
public void SetCustomFieldsForThisCmObject_ShouldSkipUnidentifiedWritingSystems()
{
// Setup
ILexEntry entry = TestEntry();

// Exercise
Converter().SetCustomFieldsForThisCmObject(entry, "entry",
new BsonDocument(MultiTextField,
MultiText("en", "Kept", "zzz-x-nonesuch", "Dropped")), null);

// Verify
BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
Assert.That(written["en"]["value"].AsString, Is.EqualTo("Kept"));
Assert.That(written.Contains("zzz-x-nonesuch"), Is.False);
}

[Test]
public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagToTheCanonicalWritingSystem()
{
// LF keeps a tag as typed, while LCM knows the writing system only by its canonical id:
// "fr-Latn" canonicalizes to "fr", French's suppress-script being dropped. Looked up
// literally the key resolves to 0, and French would then be cleared as missing from LF.
ILexEntry entry = TestEntry();

// Exercise
Converter().SetCustomFieldsForThisCmObject(entry, "entry",
new BsonDocument(MultiTextField, MultiText("en", "English", "fr-Latn", "Français")), null);

// Verify
BsonDocument written = ReadBack(entry)[MultiTextField].AsBsonDocument;
Assert.That(written.Contains("fr"), Is.True, "French should have been written, not cleared");
Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Français"));
}

[Test]
public void SetCustomFieldsForThisCmObject_ShouldResolveANonCanonicalTagInAStringField()
{
// The String case resolved the key it picked with a literal GetWsFromStr too, and gave
// up on the whole field when that returned 0, so the LF edit never reached LCM.
ILexEntry entry = TestEntry();

// Exercise
Converter().SetCustomFieldsForThisCmObject(entry, "entry",
new BsonDocument(SingleStringField, MultiText("fr-Latn", "Texte")), null);

// Verify
BsonDocument written = ReadBack(entry)[SingleStringField].AsBsonDocument;
Assert.That(written["fr"]["value"].AsString, Is.EqualTo("Texte"));
}

[Test]
public void SetCustomFieldData_FieldOfATypeItCannotWrite_IsReportedOncePerSyncNotOncePerObject()
{
// Setup: an owning sequence, a type the LF to LCM direction does not implement, on
// three entries. The test project has no custom field of that type, so a built-in
// field stands in for one; the writer does not tell them apart.
var logger = new TestLogger(TestContext.CurrentContext.Test.Name);
var converter = new ConvertMongoToLcmCustomField(_cache, _servLoc, logger, _wsEn);
var entries = _servLoc.GetInstance<ILexEntryRepository>().AllInstances().Take(3).ToList();
Assert.That(entries, Has.Count.EqualTo(3));

// Exercise
foreach (ILexEntry entry in entries)
Assert.That(converter.SetCustomFieldData(entry.Hvo, LexEntryTags.kflidSenses, new BsonString("LF data"), null), Is.False);

// Verify
int reports = logger.Messages.Split('\n').Count(line => line.Contains("not written to LCM"));
Assert.That(reports, Is.EqualTo(1));
}
}
}
Loading
Loading