diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a172e5d..b2bf1ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [SIL.LCModel] Add GetCaptionOrHeadword() to CmPicture - [SIL.LCModel] `LCModelStrings.NotSure` to allow clients to know if a grammatical category is the placeholder - [SIL.LCModel.Utils] `DateTime` extension method `ToLCMTimeFormatWithMillisString()` (replaces `ReadWriteServices.FormatDateTime`) +- [SIL.LCModel.Utils] `Surrogates.CodePointAt(string, int)` (a `char.ConvertToUtf32` that tolerates unpaired surrogates), `HasLoneSurrogate`, `ReplaceLoneSurrogates` and `StringFromCodePointOrLoneSurrogate`; [SIL.LCModel.Core] `TsStringUtils.ReplaceLoneSurrogates(ITsString)` ### Fixed @@ -39,9 +40,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [SIL.LCModel.Core] Copy `SIL.LCModel.Core.dll.config` to output directory - [SIL.LCModel] Use `CaseFunctions` (to use the `WritingSystemDefinition.CaseAlias`, if any) - [SIL.LCModel.Core] Use the `WritingSystemDefinition.CaseAlias`, if any, in `CaseFunctions` +- [SIL.LCModel.Core] Text containing an unpaired (lone) surrogate no longer makes `TsString` normalization, `TsStringSerializer.SerializeTsStringToXml`, `TextExtensions.CharAt`, `TextExtensions.IsCharWordForming` or `WordMaker.IsWhite` throw. Normalization passes the surrogate through unchanged, as ICU does. `IsCharWordForming` no longer treats a lone low surrogate as the second half of a pair, so it is no longer classified by the character before it. `TsStringSerializer` writes a lone surrogate as U+FFFD (it used to throw), so the written text is as long as the text in memory, and decides `xml:space` from the text it writes ### Changed +- [SIL.LCModel] The generated and custom plain `Unicode` property setters now replace a lone (unpaired) surrogate with U+FFFD instead of throwing, and `WfiWordformRepository.TryGetObject` treats a lone surrogate in the looked-up form as U+FFFD instead of throwing +- [SIL.LCModel.Core] `TsStringUtils.NormalizeNfd(ITsString)` replaces each unpaired surrogate with U+FFFD, so the TsString setters (String, MultiString and MultiUnicode properties, custom String fields, `MultiAccessor.set_String`, `IDomainDataByFlid.SetString` and `SetMultiStringAlt`) store U+FFFD for text containing one. This used to throw, so a caller that relied on the exception now gets U+FFFD silently; that includes direct callers of `NormalizeNfd(ITsString)`. Custom String fields are now also stored as NFD in memory when set (as model String properties already were); the saved files are unchanged. A valid surrogate pair split across two runs is read as two lone surrogates when written to XML - [SIL.LCModel] The domain-model `Generated*.cs` sources are now produced at compile time by the new `SIL.LCModel.SourceGenerators` Roslyn source generator, instead of the `LcmGenerate` MSBuild task running the NVelocity templates in a child MSBuild process. The NVelocity engine and model wrappers moved to a shared `SIL.LCModel.ModelGeneration` project used by both the source generator and the (retained) `LcmGenerate` task, which still renders other templates from the model (e.g. the FieldWorks C++ cellar-constants header). Templates and generated output are byte-for-byte identical. No public API change. - [SIL.LCModel] Replaced the internal StructureMap (`structuremap.patched`) IoC container with `Microsoft.Extensions.DependencyInjection` (8.x). No public API change. - [SIL.LCModel] Trim 12 overwordy semantic domain descriptions and fix 22 punctuation/whitespace issues in the SemDom.xml template, matching sillsdev/FwLocalizations#5 and sillsdev/FwLocalizations#7 diff --git a/src/SIL.LCModel.Core/Text/TextExtensions.cs b/src/SIL.LCModel.Core/Text/TextExtensions.cs index 0a210aef..34cf8c63 100644 --- a/src/SIL.LCModel.Core/Text/TextExtensions.cs +++ b/src/SIL.LCModel.Core/Text/TextExtensions.cs @@ -356,10 +356,11 @@ public static bool IsCharWordForming(this ITsString tss, int ich, ILgWritingSyst if (ich < 0 || ich >= tss.Length) throw new ArgumentOutOfRangeException("ich"); - if (char.IsLowSurrogate(tss.Text[ich])) + // Step back only over the second half of a real pair; a lone low surrogate is a character of its own. + if (ich > 0 && char.IsLowSurrogate(tss.Text[ich]) && char.IsHighSurrogate(tss.Text[ich - 1])) --ich; int wsHandle = tss.get_WritingSystemAt(ich); - int ch = char.ConvertToUtf32(tss.Text, ich); + int ch = Surrogates.CodePointAt(tss.Text, ich); return TsStringUtils.IsWordForming(ch, wsf, wsHandle); } @@ -371,7 +372,7 @@ public static int CharAt(this ITsString tss, int ich) if (ich < 0 || ich >= tss.Length) throw new ArgumentOutOfRangeException("ich"); - return char.ConvertToUtf32(tss.Text, ich); + return Surrogates.CodePointAt(tss.Text, ich); } /// diff --git a/src/SIL.LCModel.Core/Text/TsString.cs b/src/SIL.LCModel.Core/Text/TsString.cs index 2f657328..c2d9024c 100644 --- a/src/SIL.LCModel.Core/Text/TsString.cs +++ b/src/SIL.LCModel.Core/Text/TsString.cs @@ -289,7 +289,7 @@ private IEnumerable EnumerateSegmentLimits(Normalizer2 icuNormalizer) int i = 0; while (i < Text.Length) { - int codepoint = Char.ConvertToUtf32(Text, i); + int codepoint = Surrogates.CodePointAt(Text, i); if (icuNormalizer.HasBoundaryBefore(codepoint) && i > 0) { yield return i; @@ -302,7 +302,7 @@ private IEnumerable EnumerateSegmentLimits(Normalizer2 icuNormalizer) /// /// Helper function used in MatchUpIndexesAfterNormalization, itself a helper function for get_NormalizedForm /// - /// String to convert to UTF32 codepoints. Must not be null, and must not end with an unpaired surrogate. + /// String to convert to UTF32 codepoints. Must not be null, and may contain unpaired surrogates, which are reported as their own code unit value. /// A sequence of KeyValuePairs where the key is the index (in chars) of this codepoint in the original string, and the value is the Unicode codepoint at that index. private List> CodepointsByIndex(string s) { @@ -310,7 +310,7 @@ private List> CodepointsByIndex(string s) var result = new List>(len); for (int i = 0; i < len; i++) { - int codePoint = Char.ConvertToUtf32(s, i); + int codePoint = Surrogates.CodePointAt(s, i); result.Add(new KeyValuePair(i, codePoint)); if (codePoint > 0xffff) i++; // Skip second half of a surrogate pair @@ -359,7 +359,7 @@ private IEnumerable MatchUpIndexesAfterNormalization( { int origIdx = indexAndCodePoint.Key; int origCodePoint = indexAndCodePoint.Value; - var normalizedStringFromOrigCodePoint = icuNormalizer.GetDecomposition(origCodePoint) ?? char.ConvertFromUtf32(origCodePoint); + var normalizedStringFromOrigCodePoint = icuNormalizer.GetDecomposition(origCodePoint) ?? Surrogates.StringFromCodePointOrLoneSurrogate(origCodePoint); foreach (KeyValuePair indexAndResultingCodePoint in CodepointsByIndex(normalizedStringFromOrigCodePoint)) { int resultingCodePoint = indexAndResultingCodePoint.Value; @@ -485,7 +485,7 @@ private ITsString get_NormalizedFormAndFixOffsets(FwNormalizationMode nm, ArrayP { ITsTextProps origProperties = get_PropertiesAt(segmentMin + mapping.origIdx); int outputIdx = outputLenSoFar + mapping.normIdx; - int size = Char.IsSurrogate(normalizedSegment, mapping.normIdx) ? 2 : 1; + int size = Char.IsSurrogatePair(normalizedSegment, mapping.normIdx) ? 2 : 1; resultBuilder.SetProperties(outputIdx, outputIdx + size, origProperties); // And if we also need to fix up offsets at the end, we keep track of the ones we'll need if (willFixOffsets && mapping.isFirstCharOfDecomposition) diff --git a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs index 173b6a7a..1ff4492d 100644 --- a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs @@ -170,11 +170,11 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto } else { - string runText = normalizedTss.get_RunText(i) ?? string.Empty; - if (runText != string.Empty && runText.All(char.IsWhiteSpace)) + string writtenText = GetXmlSafeText(normalizedTss.get_RunText(i) ?? string.Empty); + if (writtenText != string.Empty && writtenText.All(char.IsWhiteSpace)) writer.WriteAttributeString("xml", "space", "", "preserve"); // TODO: should we escape quotation marks? this is not necessary but different than the behavior of the C++ implementation - writer.WriteString(StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC))); + writer.WriteString(writtenText); } writer.WriteEndElement(); @@ -196,6 +196,23 @@ public static string StripInvalidXmlChars(string text) return InvalidXmlRegex.Replace(text, string.Empty); } + /// + /// Gets the NFC form of text for writing to XML, without the characters XML cannot represent. + /// Unpaired surrogates are replaced with U+FFFD rather than removed, so that the written text is + /// as long as the text in memory (offsets of segments, analyses and notes index into it). + /// (Model setters already replace them via ; + /// this covers TsStrings that never went through the model.) + /// Characters are cleaned before normalizing because a lone surrogate is a starter: removing it + /// after NFC could leave an uncomposed sequence such as "e" + U+0301. + /// + internal static string GetXmlSafeText(string text) + { + // Replace lone surrogates first: removing a control character can bring a high and a low surrogate + // together into a "pair" that was never one, and the result must not depend on whether + // StripInvalidXmlChars itself removes lone surrogates. + string cleaned = StripInvalidXmlChars(Surrogates.ReplaceLoneSurrogates(text)); + return StripInvalidXmlChars(Normalizer.Normalize(cleaned, Normalizer.UNormalizationMode.UNORM_NFC)); + } #endregion #region Serialization Helper Methods diff --git a/src/SIL.LCModel.Core/Text/TsStringUtils.cs b/src/SIL.LCModel.Core/Text/TsStringUtils.cs index f8f1c9a6..f4f1d06c 100644 --- a/src/SIL.LCModel.Core/Text/TsStringUtils.cs +++ b/src/SIL.LCModel.Core/Text/TsStringUtils.cs @@ -557,6 +557,9 @@ public static string NormalizeNfd(string source) /// ------------------------------------------------------------------------------------ public static ITsString NormalizeNfd(ITsString source) { + // An unpaired surrogate cannot be stored (it cannot be written to XML). Replace it up front, with a + // same-length character, so that the stored text always equals the in-memory text. + source = ReplaceLoneSurrogates(source); if (source == null || source.Length == 0 || source.get_IsNormalizedForm(FwNormalizationMode.knmNFD)) { return source; @@ -565,6 +568,26 @@ public static ITsString NormalizeNfd(ITsString source) return source.get_NormalizedForm(FwNormalizationMode.knmNFD); } + /// + /// Replaces each unpaired surrogate in the string with U+FFFD, keeping the text properties of its run + /// and the length of the string. Returns the same object if there is nothing to replace. + /// + public static ITsString ReplaceLoneSurrogates(ITsString source) + { + if (source == null || !Surrogates.HasLoneSurrogate(source.Text)) + return source; + string text = source.Text; + ITsStrBldr bldr = source.GetBldr(); + for (int i = 0; i < text.Length; i++) + { + if (char.IsHighSurrogate(text[i]) && i < text.Length - 1 && char.IsLowSurrogate(text[i + 1])) + i++; + else if (char.IsSurrogate(text[i])) + bldr.Replace(i, i + 1, "\uFFFD", source.get_PropertiesAt(i)); + } + return bldr.GetString(); + } + #region ORC-related methods /// ------------------------------------------------------------------------------------ /// diff --git a/src/SIL.LCModel.SourceGenerators/Templates/vm/propaccessors_simple.vm.cs b/src/SIL.LCModel.SourceGenerators/Templates/vm/propaccessors_simple.vm.cs index b362b1c1..4d8e68d6 100644 --- a/src/SIL.LCModel.SourceGenerators/Templates/vm/propaccessors_simple.vm.cs +++ b/src/SIL.LCModel.SourceGenerators/Templates/vm/propaccessors_simple.vm.cs @@ -236,7 +236,8 @@ internal set var newValue = value; Validate${prop.NiuginianPropName}(ref newValue); if (!string.IsNullOrEmpty(newValue)) - newValue = newValue.Normalize(NormalizationForm.FormD); // Store it as NFD in LCM. + // Store it as NFD in LCM. An unpaired surrogate (which Normalize rejects) becomes U+FFFD. + newValue = SIL.LCModel.Utils.Surrogates.ReplaceLoneSurrogates(newValue).Normalize(NormalizationForm.FormD); var originalValue = m_$prop.NiuginianPropName; if (newValue == originalValue) return; diff --git a/src/SIL.LCModel.Utils/Surrogates.cs b/src/SIL.LCModel.Utils/Surrogates.cs index 5d2f44b5..7f03d025 100644 --- a/src/SIL.LCModel.Utils/Surrogates.cs +++ b/src/SIL.LCModel.Utils/Surrogates.cs @@ -27,6 +27,68 @@ public static string StringFromCodePoint(int codepoint) return char.ConvertFromUtf32(codepoint); } + /// + /// Return the full code point starting at index ich of the string. A high surrogate followed by a + /// low surrogate gives the supplementary code point. Anything else gives the char itself: that includes + /// an unpaired surrogate, and also a low surrogate that is the second half of a pair when ich points at + /// it (so the pair is not re-read from its middle). Unlike char.ConvertToUtf32(string, int), this never + /// throws on an unpaired surrogate (which can occur in pasted or imported text). + /// + public static int CodePointAt(string st, int ich) + { + char ch = st[ich]; + if (IsLeadSurrogate(ch) && ich < st.Length - 1 && IsTrailSurrogate(st[ich + 1])) + return char.ConvertToUtf32(ch, st[ich + 1]); + return ch; + } + + /// + /// Gives the string for a code point as returned by : unlike + /// , an unpaired surrogate code unit gives that one char instead of throwing. + /// + public static string StringFromCodePointOrLoneSurrogate(int codepoint) + { + if (codepoint >= kMinLeadSurrogate && codepoint <= 0xDFFF) + return ((char)codepoint).ToString(); + return char.ConvertFromUtf32(codepoint); + } + + /// + /// Whether the string contains a surrogate that is not half of a valid high+low pair. + /// + public static bool HasLoneSurrogate(string st) + { + if (st == null) + return false; + for (int i = 0; i < st.Length; i++) + { + if (IsLeadSurrogate(st[i]) && i < st.Length - 1 && IsTrailSurrogate(st[i + 1])) + i++; + else if (IsLeadSurrogate(st[i]) || IsTrailSurrogate(st[i])) + return true; + } + return false; + } + + /// + /// Replace each surrogate that is not half of a valid pair with U+FFFD REPLACEMENT CHARACTER. + /// The length of the string never changes, so offsets into it stay valid. + /// + public static string ReplaceLoneSurrogates(string st) + { + if (!HasLoneSurrogate(st)) + return st; + var chars = st.ToCharArray(); + for (int i = 0; i < chars.Length; i++) + { + if (IsLeadSurrogate(chars[i]) && i < chars.Length - 1 && IsTrailSurrogate(chars[i + 1])) + i++; + else if (IsLeadSurrogate(chars[i]) || IsTrailSurrogate(chars[i])) + chars[i] = '\uFFFD'; + } + return new string(chars); + } + /// /// Return a full 32-bit character value from the surrogate pair. /// diff --git a/src/SIL.LCModel/DomainImpl/CmObject.cs b/src/SIL.LCModel/DomainImpl/CmObject.cs index 2a623fa9..a9484650 100644 --- a/src/SIL.LCModel/DomainImpl/CmObject.cs +++ b/src/SIL.LCModel/DomainImpl/CmObject.cs @@ -2921,6 +2921,16 @@ private void SetCustomPropertyForSDA(int flid, object newValue, bool useAccessor // break; // case CellarPropertyType.Image: // Not in model. // case CellarPropertyType.Numeric: // Not in model. + case CellarPropertyType.String: + // Same input rules as a model String property: NFD, and no unpaired surrogates. + if (newValue is ITsString newTss) + newValue = TsStringUtils.NormalizeNfd(newTss); + break; + case CellarPropertyType.Unicode: + // Same input rules as a generated Unicode property: NFD, and no unpaired surrogates. + if (newValue is string newString && newString.Length > 0) + newValue = Surrogates.ReplaceLoneSurrogates(newString).Normalize(NormalizationForm.FormD); + break; case CellarPropertyType.Guid: if (newValue == null) newValue = Guid.Empty; diff --git a/src/SIL.LCModel/DomainServices/ITextUtils.cs b/src/SIL.LCModel/DomainServices/ITextUtils.cs index 991be78d..89bf8d58 100644 --- a/src/SIL.LCModel/DomainServices/ITextUtils.cs +++ b/src/SIL.LCModel/DomainServices/ITextUtils.cs @@ -1560,7 +1560,7 @@ public int NextChar(int ich) /// public bool IsWhite(int ich) { - int ch = char.ConvertToUtf32(m_st, ich); + int ch = Surrogates.CodePointAt(m_st, ich); return Character.GetCharType(ch) == Character.UCharCategory.SPACE_SEPARATOR || ch == AnalysisOccurrence.KchZws; } diff --git a/src/SIL.LCModel/Infrastructure/Impl/RepositoryAdditions.cs b/src/SIL.LCModel/Infrastructure/Impl/RepositoryAdditions.cs index 956fe598..5230eec9 100644 --- a/src/SIL.LCModel/Infrastructure/Impl/RepositoryAdditions.cs +++ b/src/SIL.LCModel/Infrastructure/Impl/RepositoryAdditions.cs @@ -888,11 +888,11 @@ public bool TryGetObject(ITsString tssTarget, out IWfiWordform wf) { ITsString tssKey = wfT.Form.get_String(ws); if (tssKey.Length > 0) - lookupTable[tssKey.Text.Normalize(NormalizationForm.FormD)] = wfT; + lookupTable[Surrogates.ReplaceLoneSurrogates(tssKey.Text).Normalize(NormalizationForm.FormD)] = wfT; } } } - return lookupTable.TryGetValue(tssTarget.Text.Normalize(NormalizationForm.FormD), out wf); + return lookupTable.TryGetValue(Surrogates.ReplaceLoneSurrogates(tssTarget.Text).Normalize(NormalizationForm.FormD), out wf); } void IWfiWordformRepositoryInternal.UpdateForm(ITsString oldForm, IWfiWordform wf, int ws) diff --git a/tests/SIL.LCModel.Core.Tests/Text/TsStringLoneSurrogateTests.cs b/tests/SIL.LCModel.Core.Tests/Text/TsStringLoneSurrogateTests.cs new file mode 100644 index 00000000..6b81acf8 --- /dev/null +++ b/tests/SIL.LCModel.Core.Tests/Text/TsStringLoneSurrogateTests.cs @@ -0,0 +1,387 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Xml.Linq; +using NUnit.Framework; +using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Utils; + +namespace SIL.LCModel.Core.Text +{ + /// + /// Text can contain unpaired UTF-16 surrogates (pasted or imported text). Normalization, serialization and + /// the character accessors must not throw on it; the model replaces such a surrogate with U+FFFD on input + /// (so stored text always equals in-memory text) and the serializer does the same as a backstop. + /// NOTE: the strings are built in code with \u escapes; NUnit attributes would turn lone surrogates into U+FFFD. + /// + [TestFixture] + public class TsStringLoneSurrogateTests : TsSerializerTestsBase + { + private const char High = '\uD800'; + private const char Low = '\uDC00'; + private const string Fffd = "\uFFFD"; + private const string Acute = "\u0301"; + + private static readonly string[][] NamedCases = + { + new[] { "LoneHighMiddle", "a" + High + "b" }, + new[] { "LoneLowMiddle", "a" + Low + "b" }, + new[] { "LoneHighAtStart", High + "ab" }, + new[] { "LoneLowAtStart", Low + "ab" }, + new[] { "LoneHighAtEnd", "ab" + High }, + new[] { "LoneLowAtEnd", "ab" + Low }, + new[] { "ReversedPair", "" + Low + High }, + new[] { "LoneHighThenValidPair", "" + High + High + Low }, + new[] { "CombiningMarkAfterLoneHigh", "e" + Acute + High + Acute }, + new[] { "OnlyLoneHigh", "" + High }, + }; + + private static string[] LoneSurrogateStrings() + { + return NamedCases.Select(c => c[1]).ToArray(); + } + + // Named with ASCII so the cases are filterable and readable in logs. + private static IEnumerable LoneSurrogateCases() + { + return NamedCases.Select(c => new TestCaseData(c[1]).SetName("LoneSurrogate_" + c[0])); + } + + private static string Replaced(string s) + { + var sb = new StringBuilder(); + for (int i = 0; i < s.Length; i++) + { + if (char.IsSurrogatePair(s, i)) + { + sb.Append(s[i]).Append(s[i + 1]); + i++; + } + else + sb.Append(char.IsSurrogate(s[i]) ? '\uFFFD' : s[i]); + } + return sb.ToString(); + } + + private static int CountLone(string s) + { + int n = 0; + for (int i = 0; i < s.Length; i++) + { + if (char.IsSurrogatePair(s, i)) + i++; + else if (char.IsSurrogate(s[i])) + n++; + } + return n; + } + + private static string Escape(string s) + { + var sb = new StringBuilder(); + foreach (char c in s) + sb.AppendFormat(" U+{0:X4}", (int)c); + return sb.ToString(); + } + + #region Serializer backstop + + [TestCaseSource(nameof(LoneSurrogateCases))] + public void SerializeTsStringToXml_ReplacesLoneSurrogate_AndReloads(string text) + { + ITsString tss = TsStringUtils.MakeString(text, EnWS); + string xml = null; + Assert.DoesNotThrow(() => xml = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager)); + Assert.DoesNotThrow(() => XElement.Parse(xml), Escape(xml)); // well-formed + ITsString reloaded = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); + // Lone surrogates become U+FFFD, so nothing is lost and lengths do not change (compared in NFC, since loading yields NFD). + string expected = Replaced(text).Normalize(NormalizationForm.FormC); + Assert.That(reloaded.get_NormalizedForm(FwNormalizationMode.knmNFC).Text, Is.EqualTo(expected), Escape(xml)); + } + + [TestCaseSource(nameof(LoneSurrogateCases))] + public void SerializeTsStringToXml_LoneSurrogate_ResaveIsStable(string text) + { + ITsString tss = TsStringUtils.MakeString(text, EnWS); + string xml1 = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager); + ITsString reloaded = TsStringSerializer.DeserializeTsStringFromXml(xml1, WritingSystemManager); + Assert.That(TsStringSerializer.SerializeTsStringToXml(reloaded, WritingSystemManager), Is.EqualTo(xml1)); + } + + [Test] + public void SerializeTsStringToXml_LoneSurrogatesAreReplacedBeforeInvalidCharsAreStripped() + { + // Order matters: if the control character were stripped first, the lone high and lone low surrogate + // would become adjacent and read as a valid pair (U+10000), changing the text. Each is lone, so each + // becomes U+FFFD, and the control character is then removed. + ITsString tss = TsStringUtils.MakeString("x" + High + "\u0002" + Low + "y", EnWS); + string xml = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager); + Assert.That(TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager).Text, + Is.EqualTo("x" + Fffd + Fffd + "y")); + } + + [Test] + public void SerializeTsStringToXml_RunOfOnlyLoneSurrogates_IsNotWrittenEmpty() + { + var bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "word", TsStringUtils.MakeProps(null, EsWS)); + bldr.Replace(bldr.Length, bldr.Length, "" + Low + High, TsStringUtils.MakeProps(null, EnWS)); + bldr.Replace(bldr.Length, bldr.Length, "word", TsStringUtils.MakeProps(null, EsWS)); + string xml = TsStringSerializer.SerializeTsStringToXml(bldr.GetString(), WritingSystemManager); + ITsString reloaded = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); + Assert.That(reloaded.Text, Is.EqualTo("word" + Fffd + Fffd + "word")); + Assert.That(reloaded.RunCount, Is.EqualTo(3)); + Assert.That(TsStringSerializer.SerializeTsStringToXml(reloaded, WritingSystemManager), Is.EqualTo(xml)); + } + + [Test] + public void SerializeTsStringToXml_WhitespaceRunPreservedWhenControlCharStripped() + { + // " " + U+0002 is written as " ": xml:space must be decided on what is written, or the run is lost. + var bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "word", TsStringUtils.MakeProps(null, EsWS)); + bldr.Replace(bldr.Length, bldr.Length, " \u0002", TsStringUtils.MakeProps(null, EnWS)); + bldr.Replace(bldr.Length, bldr.Length, "word", TsStringUtils.MakeProps(null, EsWS)); + string xml = TsStringSerializer.SerializeTsStringToXml(bldr.GetString(), WritingSystemManager); + Assert.That(xml, Does.Contain("xml:space=\"preserve\"")); + Assert.That(TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager).Text, Is.EqualTo("word word")); + } + + [Test] + public void SerializeTsStringToXml_LoneSurrogateInOneOfSeveralRuns() + { + var bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "one" + High, TsStringUtils.MakeProps(null, EnWS)); + bldr.Replace(bldr.Length, bldr.Length, "two", TsStringUtils.MakeProps(null, EsWS)); + string xml = TsStringSerializer.SerializeTsStringToXml(bldr.GetString(), WritingSystemManager); + ITsString reloaded = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); + Assert.That(reloaded.Text, Is.EqualTo("one" + Fffd + "two")); + Assert.That(reloaded.RunCount, Is.EqualTo(2)); + } + + [Test] + public void SerializeTsStringToXml_ValidSurrogatePairNextToLoneSurrogate_KeepsPair() + { + ITsString tss = TsStringUtils.MakeString("\U0001F98A" + High + "\U0001F98A", EnWS); + string xml = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager); + Assert.That(TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager).Text, + Is.EqualTo("\U0001F98A" + Fffd + "\U0001F98A")); + } + + #endregion + + #region Input sanitizing (the choke point for model setters) + + [TestCaseSource(nameof(LoneSurrogateCases))] + public void NormalizeNfd_ReplacesLoneSurrogateWithFffd_KeepingLength(string text) + { + ITsString tss = TsStringUtils.MakeString(text, EnWS); + ITsString result = TsStringUtils.NormalizeNfd(tss); + Assert.That(CountLone(result.Text), Is.EqualTo(0), Escape(result.Text)); + Assert.That(result.Text, Is.EqualTo(Replaced(text).Normalize(NormalizationForm.FormD))); + } + + [Test] + public void NormalizeNfd_ReplacementKeepsRunProperties() + { + var bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "ab", TsStringUtils.MakeProps(null, EnWS)); + bldr.Replace(bldr.Length, bldr.Length, "c" + High + "d", TsStringUtils.MakeProps(null, EsWS)); + ITsString result = TsStringUtils.NormalizeNfd(bldr.GetString()); + Assert.That(result.Text, Is.EqualTo("abc" + Fffd + "d")); + Assert.That(result.RunCount, Is.EqualTo(2)); + Assert.That(result.get_WritingSystemAt(3), Is.EqualTo(EsWS)); + Assert.That(result.get_WritingSystemAt(1), Is.EqualTo(EnWS)); + } + + [Test] + public void NormalizeNfd_ValidPairAndNormalStringUnchanged() + { + ITsString pair = TsStringUtils.MakeString("a\U0001F98Ab", EnWS); + Assert.That(TsStringUtils.NormalizeNfd(pair), Is.SameAs(pair)); + Assert.That(TsStringUtils.ReplaceLoneSurrogates(pair), Is.SameAs(pair)); + Assert.That(TsStringUtils.ReplaceLoneSurrogates(null), Is.Null); + } + + #endregion + + #region Normalization passes lone surrogates through (as ICU does) + + [TestCase(FwNormalizationMode.knmNFC)] + [TestCase(FwNormalizationMode.knmNFD)] + [TestCase(FwNormalizationMode.knmNFKC)] + [TestCase(FwNormalizationMode.knmNFKD)] + [TestCase(FwNormalizationMode.knmNFSC)] + public void NormalizedForm_LoneSurrogate_DoesNotThrow_AndPassesThrough(FwNormalizationMode mode) + { + foreach (string text in LoneSurrogateStrings()) + { + ITsString tss = TsStringUtils.MakeString(text, EnWS); + ITsString normalized = null; + Assert.DoesNotThrow(() => normalized = tss.get_NormalizedForm(mode), "mode " + mode + ", text " + Escape(text)); + // Pass-through: the lone surrogates are still there, in the same count. + Assert.That(CountLone(normalized.Text), Is.EqualTo(CountLone(text)), Escape(text)); + Assert.DoesNotThrow(() => tss.get_IsNormalizedForm(mode)); + } + } + + [Test] + public void NormalizedForm_LoneSurrogate_StillNormalizesTheRest() + { + ITsString tss = TsStringUtils.MakeString("e" + Acute + High + "o\u0308", EnWS); + Assert.That(tss.get_NormalizedForm(FwNormalizationMode.knmNFC).Text, Is.EqualTo("\u00E9" + High + "\u00F6")); + ITsString composed = TsStringUtils.MakeString("\u00E9" + Low + "\u00F6", EnWS); + Assert.That(composed.get_NormalizedForm(FwNormalizationMode.knmNFD).Text, Is.EqualTo("e" + Acute + Low + "o\u0308")); + } + + [Test] + public void NormalizedForm_LoneSurrogate_AcrossRuns_NFD() + { + var bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "\u00E9" + High, TsStringUtils.MakeProps(null, EnWS)); + bldr.Replace(bldr.Length, bldr.Length, Acute + "x", TsStringUtils.MakeProps(null, EsWS)); + Assert.DoesNotThrow(() => bldr.GetString().get_NormalizedForm(FwNormalizationMode.knmNFD)); + Assert.DoesNotThrow(() => bldr.GetString().get_NormalizedForm(FwNormalizationMode.knmNFSC)); + } + + [Test] + public void NfdAndFixOffsets_LoneSurrogate_FixesOffsets() + { + // U+00E9 decomposes to two chars, so the offset of "b" moves from 2 to 3. The lone high surrogate is kept. + ITsString tss = TsStringUtils.MakeString("\u00E9" + High + "b", EnWS); + IntPtr offset = Marshal.AllocHGlobal(sizeof(int)); + IntPtr table = Marshal.AllocHGlobal(IntPtr.Size); + try + { + Marshal.WriteInt32(offset, 2); + Marshal.WriteIntPtr(table, offset); + ITsString result; + tss.NfdAndFixOffsets(out result, new ArrayPtr(table), 1); + Assert.That(result.Text, Is.EqualTo("e" + Acute + High + "b")); + Assert.That(Marshal.ReadInt32(offset), Is.EqualTo(3)); + } + finally + { + Marshal.FreeHGlobal(offset); + Marshal.FreeHGlobal(table); + } + } + + #endregion + + #region Character accessors + + [Test] + public void CharAt_And_CharIndexNavigation_LoneSurrogate_DoNotThrow() + { + ITsString tss = TsStringUtils.MakeString("a" + High + Low + Low + High, EnWS); + Assert.That(tss.CharAt(0), Is.EqualTo('a')); + Assert.That(tss.CharAt(1), Is.EqualTo(0x10000)); // valid pair + Assert.That(tss.CharAt(3), Is.EqualTo((int)Low)); // lone low + Assert.That(tss.CharAt(4), Is.EqualTo((int)High)); // lone high at end + Assert.That(tss.NextCharIndex(3), Is.EqualTo(4)); + Assert.That(tss.NextCharIndex(4), Is.EqualTo(5)); + Assert.That(tss.PrevCharIndex(3), Is.EqualTo(1)); // back over the valid pair + Assert.That(tss.PrevCharIndex(4), Is.EqualTo(3)); + } + + [Test] + public void IsCharWordForming_LoneLowSurrogateAtStart_DoesNotThrow_AndIsNotWordForming() + { + ITsString tss = TsStringUtils.MakeString("" + Low + "ab", EnWS); + Assert.That(tss.IsCharWordForming(0, WritingSystemManager), Is.False); + Assert.That(tss.IsCharWordForming(1, WritingSystemManager), Is.True); + } + + [Test] + public void IsCharWordForming_LoneLowSurrogateAfterLetter_IsNotWordForming() + { + // It must not be classified by the letter before it. + ITsString tss = TsStringUtils.MakeString("b" + Low + "c", EnWS); + Assert.That(tss.IsCharWordForming(0, WritingSystemManager), Is.True); + Assert.That(tss.IsCharWordForming(1, WritingSystemManager), Is.False); + Assert.That(tss.IsCharWordForming(2, WritingSystemManager), Is.True); + } + + [Test] + public void IsCharWordForming_LoneHighSurrogate_IsNotWordForming() + { + ITsString tss = TsStringUtils.MakeString("b" + High + "c" + High, EnWS); + Assert.That(tss.IsCharWordForming(1, WritingSystemManager), Is.False); + Assert.That(tss.IsCharWordForming(3, WritingSystemManager), Is.False); + } + + [Test] + public void IsCharWordForming_ValidPair_StillClassifiedByTheFullCharacter() + { + // U+10400 DESERET CAPITAL LETTER LONG I is a letter; both halves of the pair report that. + ITsString tss = TsStringUtils.MakeString("a\U00010400b", EnWS); + Assert.That(tss.IsCharWordForming(1, WritingSystemManager), Is.True); + Assert.That(tss.IsCharWordForming(2, WritingSystemManager), Is.True); + } + + [Test] + public void CharacterAccessors_Fuzz_NeverThrow() + { + // Lone highs and lows at the start, middle and end of random text. + var rng = new Random(414); + string[] alphabet = { "a", "b", " ", ".", "\u00E9", Acute, "\U0001F98A", "" + High, "" + Low, "\uDBFF", "\uDFFF" }; + for (int n = 0; n < 2000; n++) + { + var sb = new StringBuilder(); + int len = rng.Next(1, 9); + for (int i = 0; i < len; i++) + sb.Append(alphabet[rng.Next(alphabet.Length)]); + string text = sb.ToString(); + ITsString tss = TsStringUtils.MakeString(text, EnWS); + string failure = Escape(text); + foreach (FwNormalizationMode mode in new[] { FwNormalizationMode.knmNFC, FwNormalizationMode.knmNFD, FwNormalizationMode.knmNFSC }) + Assert.DoesNotThrow(() => tss.get_NormalizedForm(mode), failure); + Assert.DoesNotThrow(() => TsStringSerializer.DeserializeTsStringFromXml( + TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager), WritingSystemManager), failure); + ITsString clean = TsStringUtils.NormalizeNfd(tss); + Assert.That(clean.Text ?? string.Empty, Is.EqualTo(Icu.Normalizer.Normalize(Replaced(text), Icu.Normalizer.UNormalizationMode.UNORM_NFD)), failure); // ICU: .NET Framework rejects unassigned code points such as a D800+DFFF pair + for (int i = 0; i < tss.Length; i++) + { + int ich = i; + Assert.DoesNotThrow(() => tss.CharAt(ich), failure); + Assert.DoesNotThrow(() => tss.IsCharWordForming(ich, WritingSystemManager), failure); + Assert.DoesNotThrow(() => tss.NextCharIndex(ich), failure); + Assert.DoesNotThrow(() => tss.PrevCharIndex(ich), failure); + } + // A re-save of what we wrote is identical. + string xml1 = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager); + string xml2 = TsStringSerializer.SerializeTsStringToXml( + TsStringSerializer.DeserializeTsStringFromXml(xml1, WritingSystemManager), WritingSystemManager); + Assert.That(xml2, Is.EqualTo(xml1), failure); + } + } + + [Test] + public void Surrogates_CodePointAt() + { + Assert.That(Surrogates.CodePointAt("a" + High + Low, 1), Is.EqualTo(0x10000)); + Assert.That(Surrogates.CodePointAt("a" + High + Low, 2), Is.EqualTo((int)Low)); // second half of a pair, read from its middle + Assert.That(Surrogates.CodePointAt("a" + High, 1), Is.EqualTo((int)High)); + } + + [Test] + public void Surrogates_HasLoneSurrogate_And_ReplaceLoneSurrogates() + { + Assert.That(Surrogates.HasLoneSurrogate(null), Is.False); + Assert.That(Surrogates.HasLoneSurrogate("a\U0001F98Ab"), Is.False); + Assert.That(Surrogates.HasLoneSurrogate("a" + High), Is.True); + Assert.That(Surrogates.HasLoneSurrogate("" + Low + High), Is.True); + Assert.That(Surrogates.ReplaceLoneSurrogates("a" + High + "b" + Low + "\U0001F98A"), Is.EqualTo("a" + Fffd + "b" + Fffd + "\U0001F98A")); + Assert.That(Surrogates.ReplaceLoneSurrogates(null), Is.Null); + } + + #endregion + } +} diff --git a/tests/SIL.LCModel.Tests/DomainImpl/LoneSurrogatePlainStringTests.cs b/tests/SIL.LCModel.Tests/DomainImpl/LoneSurrogatePlainStringTests.cs new file mode 100644 index 00000000..d62b2580 --- /dev/null +++ b/tests/SIL.LCModel.Tests/DomainImpl/LoneSurrogatePlainStringTests.cs @@ -0,0 +1,42 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using NUnit.Framework; +using SIL.LCModel.Core.Text; + +namespace SIL.LCModel.DomainImpl +{ + /// + /// Plain-string (not ITsString) model paths must replace a lone surrogate with U+FFFD, not throw. + /// NOTE: strings are built with \u escapes; NUnit attributes would turn lone surrogates into U+FFFD. + /// + [TestFixture] + public class LoneSurrogatePlainStringTests : MemoryOnlyBackendProviderRestoredForEachTestTestBase + { + /// Plain Unicode property setter (the generated accessor used to call string.Normalize). + [Test] + public void PlainUnicodeProperty_LoneSurrogate_DoesNotThrow() + { + var entry = Cache.ServiceLocator.GetInstance().Create(); + Assert.That(() => entry.LiftResidue = "a\uD800b", Throws.Nothing); + Assert.That(entry.LiftResidue, Is.EqualTo("a\uFFFDb")); + entry.LiftResidue = "\uDC00x\uD800"; + Assert.That(entry.LiftResidue, Is.EqualTo("\uFFFDx\uFFFD")); + } + + /// WfiWordformRepository.TryGetObject used to normalize the target text with string.Normalize. + [Test] + public void WordformTryGetObject_LoneSurrogateTarget_DoesNotThrow() + { + var wf = Cache.ServiceLocator.GetInstance().Create( + TsStringUtils.MakeString("abc", Cache.DefaultVernWs)); + var repo = Cache.ServiceLocator.GetInstance(); + IWfiWordform found; + Assert.That(() => repo.TryGetObject(TsStringUtils.MakeString("b\uDC00c", Cache.DefaultVernWs), out found), + Throws.Nothing); + Assert.That(repo.TryGetObject(TsStringUtils.MakeString("abc", Cache.DefaultVernWs), out found), Is.True); + Assert.That(found, Is.SameAs(wf)); + } + } +} diff --git a/tests/SIL.LCModel.Tests/DomainServices/LoneSurrogateModelTests.cs b/tests/SIL.LCModel.Tests/DomainServices/LoneSurrogateModelTests.cs new file mode 100644 index 00000000..9ad2badf --- /dev/null +++ b/tests/SIL.LCModel.Tests/DomainServices/LoneSurrogateModelTests.cs @@ -0,0 +1,256 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using System.IO; +using System.Linq; +using System.Text; +using NUnit.Framework; +using SIL.LCModel.Core.Cellar; +using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.Text; +using SIL.LCModel.Infrastructure; +using SIL.TestUtilities; + +namespace SIL.LCModel.DomainServices +{ + /// + /// On master a lone (unpaired) surrogate could never enter the model: every TsString setter threw while + /// normalizing. Now the setters replace it with U+FFFD, so stored text equals in-memory text and nothing + /// that indexes into a paragraph (segment offsets, analyses) can drift after a save and reload. + /// NOTE: the strings are built in code with \u escapes; NUnit attributes would turn lone surrogates into U+FFFD. + /// + [TestFixture] + public class LoneSurrogateModelTests : MemoryOnlyBackendProviderRestoredForEachTestTestBase + { + private const string High = "\uD800"; + private const string Low = "\uDC00"; + private const string Fffd = "\uFFFD"; + + // The test base already has a unit of work open for each test. + private static void InUow(Action task) + { + task(); + } + + private IStTxtPara AddPara(ITsString contents) + { + IStTxtPara para = null; + InUow(() => + { + var text = Cache.ServiceLocator.GetInstance().Create(); + var stText = Cache.ServiceLocator.GetInstance().Create(); + text.ContentsOA = stText; + para = stText.AddNewTextPara(null); + para.Contents = contents; + }); + return para; + } + + // Built in code and named in ASCII: NUnit attribute arguments would turn lone surrogates into U+FFFD. + private static TestCaseData[] SetterCases() + { + return new[] + { + new TestCaseData("a" + High + "b", "a" + Fffd + "b").SetName("Setter_LoneHighMiddle"), + new TestCaseData("a" + Low + "b", "a" + Fffd + "b").SetName("Setter_LoneLowMiddle"), + new TestCaseData(High + "ab", Fffd + "ab").SetName("Setter_LoneHighAtStart"), + new TestCaseData(Low + "ab", Fffd + "ab").SetName("Setter_LoneLowAtStart"), + new TestCaseData("ab" + High, "ab" + Fffd).SetName("Setter_LoneHighAtEnd"), + new TestCaseData("ab" + Low, "ab" + Fffd).SetName("Setter_LoneLowAtEnd"), + }; + } + + private static TestCaseData[] ParserCases() + { + return new[] + { + new TestCaseData(Low + "ab cd. Ef gh.").SetName("Parse_LoneLowAtStart"), + new TestCaseData("b" + Low + "c de.").SetName("Parse_LoneLowAfterLetter"), + new TestCaseData("b" + High + "c de.").SetName("Parse_LoneHighAfterLetter"), + new TestCaseData(High + "ab cd. Ef gh.").SetName("Parse_LoneHighAtStart"), + new TestCaseData("ab cd. Ef gh." + Low).SetName("Parse_LoneLowAtEnd"), + new TestCaseData("ab cd. Ef gh." + High).SetName("Parse_LoneHighAtEnd"), + new TestCaseData("ab " + Low + " cd" + High + ". " + Low + High).SetName("Parse_Mixed"), + }; + } + + [TestCaseSource(nameof(SetterCases))] + public void ParagraphContents_Setter_ReplacesLoneSurrogate(string text, string expected) + { + var para = AddPara(TsStringUtils.MakeString(text, Cache.DefaultVernWs)); + Assert.That(para.Contents.Text, Is.EqualTo(expected)); + } + + [Test] + public void MultiStringAndMultiUnicode_Setters_ReplaceLoneSurrogate() + { + int ws = Cache.DefaultAnalWs; + InUow(() => + { + Cache.LangProject.PartsOfSpeechOA.Name.set_String(ws, TsStringUtils.MakeString("n" + High + "m", ws)); + Cache.LangProject.PartsOfSpeechOA.Description.set_String(ws, TsStringUtils.MakeString("d" + Low + "e", ws)); + }); + Assert.That(Cache.LangProject.PartsOfSpeechOA.Name.get_String(ws).Text, Is.EqualTo("n" + Fffd + "m")); + Assert.That(Cache.LangProject.PartsOfSpeechOA.Description.get_String(ws).Text, Is.EqualTo("d" + Fffd + "e")); + } + + [Test] + public void CustomStringField_SetString_ReplacesLoneSurrogate_AndNormalizes() + { + var mdc = Cache.ServiceLocator.GetInstance(); + int flid = mdc.AddCustomField("WfiWordform", "LoneSurrogateProp" + Guid.NewGuid().ToString("N"), + CellarPropertyType.String, 0); + IWfiWordform wf = null; + InUow(() => + { + wf = Cache.ServiceLocator.GetInstance().Create(); + Cache.DomainDataByFlid.SetString(wf.Hvo, flid, + TsStringUtils.MakeString("c" + High + "d" + Low + "\u00E9", Cache.DefaultVernWs)); + }); + Assert.That(Cache.DomainDataByFlid.get_StringProp(wf.Hvo, flid).Text, + Is.EqualTo("c" + Fffd + "d" + Fffd + "e\u0301")); + } + + [Test] + public void DomainDataByFlid_SetString_ReplacesLoneSurrogate() + { + var para = AddPara(TsStringUtils.MakeString("x", Cache.DefaultVernWs)); + InUow(() => + Cache.DomainDataByFlid.SetString(para.Hvo, StTxtParaTags.kflidContents, + TsStringUtils.MakeString("p" + High + "q", Cache.DefaultVernWs))); + Assert.That(para.Contents.Text, Is.EqualTo("p" + Fffd + "q")); + } + + /// + /// Even text that got into the model some other way (here by bypassing the setter, as undo and load do) + /// must not make the parser throw. Lone highs and lows at the start, middle and end. + /// + [TestCaseSource(nameof(ParserCases))] + public void ParagraphParser_TextWithLoneSurrogateThatBypassedTheSetter_DoesNotThrow(string text) + { + var para = AddPara(TsStringUtils.MakeString("placeholder", Cache.DefaultVernWs)); + InUow(() => + ((ICmObjectInternal)para).SetProperty(StTxtParaTags.kflidContents, + TsStringUtils.MakeString(text, Cache.DefaultVernWs), false)); + Assert.That(para.Contents.Text, Is.EqualTo(text), "the setter was bypassed"); + InUow(() => + Assert.DoesNotThrow(() => ParagraphParser.ParseParagraph(para))); + } + + [Test] + public void ParagraphParser_Fuzz_NeverThrows() + { + var rng = new Random(414); + string[] alphabet = { "a", "b", "c", " ", ".", "\u00E9", "\u0301", "\U0001F98A", High, Low, "\uDBFF", "\uDFFF" }; + var para = AddPara(TsStringUtils.MakeString("placeholder", Cache.DefaultVernWs)); + for (int n = 0; n < 150; n++) + { + var sb = new StringBuilder(); + int len = rng.Next(1, 10); + for (int i = 0; i < len; i++) + sb.Append(alphabet[rng.Next(alphabet.Length)]); + string text = sb.ToString(); + string description = string.Join(" ", text.Select(c => ((int)c).ToString("X4"))); + InUow(() => + { + ((ICmObjectInternal)para).SetProperty(StTxtParaTags.kflidContents, + TsStringUtils.MakeString(text, Cache.DefaultVernWs), false); + Assert.DoesNotThrow(() => ParagraphParser.ParseParagraph(para), description); + }); + } + } + } + + /// + /// The scenario that matters: a paragraph with a lone surrogate, parsed, saved with the XML backend, + /// and reloaded must still have segments that line up with its text. + /// + [TestFixture] + public class LoneSurrogateSaveReloadTests + { + private TemporaryFolder m_projectsFolder; + private ILcmDirectories m_lcmDirectories; + + [SetUp] + public void TestSetup() + { + m_projectsFolder = new TemporaryFolder("LoneSurrogate" + Guid.NewGuid().ToString("N")); + m_lcmDirectories = new TestLcmDirectories(m_projectsFolder.Path); + } + + [TearDown] + public void TestTeardown() + { + m_projectsFolder.Dispose(); + } + + [Test] + public void CustomUnicodeField_LoneSurrogate_DoesNotBreakSave() + { + var projectName = "LoneSurrogateCustom" + new Random().Next(1000000); + var path = Path.Combine(m_projectsFolder.Path, LcmFileHelper.GetXmlDataFileName(projectName)); + var projectId = new TestProjectId(BackendProviderType.kXMLWithMemoryOnlyWsMgr, path); + using (var cache = LcmCache.CreateCacheWithNewBlankLangProj(projectId, "en", "fr", "en", + new DummyLcmUI(), m_lcmDirectories, new LcmSettings())) + { + var mdc = cache.ServiceLocator.GetInstance(); + int flid = mdc.AddCustomField("LexEntry", "LoneSurrogateUnicode", CellarPropertyType.Unicode, 0); + ILexEntry entry = null; + UndoableUnitOfWorkHelper.Do("doit", "undoit", cache.ActionHandlerAccessor, () => + { + entry = cache.ServiceLocator.GetInstance().Create(); + cache.DomainDataByFlid.set_UnicodeProp(entry.Hvo, flid, "a\uD800b"); + }); + Assert.That(cache.DomainDataByFlid.get_UnicodeProp(entry.Hvo, flid), Is.EqualTo("a\uFFFDb")); + Assert.DoesNotThrow(() => cache.ServiceLocator.GetInstance().Save()); + } + } + + [Test] + public void ParsedParagraphWithLoneSurrogate_SegmentOffsetsSurviveSaveAndReload() + { + var projectName = "LoneSurrogate" + new Random().Next(1000000); + var path = Path.Combine(m_projectsFolder.Path, LcmFileHelper.GetXmlDataFileName(projectName)); + var projectId = new TestProjectId(BackendProviderType.kXMLWithMemoryOnlyWsMgr, path); + + Guid paraGuid; + int[] offsetsBefore; + string[] baselinesBefore; + string textBefore; + using (var cache = LcmCache.CreateCacheWithNewBlankLangProj(projectId, "en", "fr", "en", + new DummyLcmUI(), m_lcmDirectories, new LcmSettings())) + { + IStTxtPara para = null; + UndoableUnitOfWorkHelper.Do("doit", "undoit", cache.ActionHandlerAccessor, () => + { + var text = cache.ServiceLocator.GetInstance().Create(); + var stText = cache.ServiceLocator.GetInstance().Create(); + text.ContentsOA = stText; + para = stText.AddNewTextPara(null); + para.Contents = TsStringUtils.MakeString("One a\uD800b two. Three four. Five six.", cache.DefaultVernWs); + ParagraphParser.ParseParagraph(para); + }); + paraGuid = para.Guid; + textBefore = para.Contents.Text; + offsetsBefore = para.SegmentsOS.Select(s => s.BeginOffset).ToArray(); + baselinesBefore = para.SegmentsOS.Select(s => s.BaselineText.Text).ToArray(); + Assert.That(textBefore, Is.EqualTo("One a\uFFFDb two. Three four. Five six.")); + Assert.That(offsetsBefore, Is.EqualTo(new[] { 0, 13, 25 }), "sanity: three segments"); + + cache.ServiceLocator.GetInstance().Save(); + } + + using (var reloaded = LcmCache.CreateCacheFromExistingData(projectId, "en", new DummyLcmUI(), + m_lcmDirectories, new LcmSettings(), new DummyProgressDlg())) + { + var para = (IStTxtPara)reloaded.ServiceLocator.GetObject(paraGuid); + Assert.That(para.Contents.Text, Is.EqualTo(textBefore), "text length must not change on save"); + Assert.That(para.SegmentsOS.Select(s => s.BeginOffset).ToArray(), Is.EqualTo(offsetsBefore)); + Assert.That(para.SegmentsOS.Select(s => s.BaselineText.Text).ToArray(), Is.EqualTo(baselinesBefore)); + Assert.That(para.ParseIsCurrent, Is.True); + } + } + } +} diff --git a/tests/SIL.LCModel.Tests/DomainServices/WordMakerLoneSurrogateTests.cs b/tests/SIL.LCModel.Tests/DomainServices/WordMakerLoneSurrogateTests.cs new file mode 100644 index 00000000..8f351481 --- /dev/null +++ b/tests/SIL.LCModel.Tests/DomainServices/WordMakerLoneSurrogateTests.cs @@ -0,0 +1,31 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using NUnit.Framework; +using SIL.LCModel.Core.Text; +using SIL.LCModel.Core.WritingSystems; + +namespace SIL.LCModel.DomainServices +{ + /// + /// WordMaker must not throw on text containing an unpaired surrogate (built in code, since NUnit + /// attributes would turn it into U+FFFD). + /// + [TestFixture] + public class WordMakerLoneSurrogateTests + { + [Test] + public void IsWhite_LoneSurrogate_DoesNotThrow_AndIsNotWhite() + { + var wsManager = new WritingSystemManager(); + CoreWritingSystemDefinition enWs; + wsManager.GetOrSet("en", out enWs); + var tss = TsStringUtils.MakeString("a\uD800 \uDC00", enWs.Handle); + var maker = new WordMaker(tss, wsManager); + Assert.That(maker.IsWhite(1), Is.False); // lone high surrogate + Assert.That(maker.IsWhite(2), Is.True); // the space + Assert.That(maker.IsWhite(3), Is.False); // lone low surrogate at end + } + } +}