Skip to content
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- [SIL.LCModel] Add GetCaptionOrHeadword() to CmPicture
- [SIL.LCModel] `LCModelStrings.NotSure` to allow clients to know if a grammatical category is the placeholder
- [SIL.LCModel.Utils] `DateTime` extension method `ToLCMTimeFormatWithMillisString()` (replaces `ReadWriteServices.FormatDateTime`)
- [SIL.LCModel.Utils] `Surrogates.CodePointAt(string, int)` (a `char.ConvertToUtf32` that tolerates unpaired surrogates), `HasLoneSurrogate`, `ReplaceLoneSurrogates` and `StringFromCodePointOrLoneSurrogate`; [SIL.LCModel.Core] `TsStringUtils.ReplaceLoneSurrogates(ITsString)`

### Fixed

Expand All @@ -39,9 +40,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- [SIL.LCModel.Core] Copy `SIL.LCModel.Core.dll.config` to output directory
- [SIL.LCModel] Use `CaseFunctions` (to use the `WritingSystemDefinition.CaseAlias`, if any)
- [SIL.LCModel.Core] Use the `WritingSystemDefinition.CaseAlias`, if any, in `CaseFunctions`
- [SIL.LCModel.Core] Text containing an unpaired (lone) surrogate no longer makes `TsString` normalization, `TsStringSerializer.SerializeTsStringToXml`, `TextExtensions.CharAt`, `TextExtensions.IsCharWordForming` or `WordMaker.IsWhite` throw. Normalization passes the surrogate through unchanged, as ICU does. `IsCharWordForming` no longer treats a lone low surrogate as the second half of a pair, so it is no longer classified by the character before it. `TsStringSerializer` writes a lone surrogate as U+FFFD (it used to throw), so the written text is as long as the text in memory, and decides `xml:space` from the text it writes

### Changed

- [SIL.LCModel] The generated and custom plain `Unicode` property setters now replace a lone (unpaired) surrogate with U+FFFD instead of throwing, and `WfiWordformRepository.TryGetObject` treats a lone surrogate in the looked-up form as U+FFFD instead of throwing
- [SIL.LCModel.Core] `TsStringUtils.NormalizeNfd(ITsString)` replaces each unpaired surrogate with U+FFFD, so the TsString setters (String, MultiString and MultiUnicode properties, custom String fields, `MultiAccessor.set_String`, `IDomainDataByFlid.SetString` and `SetMultiStringAlt`) store U+FFFD for text containing one. This used to throw, so a caller that relied on the exception now gets U+FFFD silently; that includes direct callers of `NormalizeNfd(ITsString)`. Custom String fields are now also stored as NFD in memory when set (as model String properties already were); the saved files are unchanged. A valid surrogate pair split across two runs is read as two lone surrogates when written to XML
- [SIL.LCModel] The domain-model `Generated*.cs` sources are now produced at compile time by the new `SIL.LCModel.SourceGenerators` Roslyn source generator, instead of the `LcmGenerate` MSBuild task running the NVelocity templates in a child MSBuild process. The NVelocity engine and model wrappers moved to a shared `SIL.LCModel.ModelGeneration` project used by both the source generator and the (retained) `LcmGenerate` task, which still renders other templates from the model (e.g. the FieldWorks C++ cellar-constants header). Templates and generated output are byte-for-byte identical. No public API change.
- [SIL.LCModel] Replaced the internal StructureMap (`structuremap.patched`) IoC container with `Microsoft.Extensions.DependencyInjection` (8.x). No public API change.
- [SIL.LCModel] Trim 12 overwordy semantic domain descriptions and fix 22 punctuation/whitespace issues in the SemDom.xml template, matching sillsdev/FwLocalizations#5 and sillsdev/FwLocalizations#7
Expand Down
7 changes: 4 additions & 3 deletions src/SIL.LCModel.Core/Text/TextExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -356,10 +356,11 @@ public static bool IsCharWordForming(this ITsString tss, int ich, ILgWritingSyst
if (ich < 0 || ich >= tss.Length)
throw new ArgumentOutOfRangeException("ich");

if (char.IsLowSurrogate(tss.Text[ich]))
// Step back only over the second half of a real pair; a lone low surrogate is a character of its own.
if (ich > 0 && char.IsLowSurrogate(tss.Text[ich]) && char.IsHighSurrogate(tss.Text[ich - 1]))
--ich;
int wsHandle = tss.get_WritingSystemAt(ich);
int ch = char.ConvertToUtf32(tss.Text, ich);
int ch = Surrogates.CodePointAt(tss.Text, ich);
return TsStringUtils.IsWordForming(ch, wsf, wsHandle);
}

Expand All @@ -371,7 +372,7 @@ public static int CharAt(this ITsString tss, int ich)
if (ich < 0 || ich >= tss.Length)
throw new ArgumentOutOfRangeException("ich");

return char.ConvertToUtf32(tss.Text, ich);
return Surrogates.CodePointAt(tss.Text, ich);
}

/// <summary>
Expand Down
10 changes: 5 additions & 5 deletions src/SIL.LCModel.Core/Text/TsString.cs
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ private IEnumerable<int> EnumerateSegmentLimits(Normalizer2 icuNormalizer)
int i = 0;
while (i < Text.Length)
{
int codepoint = Char.ConvertToUtf32(Text, i);
int codepoint = Surrogates.CodePointAt(Text, i);
if (icuNormalizer.HasBoundaryBefore(codepoint) && i > 0)
{
yield return i;
Expand All @@ -302,15 +302,15 @@ private IEnumerable<int> EnumerateSegmentLimits(Normalizer2 icuNormalizer)
/// <summary>
/// Helper function used in MatchUpIndexesAfterNormalization, itself a helper function for get_NormalizedForm
/// </summary>
/// <param name="s">String to convert to UTF32 codepoints. Must not be null, and must not end with an unpaired surrogate.</param>
/// <param name="s">String to convert to UTF32 codepoints. Must not be null, and may contain unpaired surrogates, which are reported as their own code unit value.</param>
/// <returns>A sequence of KeyValuePairs where the key is the index (in chars) of this codepoint in the original string, and the value is the Unicode codepoint at that index.</returns>
private List<KeyValuePair<int, int>> CodepointsByIndex(string s)
{
int len = s.Length;
var result = new List<KeyValuePair<int, int>>(len);
for (int i = 0; i < len; i++)
{
int codePoint = Char.ConvertToUtf32(s, i);
int codePoint = Surrogates.CodePointAt(s, i);
result.Add(new KeyValuePair<int, int>(i, codePoint));
if (codePoint > 0xffff)
i++; // Skip second half of a surrogate pair
Expand Down Expand Up @@ -359,7 +359,7 @@ private IEnumerable<RearrangedIndexMapping> MatchUpIndexesAfterNormalization(
{
int origIdx = indexAndCodePoint.Key;
int origCodePoint = indexAndCodePoint.Value;
var normalizedStringFromOrigCodePoint = icuNormalizer.GetDecomposition(origCodePoint) ?? char.ConvertFromUtf32(origCodePoint);
var normalizedStringFromOrigCodePoint = icuNormalizer.GetDecomposition(origCodePoint) ?? Surrogates.StringFromCodePointOrLoneSurrogate(origCodePoint);
foreach (KeyValuePair<int, int> indexAndResultingCodePoint in CodepointsByIndex(normalizedStringFromOrigCodePoint))
{
int resultingCodePoint = indexAndResultingCodePoint.Value;
Expand Down Expand Up @@ -485,7 +485,7 @@ private ITsString get_NormalizedFormAndFixOffsets(FwNormalizationMode nm, ArrayP
{
ITsTextProps origProperties = get_PropertiesAt(segmentMin + mapping.origIdx);
int outputIdx = outputLenSoFar + mapping.normIdx;
int size = Char.IsSurrogate(normalizedSegment, mapping.normIdx) ? 2 : 1;
int size = Char.IsSurrogatePair(normalizedSegment, mapping.normIdx) ? 2 : 1;
resultBuilder.SetProperties(outputIdx, outputIdx + size, origProperties);
// And if we also need to fix up offsets at the end, we keep track of the ones we'll need
if (willFixOffsets && mapping.isFirstCharOfDecomposition)
Expand Down
23 changes: 20 additions & 3 deletions src/SIL.LCModel.Core/Text/TsStringSerializer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,11 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto
}
else
{
string runText = normalizedTss.get_RunText(i) ?? string.Empty;
if (runText != string.Empty && runText.All(char.IsWhiteSpace))
string writtenText = GetXmlSafeText(normalizedTss.get_RunText(i) ?? string.Empty);
if (writtenText != string.Empty && writtenText.All(char.IsWhiteSpace))
writer.WriteAttributeString("xml", "space", "", "preserve");
// TODO: should we escape quotation marks? this is not necessary but different than the behavior of the C++ implementation
writer.WriteString(StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC)));
writer.WriteString(writtenText);
}

writer.WriteEndElement();
Expand All @@ -196,6 +196,23 @@ public static string StripInvalidXmlChars(string text)
return InvalidXmlRegex.Replace(text, string.Empty);
}

/// <summary>
/// Gets the NFC form of text for writing to XML, without the characters XML cannot represent.
/// Unpaired surrogates are replaced with U+FFFD rather than removed, so that the written text is
/// as long as the text in memory (offsets of segments, analyses and notes index into it).
/// (Model setters already replace them via <see cref="TsStringUtils.NormalizeNfd(ITsString)"/>;
/// this covers TsStrings that never went through the model.)
/// Characters are cleaned before normalizing because a lone surrogate is a starter: removing it
/// after NFC could leave an uncomposed sequence such as "e" + U+0301.
/// </summary>
internal static string GetXmlSafeText(string text)
{
// Replace lone surrogates first: removing a control character can bring a high and a low surrogate
// together into a "pair" that was never one, and the result must not depend on whether
// StripInvalidXmlChars itself removes lone surrogates.
string cleaned = StripInvalidXmlChars(Surrogates.ReplaceLoneSurrogates(text));
return StripInvalidXmlChars(Normalizer.Normalize(cleaned, Normalizer.UNormalizationMode.UNORM_NFC));
}
#endregion

#region Serialization Helper Methods
Expand Down
23 changes: 23 additions & 0 deletions src/SIL.LCModel.Core/Text/TsStringUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,9 @@ public static string NormalizeNfd(string source)
/// ------------------------------------------------------------------------------------
public static ITsString NormalizeNfd(ITsString source)
{
// An unpaired surrogate cannot be stored (it cannot be written to XML). Replace it up front, with a
// same-length character, so that the stored text always equals the in-memory text.
source = ReplaceLoneSurrogates(source);
if (source == null || source.Length == 0 || source.get_IsNormalizedForm(FwNormalizationMode.knmNFD))
{
return source;
Expand All @@ -565,6 +568,26 @@ public static ITsString NormalizeNfd(ITsString source)
return source.get_NormalizedForm(FwNormalizationMode.knmNFD);
}

/// <summary>
/// Replaces each unpaired surrogate in the string with U+FFFD, keeping the text properties of its run
/// and the length of the string. Returns the same object if there is nothing to replace.
/// </summary>
public static ITsString ReplaceLoneSurrogates(ITsString source)
{
if (source == null || !Surrogates.HasLoneSurrogate(source.Text))
return source;
string text = source.Text;
ITsStrBldr bldr = source.GetBldr();
for (int i = 0; i < text.Length; i++)
{
if (char.IsHighSurrogate(text[i]) && i < text.Length - 1 && char.IsLowSurrogate(text[i + 1]))
i++;
else if (char.IsSurrogate(text[i]))
bldr.Replace(i, i + 1, "\uFFFD", source.get_PropertiesAt(i));
}
return bldr.GetString();
}

#region ORC-related methods
/// ------------------------------------------------------------------------------------
/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,8 @@ internal set
var newValue = value;
Validate${prop.NiuginianPropName}(ref newValue);
if (!string.IsNullOrEmpty(newValue))
newValue = newValue.Normalize(NormalizationForm.FormD); // Store it as NFD in LCM.
// Store it as NFD in LCM. An unpaired surrogate (which Normalize rejects) becomes U+FFFD.
newValue = SIL.LCModel.Utils.Surrogates.ReplaceLoneSurrogates(newValue).Normalize(NormalizationForm.FormD);
var originalValue = m_$prop.NiuginianPropName;
if (newValue == originalValue)
return;
Expand Down
62 changes: 62 additions & 0 deletions src/SIL.LCModel.Utils/Surrogates.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,68 @@ public static string StringFromCodePoint(int codepoint)
return char.ConvertFromUtf32(codepoint);
}

/// <summary>
/// Return the full code point starting at index ich of the string. A high surrogate followed by a
/// low surrogate gives the supplementary code point. Anything else gives the char itself: that includes
/// an unpaired surrogate, and also a low surrogate that is the second half of a pair when ich points at
/// it (so the pair is not re-read from its middle). Unlike char.ConvertToUtf32(string, int), this never
/// throws on an unpaired surrogate (which can occur in pasted or imported text).
/// </summary>
public static int CodePointAt(string st, int ich)
{
char ch = st[ich];
if (IsLeadSurrogate(ch) && ich < st.Length - 1 && IsTrailSurrogate(st[ich + 1]))
return char.ConvertToUtf32(ch, st[ich + 1]);
return ch;
}

/// <summary>
/// Gives the string for a code point as returned by <see cref="CodePointAt"/>: unlike
/// <see cref="StringFromCodePoint"/>, an unpaired surrogate code unit gives that one char instead of throwing.
/// </summary>
public static string StringFromCodePointOrLoneSurrogate(int codepoint)
{
if (codepoint >= kMinLeadSurrogate && codepoint <= 0xDFFF)
return ((char)codepoint).ToString();
return char.ConvertFromUtf32(codepoint);
}

/// <summary>
/// Whether the string contains a surrogate that is not half of a valid high+low pair.
/// </summary>
public static bool HasLoneSurrogate(string st)
{
if (st == null)
return false;
for (int i = 0; i < st.Length; i++)
{
if (IsLeadSurrogate(st[i]) && i < st.Length - 1 && IsTrailSurrogate(st[i + 1]))
i++;
else if (IsLeadSurrogate(st[i]) || IsTrailSurrogate(st[i]))
return true;
}
return false;
}

/// <summary>
/// Replace each surrogate that is not half of a valid pair with U+FFFD REPLACEMENT CHARACTER.
/// The length of the string never changes, so offsets into it stay valid.
/// </summary>
public static string ReplaceLoneSurrogates(string st)
{
if (!HasLoneSurrogate(st))
return st;
var chars = st.ToCharArray();
for (int i = 0; i < chars.Length; i++)
{
if (IsLeadSurrogate(chars[i]) && i < chars.Length - 1 && IsTrailSurrogate(chars[i + 1]))
i++;
else if (IsLeadSurrogate(chars[i]) || IsTrailSurrogate(chars[i]))
chars[i] = '\uFFFD';
}
return new string(chars);
}

/// <summary>
/// Return a full 32-bit character value from the surrogate pair.
/// </summary>
Expand Down
10 changes: 10 additions & 0 deletions src/SIL.LCModel/DomainImpl/CmObject.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2921,6 +2921,16 @@ private void SetCustomPropertyForSDA(int flid, object newValue, bool useAccessor
// break;
// case CellarPropertyType.Image: // Not in model.
// case CellarPropertyType.Numeric: // Not in model.
case CellarPropertyType.String:
// Same input rules as a model String property: NFD, and no unpaired surrogates.
if (newValue is ITsString newTss)
newValue = TsStringUtils.NormalizeNfd(newTss);
break;
case CellarPropertyType.Unicode:
// Same input rules as a generated Unicode property: NFD, and no unpaired surrogates.
if (newValue is string newString && newString.Length > 0)
newValue = Surrogates.ReplaceLoneSurrogates(newString).Normalize(NormalizationForm.FormD);
break;
case CellarPropertyType.Guid:
if (newValue == null)
newValue = Guid.Empty;
Expand Down
2 changes: 1 addition & 1 deletion src/SIL.LCModel/DomainServices/ITextUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1560,7 +1560,7 @@ public int NextChar(int ich)
/// <returns></returns>
public bool IsWhite(int ich)
{
int ch = char.ConvertToUtf32(m_st, ich);
int ch = Surrogates.CodePointAt(m_st, ich);
return Character.GetCharType(ch) == Character.UCharCategory.SPACE_SEPARATOR || ch == AnalysisOccurrence.KchZws;
}

Expand Down
4 changes: 2 additions & 2 deletions src/SIL.LCModel/Infrastructure/Impl/RepositoryAdditions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -888,11 +888,11 @@ public bool TryGetObject(ITsString tssTarget, out IWfiWordform wf)
{
ITsString tssKey = wfT.Form.get_String(ws);
if (tssKey.Length > 0)
lookupTable[tssKey.Text.Normalize(NormalizationForm.FormD)] = wfT;
lookupTable[Surrogates.ReplaceLoneSurrogates(tssKey.Text).Normalize(NormalizationForm.FormD)] = wfT;
}
}
}
return lookupTable.TryGetValue(tssTarget.Text.Normalize(NormalizationForm.FormD), out wf);
return lookupTable.TryGetValue(Surrogates.ReplaceLoneSurrogates(tssTarget.Text).Normalize(NormalizationForm.FormD), out wf);
}

void IWfiWordformRepositoryInternal.UpdateForm(ITsString oldForm, IWfiWordform wf, int ws)
Expand Down
Loading
Loading