From 3c39f908a247c52b29d27f7e65bc187a73019ca1 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 5 Nov 2026 17:11:09 -0500 Subject: [PATCH 1/4] Fix unknown ws handle written as ws="" by text props serializers (#411) Writer: omit ws/wsBase props whose handle the factory does not know (logged with Debug.WriteLine); throw instead of writing a text run or with an unknown ws, since readers require a ws on text runs. Reader: ignore an empty ws/wsBase/WsProp ws so existing records load. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + .../Text/TsPropsSerializer.cs | 32 +++-- .../Text/TsStringSerializer.cs | 18 ++- .../Text/TsUnknownWsHandleTests.cs | 115 ++++++++++++++++++ 4 files changed, 155 insertions(+), 11 deletions(-) create mode 100644 tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a172e5d0..4b7b3614d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +- [SIL.LCModel.Core] `TsPropsSerializer` no longer writes `ws=""`/`wsBase=""` for a writing-system handle the factory does not know (the attribute is omitted instead), and `TsStringSerializer` throws instead of writing a run or `` with an unknown ws. Readers now ignore an existing empty `ws`/`wsBase`, so records saved with `ws=""` load (#411) - [SIL.LCModel] BaseStyleInfo now loads default font features (ktptFontVariations) from style rules, fixing font features set as style defaults being dropped on reload (LT-22351) - [SIL.LCModel] BaseStyleInfo no longer clobbers ws-specific font overrides inherited from a based-on style with the default font info's merely-inherited values; only explicitly-set default properties are propagated to ws overrides (LT-22351) - [SIL.LCModel] BulletInfo now decodes all properties of an encoded bullet font info instead of stopping at the first string property, fixing bullet font features (ktptFontVariations) being lost when a bullet font name was also set (LT-22351) diff --git a/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs b/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs index 5b07b0e88..63ac2846a 100644 --- a/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs @@ -95,7 +95,13 @@ internal static void WriteIntProperty(XmlWriter writer, ILgWritingSystemFactory if (value > 0) { string id = lgwsf.GetStrFromWs(value); - writer.WriteAttributeString(tpt == FwTextPropType.ktptWs ? "ws" : "wsBase", Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); + string attrName = tpt == FwTextPropType.ktptWs ? "ws" : "wsBase"; + // An unknown handle has no id. Writing it as ws="" would produce XML that no reader accepts + // (the property is optional, so it is safer to leave it out). + if (string.IsNullOrEmpty(id)) + Debug.WriteLine("TsPropsSerializer: omitting '" + attrName + "' attribute for unknown writing system handle " + value); + else + writer.WriteAttributeString(attrName, Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); } break; case FwTextPropType.ktptFontSize: @@ -863,6 +869,8 @@ private static void AddWsStyleInfoToBldr(ITsPropsBldr bldr, XElement wsStyleInfo if (wsAttrib == null) throw new XmlSchemaException("WsProp must contain a 'ws' attribute"); wsAttrib.Remove(); // Make sure we don't count it twice + if (wsAttrib.Value.Length == 0) + continue; // tolerate an empty ws written by older versions for an unknown handle WriteIntToStrBuilder(wsStyleProps, TsStringSerializer.GetWsForId(wsAttrib.Value, lgwsf)); // Get the font family for the prop override @@ -1138,12 +1146,15 @@ internal static ITsPropsBldr GetPropAttributesForElement(XElement xml, ILgWritin GetIntValueFromStr(attr.Value, false)); break; case "ws": - propsBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + // Older versions wrote ws="" for an unknown handle; drop it rather than failing to load. + if (attr.Value.Length > 0) + propsBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsBase": - propsBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + if (attr.Value.Length > 0) + propsBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsStyle": Debug.Fail("We don't support the old wsStyle attribute!"); @@ -1395,12 +1406,15 @@ internal static bool GetPropAttributesForElement(XElement xml, ILgWritingSystemF GetIntValueFromStr(attr.Value, false)); break; case "ws": - strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + // Older versions wrote ws="" for an unknown handle; drop it rather than failing to load. + if (attr.Value.Length > 0) + strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsBase": - strBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + if (attr.Value.Length > 0) + strBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsStyle": Debug.Fail("We don't support the old wsStyle attribute!"); diff --git a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs index 173b6a7ab..5ea84b1ed 100644 --- a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs @@ -51,6 +51,8 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto if (ws > 0) { string id = lgwsf.GetStrFromWs(ws); + if (string.IsNullOrEmpty(id)) + throw new ArgumentException("Unknown writing system handle " + ws + "; cannot write an without a ws.", "ws"); writer.WriteStartElement("AStr"); writer.WriteAttributeString("ws", Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); } @@ -89,6 +91,18 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto fieldName = runFieldName; } + // A run of ordinary text must carry a ws (the reader rejects it otherwise), and an unknown handle + // has no id to write. Fail here rather than save a record that cannot be loaded. + int runWs; + FwTextPropVar wsVar; + if (textProps.TryGetIntValue(FwTextPropType.ktptWs, out wsVar, out runWs) && runWs > 0 + && string.IsNullOrEmpty(lgwsf.GetStrFromWs(runWs))) + { + string wsCheckText = normalizedTss.get_RunText(i) ?? string.Empty; + if (wsCheckText.Length > 0 && wsCheckText[0] > 13) + throw new InvalidOperationException("Cannot serialize run with unknown writing system handle " + runWs + ". Run text: " + wsCheckText); + } + bool markItem; FwTextPropVar var; int markItemValue; @@ -323,8 +337,8 @@ private static ITsString HandleSimpleString(XElement rootXml, ILgWritingSystemFa return null; // Way too complex for this simple case XAttribute wsAttribute = textElement.Attributes().First(); - if (wsAttribute.Name.LocalName != "ws") - return null; // we handle only single runs with only the ws attribute. + if (wsAttribute.Name.LocalName != "ws" || wsAttribute.Value.Length == 0) + return null; // we handle only single runs with only a (non-empty) ws attribute; an empty ws, written by older versions for an unknown handle, is dropped by the general path. // Make sure the text is in the decomposed form (FWR-148) string runText = Normalizer.Normalize(textElement.Value, Normalizer.UNormalizationMode.UNORM_NFD); return TsStringUtils.MakeString(runText, GetWsForId(wsAttribute.Value, lgwsf)); diff --git a/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs new file mode 100644 index 000000000..9bbdf75e1 --- /dev/null +++ b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs @@ -0,0 +1,115 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using NUnit.Framework; +using SIL.LCModel.Core.KernelInterfaces; + +namespace SIL.LCModel.Core.Text +{ + /// + /// An unknown writing system handle used to be written as ws="", which no reader accepts. + /// + [TestFixture] + public class TsUnknownWsHandleTests : TsSerializerTestsBase + { + private const int UnknownWs = 987654; + + private ITsTextProps MakeProps(FwTextPropType wsType, int ws) + { + ITsPropsBldr bldr = TsStringUtils.MakePropsBldr(); + bldr.SetStrPropValue((int)FwTextPropType.ktptNamedStyle, "Emphasis"); + bldr.SetIntPropValues((int)wsType, 0, ws); + return bldr.GetTextProps(); + } + + [TestCase(FwTextPropType.ktptWs, "ws")] + [TestCase(FwTextPropType.ktptBaseWs, "wsBase")] + public void SerializeProps_UnknownWs_OmitsAttributeAndRoundTrips(FwTextPropType wsType, string attrName) + { + string xml = TsPropsSerializer.SerializePropsToXml(MakeProps(wsType, UnknownWs), WritingSystemManager); + + Assert.That(xml, Does.Not.Contain(attrName + "=\"\"")); + Assert.That(xml, Does.Not.Contain(attrName + "=")); + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); + int var; + Assert.That(result.GetIntPropValues((int)wsType, out var), Is.EqualTo(-1)); + } + + [Test] + public void SerializeProps_KnownWs_StillWritten() + { + string xml = TsPropsSerializer.SerializePropsToXml(MakeProps(FwTextPropType.ktptWs, EnWS), WritingSystemManager); + Assert.That(xml, Does.Contain("ws=\"en\"")); + } + + [Test] + public void SerializeString_RunWithUnknownBaseWs_RoundTrips() + { + ITsStrBldr bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "text", TsStringUtils.MakeProps("Emphasis", EnWS)); + bldr.SetIntPropValues(0, 4, (int)FwTextPropType.ktptBaseWs, 0, UnknownWs); + + string xml = TsStringSerializer.SerializeTsStringToXml(bldr.GetString(), WritingSystemManager); + + Assert.That(xml, Does.Not.Contain("wsBase")); + ITsString result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); + Assert.That(result.Text, Is.EqualTo("text")); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(EnWS)); + } + + [Test] + public void SerializeString_RunWithUnknownWs_FailsInsteadOfWritingBadXml() + { + // A run of text must have a ws, so there is nothing safe to write for it. + ITsString tss = TsStringUtils.MakeString("text", UnknownWs); + Assert.That(() => TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager), + Throws.TypeOf()); + } + + [Test] + public void SerializeString_AStrWithUnknownWs_Fails() + { + ITsString tss = TsStringUtils.MakeString("text", EnWS); + Assert.That(() => TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager, UnknownWs), + Throws.TypeOf()); + } + + [TestCase("")] + [TestCase("")] + public void DeserializeProps_EmptyWs_IsDropped(string xml) + { + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); + int var; + Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptWs, out var), Is.EqualTo(-1)); + Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptBaseWs, out var), Is.EqualTo(-1)); + } + + [Test] + public void DeserializeProps_EmptyWsInWsProp_IsDropped() + { + string xml = "" + + ""; + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + string wsStyles = result.GetStrPropValue((int)FwTextPropType.ktptWsStyle); + Assert.That(wsStyles, Does.Contain("Times")); + Assert.That(wsStyles, Does.Not.Contain("Arial")); + } + + [TestCase("text")] + [TestCase("text")] + [TestCase("text")] + [TestCase("text")] + [TestCase("text")] + public void DeserializeString_EmptyWs_IsDropped(string xml) + { + ITsString result = null; + Assert.That(() => result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), + Throws.Nothing); + Assert.That(result.Text, Is.EqualTo("text")); + } + } +} From 70e8e6b685ec04dead3076ea9b7eec414f83becb Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 5 Nov 2026 17:58:16 -0500 Subject: [PATCH 2/4] Give runs with an unknown or empty ws a fallback ws instead of throwing (#411) Writer: a text run (or empty/ORC/link run, judged on the text actually written) with no usable ws is written with a fallback ws: the enclosing AStr ws, else the previous run's, else the user ws; a trace warning is logged. This replaces the throw, which would have failed every save. Reader: a run with an existing empty ws gets the same fallback so it can be saved again, except under an AStr with no ws, where inventing one could overwrite a real alternative. MultiAccessor skips alternatives keyed by an unknown ws, and custom String fields no longer use an unknown first-run ws as the AStr ws. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 +- .../Text/TsPropsSerializer.cs | 2 +- .../Text/TsStringSerializer.cs | 117 ++++++++-- src/SIL.LCModel/DomainImpl/CmObject.cs | 4 + src/SIL.LCModel/DomainImpl/Strings.cs | 8 + .../Text/TsUnknownWsHandleTests.cs | 203 +++++++++++++++--- .../DomainImpl/StringsTests.cs | 65 ++++++ 7 files changed, 347 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7b3614d..55dda9d2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed -- [SIL.LCModel.Core] `TsPropsSerializer` no longer writes `ws=""`/`wsBase=""` for a writing-system handle the factory does not know (the attribute is omitted instead), and `TsStringSerializer` throws instead of writing a run or `` with an unknown ws. Readers now ignore an existing empty `ws`/`wsBase`, so records saved with `ws=""` load (#411) +- [SIL.LCModel.Core] A writing-system handle the factory does not know is no longer saved as `ws=""`/`wsBase=""`, which no reader could load. An unknown `ws`/`wsBase` on text properties is omitted; a text run with no usable ws is written with a fallback ws (the enclosing `` ws, else the previous run's, else the user ws) and a trace warning. Readers give a run with an existing empty `ws` the same fallback, so such records load and can be saved again. `MultiAccessor` skips alternatives keyed by an unknown ws (#411) - [SIL.LCModel] BaseStyleInfo now loads default font features (ktptFontVariations) from style rules, fixing font features set as style defaults being dropped on reload (LT-22351) - [SIL.LCModel] BaseStyleInfo no longer clobbers ws-specific font overrides inherited from a based-on style with the default font info's merely-inherited values; only explicitly-set default properties are propagated to ws overrides (LT-22351) - [SIL.LCModel] BulletInfo now decodes all properties of an encoded bullet font info instead of stopping at the first string property, fixing bullet font features (ktptFontVariations) being lost when a bullet font name was also set (LT-22351) diff --git a/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs b/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs index 63ac2846a..6503a7056 100644 --- a/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs @@ -99,7 +99,7 @@ internal static void WriteIntProperty(XmlWriter writer, ILgWritingSystemFactory // An unknown handle has no id. Writing it as ws="" would produce XML that no reader accepts // (the property is optional, so it is safer to leave it out). if (string.IsNullOrEmpty(id)) - Debug.WriteLine("TsPropsSerializer: omitting '" + attrName + "' attribute for unknown writing system handle " + value); + Trace.TraceWarning("TsPropsSerializer: omitting '" + attrName + "' attribute for unknown writing system handle " + value); else writer.WriteAttributeString(attrName, Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); } diff --git a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs index 5ea84b1ed..2c29aeb2d 100644 --- a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs @@ -9,6 +9,7 @@ // using System; +using System.Diagnostics; using System.Linq; using System.Text; using System.Text.RegularExpressions; @@ -63,6 +64,7 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto // Write the properties and text for each run string fieldName = null; + int previousWs = 0; for (int i = 0; i < normalizedTss.RunCount; i++) { TsRunInfo tri; @@ -91,18 +93,6 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto fieldName = runFieldName; } - // A run of ordinary text must carry a ws (the reader rejects it otherwise), and an unknown handle - // has no id to write. Fail here rather than save a record that cannot be loaded. - int runWs; - FwTextPropVar wsVar; - if (textProps.TryGetIntValue(FwTextPropType.ktptWs, out wsVar, out runWs) && runWs > 0 - && string.IsNullOrEmpty(lgwsf.GetStrFromWs(runWs))) - { - string wsCheckText = normalizedTss.get_RunText(i) ?? string.Empty; - if (wsCheckText.Length > 0 && wsCheckText[0] > 13) - throw new InvalidOperationException("Cannot serialize run with unknown writing system handle " + runWs + ". Run text: " + wsCheckText); - } - bool markItem; FwTextPropVar var; int markItemValue; @@ -166,6 +156,10 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto } } + // Work out the text that will actually be written, so that we can tell whether the reader + // will require a ws on this run. + string writtenText; + bool preserveSpace = false; if (pict != null) { // Write the bytes of the picture data @@ -176,21 +170,47 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto if (j % 32 == 31) sb.AppendLine(); } - writer.WriteString(sb.ToString()); + writtenText = sb.ToString(); } else if (hotGuid) { - writer.WriteString(string.Empty); + writtenText = string.Empty; } else { string runText = normalizedTss.get_RunText(i) ?? string.Empty; - if (runText != string.Empty && runText.All(char.IsWhiteSpace)) - writer.WriteAttributeString("xml", "space", "", "preserve"); + preserveSpace = runText != string.Empty && runText.All(char.IsWhiteSpace); // TODO: should we escape quotation marks? this is not necessary but different than the behavior of the C++ implementation - writer.WriteString(StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC))); + writtenText = StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC)); } + // A run that needs a ws but has none we can write (an unknown handle, or no ws at all) gets a + // fallback ws. Otherwise we would write a record that no reader can load, and throwing here + // would stop the whole project from being saved. + int runWs; + FwTextPropVar wsVar; + bool hasKnownWs = textProps.TryGetIntValue(FwTextPropType.ktptWs, out wsVar, out runWs) && runWs > 0 + && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(runWs)); + if (hasKnownWs) + { + previousWs = runWs; + } + else if (RunNeedsWs(writtenText)) + { + int fallbackWs = GetFallbackWs(lgwsf, ws, previousWs); + if (fallbackWs > 0) + { + Trace.TraceWarning("TsStringSerializer: run has no usable writing system (handle " + runWs + + "); writing it with fallback ws '" + lgwsf.GetStrFromWs(fallbackWs) + "'."); + writer.WriteAttributeString("ws", Normalizer.Normalize(lgwsf.GetStrFromWs(fallbackWs), Normalizer.UNormalizationMode.UNORM_NFC)); + previousWs = fallbackWs; + } + } + + if (preserveSpace) + writer.WriteAttributeString("xml", "space", "", "preserve"); + writer.WriteString(writtenText); + writer.WriteEndElement(); if (markItem) writer.WriteEndElement(); @@ -291,18 +311,54 @@ private static ITsString HandleComplexString(XElement xml, ILgWritingSystemFacto var strBldr = TsStringUtils.MakeIncStrBldr(); + // The ws of the enclosing , if it has one: the best fallback for a run whose ws is empty. + int enclosingWs = 0; + bool enclosingWsMissing = false; + if (xml.Name.LocalName == "AStr") + { + XAttribute astrWs = xml.Attribute("ws"); + if (astrWs != null && astrWs.Value.Length > 0) + enclosingWs = GetWsForId(astrWs.Value, lgwsf); + else + enclosingWsMissing = true; + } + int previousWs = 0; + foreach (XElement runElement in runs) { if (runElement == null) throw new XmlSchemaException("TsString XML must contain a element contained in a <" + xml.Name.LocalName + "> element"); string runText = runElement.Value; - if (runElement.Attribute("ws") == null && (runText.Length == 0 || runText[0] > 13)) + XAttribute runWsAttr = runElement.Attribute("ws"); + if (runWsAttr == null && RunNeedsWs(runText)) throw new XmlSchemaException("Run element must contain a ws attribute. Run text: " + runElement.Value); + bool needsFallbackWs = runWsAttr != null && runWsAttr.Value.Length == 0 && RunNeedsWs(runText); // Make sure the text is in the decomposed form (FWR-148) runText = Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFD); bool isOrcNeeded = TsPropsSerializer.GetPropAttributesForElement(runElement, lgwsf, strBldr); + if (runWsAttr != null && runWsAttr.Value.Length > 0) + { + previousWs = GetWsForId(runWsAttr.Value, lgwsf); + } + else if (needsFallbackWs) + { + // An empty ws was written by older versions for an unknown handle. Give the run a known ws so that + // it can be saved again. If the enclosing has no ws either, the run's ws would become the key + // of the alternative (see MultiAccessor.ReadAstrElementOfMultiString), so do not invent one then: + // that could overwrite a real alternative. + int fallbackWs = enclosingWs > 0 ? enclosingWs : previousWs > 0 ? previousWs : + (enclosingWsMissing ? 0 : GetFallbackWs(lgwsf, 0, 0)); + Trace.TraceWarning("TsStringSerializer: run has an empty ws; " + (fallbackWs > 0 + ? "using fallback ws '" + lgwsf.GetStrFromWs(fallbackWs) + "'." : "leaving it without a ws.")); + if (fallbackWs > 0) + { + strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, fallbackWs); + previousWs = fallbackWs; + } + } + // Add an ORC character, if needed, for the run if (runText.Length == 0 && isOrcNeeded) runText = StringUtils.kszObject; @@ -347,6 +403,31 @@ private static ITsString HandleSimpleString(XElement rootXml, ILgWritingSystemFa #region Internal methods /// ------------------------------------------------------------------------------------ + /// + /// Whether the reader requires a ws attribute on a run whose text, as written to XML, is + /// . A run that is empty (e.g. an ORC or link run) or starts with an ordinary + /// character needs one; a run starting with a control character up to U+000D (tab, newline...) does not. + /// + internal static bool RunNeedsWs(string writtenText) + { + return writtenText.Length == 0 || writtenText[0] > 13; + } + + /// + /// Chooses a known ws for a run that has none: the enclosing ws, else the previous run's ws, else the user ws. + /// Returns 0 if none is known. + /// + internal static int GetFallbackWs(ILgWritingSystemFactory lgwsf, int enclosingWs, int previousWs) + { + foreach (int candidate in new[] { enclosingWs, previousWs }) + { + if (candidate > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(candidate))) + return candidate; + } + int userWs = lgwsf.UserWs; + return userWs > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(userWs)) ? userWs : 0; + } + /// /// Gets the ws handle for specified writing system identifier. /// diff --git a/src/SIL.LCModel/DomainImpl/CmObject.cs b/src/SIL.LCModel/DomainImpl/CmObject.cs index 2a623fa90..045196ba6 100644 --- a/src/SIL.LCModel/DomainImpl/CmObject.cs +++ b/src/SIL.LCModel/DomainImpl/CmObject.cs @@ -3114,6 +3114,10 @@ private void UserDefinedProperties(XmlWriter writer) var tsString = (ITsString)data; int temp; var wsHvo = tsString.get_Properties(0).GetIntPropValues((int)FwTextPropType.ktptWs, out temp); + // A first run in an unknown ws cannot be the ws of an ; write a and let each run + // get a fallback ws instead. + if (wsHvo > 0 && string.IsNullOrEmpty(wsf.GetStrFromWs(wsHvo))) + wsHvo = 0; writer.WriteRaw(TsStringUtils.GetXmlRep(tsString, wsf, wsHvo)); break; case CellarPropertyType.Binary: diff --git a/src/SIL.LCModel/DomainImpl/Strings.cs b/src/SIL.LCModel/DomainImpl/Strings.cs index c554f5e9e..31d0c53ee 100644 --- a/src/SIL.LCModel/DomainImpl/Strings.cs +++ b/src/SIL.LCModel/DomainImpl/Strings.cs @@ -368,6 +368,14 @@ protected virtual void ToXml(XmlWriter writer, ILgWritingSystemFactory wsf, int if (string.IsNullOrEmpty(alternative.Text)) return; // Skip writing TsStrings with no content. + // An alternative whose ws the factory does not know has no id to write, and giving it another ws could + // overwrite a real alternative. Skip it rather than fail the whole save. + if (string.IsNullOrEmpty(wsf.GetStrFromWs(ws))) + { + Trace.TraceWarning("MultiAccessor: skipping alternative with unknown writing system handle " + ws); + return; + } + writer.WriteRaw(TsStringUtils.GetXmlRep(alternative, wsf, ws)); } diff --git a/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs index 9bbdf75e1..ee171b81b 100644 --- a/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs +++ b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs @@ -9,7 +9,8 @@ namespace SIL.LCModel.Core.Text { /// - /// An unknown writing system handle used to be written as ws="", which no reader accepts. + /// An unknown writing system handle used to be written as ws="", which no reader accepts. A run of text + /// cannot be written without a ws either, so such runs are written (and loaded) with a fallback ws. /// [TestFixture] public class TsUnknownWsHandleTests : TsSerializerTestsBase @@ -24,13 +25,48 @@ private ITsTextProps MakeProps(FwTextPropType wsType, int ws) return bldr.GetTextProps(); } + private static int WsOfRun(ITsString tss, int irun) + { + int var; + return tss.get_Properties(irun).GetIntPropValues((int)FwTextPropType.ktptWs, out var); + } + + /// Asserts that every run has a ws the factory knows. + private void AssertAllRunsHaveKnownWs(ITsString tss) + { + for (int i = 0; i < tss.RunCount; i++) + { + int ws = WsOfRun(tss, i); + Assert.That(ws, Is.GreaterThan(0), "run " + i); + Assert.That(WritingSystemManager.GetStrFromWs(ws), Is.Not.Null.And.Not.Empty, "run " + i); + } + } + + /// + /// Serializes, reads, re-saves and reads again, checking that no step throws, that no empty ws is written + /// and that every run of the final string has a known ws. + /// + private ITsString SaveAndReloadTwice(ITsString tss, int astrWs = 0) + { + string xml1 = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager, astrWs); + Assert.That(xml1, Does.Not.Contain("ws=\"\"")); + ITsString read1 = TsStringSerializer.DeserializeTsStringFromXml(xml1, WritingSystemManager); + string xml2 = TsStringSerializer.SerializeTsStringToXml(read1, WritingSystemManager, astrWs); + Assert.That(xml2, Does.Not.Contain("ws=\"\"")); + ITsString read2 = TsStringSerializer.DeserializeTsStringFromXml(xml2, WritingSystemManager); + Assert.That(read2.Text ?? string.Empty, Is.EqualTo(tss.Text ?? string.Empty).Or.EqualTo( + (tss.Text ?? string.Empty).Replace("\u0001", string.Empty))); + AssertAllRunsHaveKnownWs(read2); + return read2; + } + + #region Props [TestCase(FwTextPropType.ktptWs, "ws")] [TestCase(FwTextPropType.ktptBaseWs, "wsBase")] public void SerializeProps_UnknownWs_OmitsAttributeAndRoundTrips(FwTextPropType wsType, string attrName) { string xml = TsPropsSerializer.SerializePropsToXml(MakeProps(wsType, UnknownWs), WritingSystemManager); - Assert.That(xml, Does.Not.Contain(attrName + "=\"\"")); Assert.That(xml, Does.Not.Contain(attrName + "=")); ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); @@ -45,6 +81,31 @@ public void SerializeProps_KnownWs_StillWritten() Assert.That(xml, Does.Contain("ws=\"en\"")); } + [TestCase("")] + [TestCase("")] + [TestCase("")] + public void DeserializeProps_EmptyWs_IsDropped(string xml) + { + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); + int var; + Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptWs, out var), Is.EqualTo(-1)); + Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptBaseWs, out var), Is.EqualTo(-1)); + } + + [Test] + public void DeserializeProps_EmptyWsInWsProp_IsDropped() + { + string xml = "" + + ""; + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + string wsStyles = result.GetStrPropValue((int)FwTextPropType.ktptWsStyle); + Assert.That(wsStyles, Does.Contain("Times")); + Assert.That(wsStyles, Does.Not.Contain("Arial")); + } + #endregion + + #region Runs: writing [Test] public void SerializeString_RunWithUnknownBaseWs_RoundTrips() { @@ -55,61 +116,135 @@ public void SerializeString_RunWithUnknownBaseWs_RoundTrips() string xml = TsStringSerializer.SerializeTsStringToXml(bldr.GetString(), WritingSystemManager); Assert.That(xml, Does.Not.Contain("wsBase")); - ITsString result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); - Assert.That(result.Text, Is.EqualTo("text")); + ITsString result = SaveAndReloadTwice(bldr.GetString()); Assert.That(result.get_WritingSystem(0), Is.EqualTo(EnWS)); } [Test] - public void SerializeString_RunWithUnknownWs_FailsInsteadOfWritingBadXml() + public void SerializeString_RunWithUnknownWs_UsesUserWs() + { + ITsString result = SaveAndReloadTwice(TsStringUtils.MakeString("text", UnknownWs)); + Assert.That(result.Text, Is.EqualTo("text")); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(WritingSystemManager.UserWs)); + } + + [Test] + public void SerializeString_RunWithUnknownWs_UsesPreviousRunWs() + { + ITsStrBldr bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "one", TsStringUtils.MakeProps(null, EsWS)); + bldr.Replace(3, 3, "two", TsStringUtils.MakeProps(null, UnknownWs)); + + ITsString result = SaveAndReloadTwice(bldr.GetString()); + + Assert.That(WsOfRun(result, result.RunCount - 1), Is.EqualTo(EsWS)); + Assert.That(result.Text, Is.EqualTo("onetwo")); + } + + [Test] + public void SerializeString_AStrRunWithUnknownWs_UsesAStrWs() + { + ITsString result = SaveAndReloadTwice(TsStringUtils.MakeString("text", UnknownWs), EsWS); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(EsWS)); + } + + [Test] + public void SerializeString_EmptyRunWithUnknownWs_Loads() + { + // The reader requires a ws on an empty run too. + string xml = TsStringSerializer.SerializeTsStringToXml(TsStringUtils.MakeString(string.Empty, UnknownWs), WritingSystemManager); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + Assert.That(() => TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), Throws.Nothing); + } + + [Test] + public void SerializeString_RunStartingWithStrippedControlCharAndUnknownWs_Loads() + { + // U+0001 is dropped when written, so the run text the reader sees starts with 'a' and needs a ws. + ITsString result = SaveAndReloadTwice(TsStringUtils.MakeString("\u0001abc", UnknownWs)); + Assert.That(result.Text, Is.EqualTo("abc")); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(WritingSystemManager.UserWs)); + } + + [Test] + public void SerializeString_RunStartingWithTabAndUnknownWs_NeedsNoWs() + { + string xml = TsStringSerializer.SerializeTsStringToXml(TsStringUtils.MakeString("\tabc", UnknownWs), WritingSystemManager); + Assert.That(xml, Does.Not.Contain("ws=")); + Assert.That(TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager).Text, Is.EqualTo("\tabc")); + } + + [TestCase(FwObjDataTypes.kodtOwnNameGuidHot)] + [TestCase(FwObjDataTypes.kodtNameGuidHot)] + public void SerializeString_OrcRunWithUnknownWs_Loads(FwObjDataTypes type) + { + ITsString orc = TsStringUtils.CreateOrcFromGuid(Guid.NewGuid(), type, UnknownWs); + ITsString result = null; + string xml = TsStringSerializer.SerializeTsStringToXml(orc, WritingSystemManager); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + Assert.That(() => result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), Throws.Nothing); + AssertAllRunsHaveKnownWs(result); + } + + [Test] + public void SerializeString_KnownWs_IsUnchanged() { - // A run of text must have a ws, so there is nothing safe to write for it. - ITsString tss = TsStringUtils.MakeString("text", UnknownWs); - Assert.That(() => TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager), - Throws.TypeOf()); + string xml = TsStringSerializer.SerializeTsStringToXml(TsStringUtils.MakeString("text", EsWS), WritingSystemManager); + Assert.That(xml, Does.Contain("text")); } [Test] public void SerializeString_AStrWithUnknownWs_Fails() { + // There is no ws we could safely use as the key of the alternative. LCM skips such alternatives + // before they get here. ITsString tss = TsStringUtils.MakeString("text", EnWS); Assert.That(() => TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager, UnknownWs), Throws.TypeOf()); } + #endregion - [TestCase("")] - [TestCase("")] - public void DeserializeProps_EmptyWs_IsDropped(string xml) + #region Runs: reading legacy ws="" + [TestCase("text", "en")] + [TestCase("text", "en")] + [TestCase("text", "es")] + [TestCase("text", "es")] + [TestCase("text", "es")] + [TestCase("text", "en")] + public void DeserializeString_EmptyWs_GetsFallbackWsAndResaves(string xml, string expectedWsOfLastRun) { - ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); - Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); - int var; - Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptWs, out var), Is.EqualTo(-1)); - Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptBaseWs, out var), Is.EqualTo(-1)); + ITsString read1 = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); + + Assert.That(read1.Text, Is.EqualTo("text")); + AssertAllRunsHaveKnownWs(read1); + Assert.That(WsOfRun(read1, read1.RunCount - 1), Is.EqualTo(WritingSystemManager.GetWsFromStr(expectedWsOfLastRun))); + // and the result can be saved and loaded again + bool isAStr = xml.StartsWith("" - + ""; - ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); - string wsStyles = result.GetStrPropValue((int)FwTextPropType.ktptWsStyle); - Assert.That(wsStyles, Does.Contain("Times")); - Assert.That(wsStyles, Does.Not.Contain("Arial")); + ITsString read1 = TsStringSerializer.DeserializeTsStringFromXml("", WritingSystemManager); + string xml2 = TsStringSerializer.SerializeTsStringToXml(read1, WritingSystemManager); + Assert.That(xml2, Does.Not.Contain("ws=\"\"")); + Assert.That(() => TsStringSerializer.DeserializeTsStringFromXml(xml2, WritingSystemManager), Throws.Nothing); } - [TestCase("text")] - [TestCase("text")] - [TestCase("text")] - [TestCase("text")] - [TestCase("text")] - public void DeserializeString_EmptyWs_IsDropped(string xml) + [Test] + public void DeserializeString_EmptyWsInAStrWithoutWs_IsNotGivenAWs() { - ITsString result = null; - Assert.That(() => result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), - Throws.Nothing); - Assert.That(result.Text, Is.EqualTo("text")); + // The run's ws is the key of the alternative when has no ws, so inventing one could replace a real + // alternative. Such a string stays without a ws, as before. + ITsString read = TsStringSerializer.DeserializeTsStringFromXml("text", WritingSystemManager); + Assert.That(read.get_WritingSystem(0), Is.LessThanOrEqualTo(0)); } + #endregion } } diff --git a/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs b/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs index ee6360b85..670706d0c 100644 --- a/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs +++ b/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Xml.Linq; using NUnit.Framework; using SIL.LCModel.Core.KernelInterfaces; using SIL.LCModel.Core.Text; @@ -305,6 +306,70 @@ public void ToXml_WithControlCharacterWorks() Assert.AreEqual("test", xml, "XML does not contain expected string element."); } + private const int UnknownWs = 987654; + + /// + /// An alternative keyed by a writing system handle the factory does not know cannot be written. It is skipped + /// (rather than failing the whole save), and the real alternatives are still written. + /// + [Test] + public void ToXml_AlternativeWithUnknownWs_IsSkipped() + { + var english = Cache.LangProject.CurrentAnalysisWritingSystems.First(); + var region = Cache.LangProject.Description; + region.set_String(english.Handle, TsStringUtils.MakeString("Real", english.Handle)); + ((MultiAccessor)region).Add(UnknownWs, TsStringUtils.MakeString("Stale", UnknownWs)); + + string xml = null; + Assert.That(() => xml = ToXml(region), Throws.Nothing); + + Assert.That(xml, Does.Contain("Real")); + Assert.That(xml, Does.Not.Contain("Stale")); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + } + + /// + /// A run with an empty ws, written by older versions, is loaded with the ws of its alternative, and what is + /// saved from it can be loaded again. + /// + [Test] + public void FromXml_AStrWithEmptyRunWs_UsesAStrWsAndResaves() + { + var english = Cache.LangProject.CurrentAnalysisWritingSystems.First(); + var region = Cache.LangProject.Description; + var wsf = Cache.WritingSystemFactory; + + ((MultiAccessor)region).Clear(); + ((MultiAccessor)region).LoadFromDataStoreInternal( + XElement.Parse("legacy"), wsf); + + Assert.That(region.get_String(english.Handle).Text, Is.EqualTo("legacy")); + Assert.That(region.get_String(english.Handle).get_WritingSystem(0), Is.EqualTo(english.Handle)); + string xml = ToXml(region); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + ITsString tss; + Assert.That(MultiAccessor.ReadAstrElementOfMultiString(XElement.Parse("" + xml + "").Elements("AStr").First(a => a.Attribute("ws").Value == "en"), wsf, out tss), Is.EqualTo(english.Handle)); + Assert.That(tss.Text, Is.EqualTo("legacy")); + } + + /// + /// An alternative with no ws on either the AStr or its run cannot be placed. It must not be given the user + /// ws, which would overwrite the real alternative in that ws. + /// + [Test] + public void FromXml_AStrAndRunWithEmptyWs_DoesNotOverwriteRealAlternative() + { + var english = Cache.WritingSystemFactory.GetWsFromStr("en"); + var region = Cache.LangProject.Description; + var wsf = Cache.WritingSystemFactory; + Cache.ServiceLocator.WritingSystemManager.UserWs = english; + region.set_String(english, TsStringUtils.MakeString("Real", english)); + ((MultiAccessor)region).LoadFromDataStoreInternal( + XElement.Parse("lost"), wsf); + + Assert.That(region.get_String(english).Text, Is.EqualTo("Real")); + } + private string ToXml(ITsMultiString multiString) { using var ms = new MemoryStream(); From 4922861a7e40617c2eaae98955af024db6bc22d5 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 5 Nov 2026 18:02:21 -0500 Subject: [PATCH 3/4] Skip MultiUnicode alternatives with an unknown ws when saving (#411) MultiUnicodeAccessor.ToXml threw for an unknown ws, failing the whole save. Skip and trace it like MultiAccessor.ToXml does. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 +- src/SIL.LCModel/DomainImpl/Strings.cs | 7 +++++++ .../DomainImpl/StringsTests.cs | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55dda9d2c..3bd74d13b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed -- [SIL.LCModel.Core] A writing-system handle the factory does not know is no longer saved as `ws=""`/`wsBase=""`, which no reader could load. An unknown `ws`/`wsBase` on text properties is omitted; a text run with no usable ws is written with a fallback ws (the enclosing `` ws, else the previous run's, else the user ws) and a trace warning. Readers give a run with an existing empty `ws` the same fallback, so such records load and can be saved again. `MultiAccessor` skips alternatives keyed by an unknown ws (#411) +- [SIL.LCModel.Core] A writing-system handle the factory does not know is no longer saved as `ws=""`/`wsBase=""`, which no reader could load. An unknown `ws`/`wsBase` on text properties is omitted; a text run with no usable ws is written with a fallback ws (the enclosing `` ws, else the previous run's, else the user ws) and a trace warning. Readers give a run with an existing empty `ws` the same fallback, so such records load and can be saved again. `MultiAccessor` and `MultiUnicodeAccessor` skip alternatives keyed by an unknown ws (#411) - [SIL.LCModel] BaseStyleInfo now loads default font features (ktptFontVariations) from style rules, fixing font features set as style defaults being dropped on reload (LT-22351) - [SIL.LCModel] BaseStyleInfo no longer clobbers ws-specific font overrides inherited from a based-on style with the default font info's merely-inherited values; only explicitly-set default properties are propagated to ws overrides (LT-22351) - [SIL.LCModel] BulletInfo now decodes all properties of an encoded bullet font info instead of stopping at the first string property, fixing bullet font features (ktptFontVariations) being lost when a bullet font name was also set (LT-22351) diff --git a/src/SIL.LCModel/DomainImpl/Strings.cs b/src/SIL.LCModel/DomainImpl/Strings.cs index 31d0c53ee..05b597d58 100644 --- a/src/SIL.LCModel/DomainImpl/Strings.cs +++ b/src/SIL.LCModel/DomainImpl/Strings.cs @@ -906,6 +906,13 @@ protected override void ToXml(XmlWriter writer, ILgWritingSystemFactory wsf, int if (string.IsNullOrEmpty(text)) return; // Skip writing TsStrings with no content. + // An alternative whose ws the factory does not know has no id to write. Skip it rather than fail the whole save. + if (string.IsNullOrEmpty(wsf.GetStrFromWs(ws))) + { + Trace.TraceWarning("MultiUnicodeAccessor: skipping alternative with unknown writing system handle " + ws); + return; + } + writer.WriteStartElement("AUni"); writer.WriteAttributeString("ws", m_object.Services.WritingSystemManager.Get(ws).Id); text = TsStringSerializer.StripInvalidXmlChars(Normalizer.Normalize(text, Normalizer.UNormalizationMode.UNORM_NFC)); diff --git a/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs b/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs index 670706d0c..aafbd4b95 100644 --- a/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs +++ b/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs @@ -328,6 +328,24 @@ public void ToXml_AlternativeWithUnknownWs_IsSkipped() Assert.That(xml, Does.Not.Contain("ws=\"\"")); } + /// + /// The same for a MultiUnicode (<AUni>) alternative. + /// + [Test] + public void ToXml_UnicodeAlternativeWithUnknownWs_IsSkipped() + { + var english = Cache.LangProject.CurrentAnalysisWritingSystems.First(); + var country = Cache.LangProject.MainCountry; + country.set_String(english.Handle, TsStringUtils.MakeString("Real", english.Handle)); + ((MultiAccessor)country).Add(UnknownWs, TsStringUtils.MakeString("Stale", UnknownWs)); + + string xml = null; + Assert.That(() => xml = ToXml(country), Throws.Nothing); + + Assert.That(xml, Does.Contain("Real")); + Assert.That(xml, Does.Not.Contain("Stale")); + } + /// /// A run with an empty ws, written by older versions, is loaded with the ws of its alternative, and what is /// saved from it can be loaded again. From 2f0412d33f148419d1eb9091a2f2840079dc55bd Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 5 Nov 2026 18:31:13 -0500 Subject: [PATCH 4/4] Resolve the enclosing AStr ws lazily when reading runs (#411) Only a run with an empty ws needs the AStr ws as a fallback. Resolving it for every AStr could fail, or create a writing system, for records that load fine without it. Also document why the user-ws fallback may create "en". Co-Authored-By: Claude Sonnet 5.5 --- .../Text/TsStringSerializer.cs | 22 ++++++++++++++++--- .../Text/TsUnknownWsHandleTests.cs | 13 +++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs index 2c29aeb2d..efebc88c4 100644 --- a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs @@ -311,14 +311,16 @@ private static ITsString HandleComplexString(XElement xml, ILgWritingSystemFacto var strBldr = TsStringUtils.MakeIncStrBldr(); - // The ws of the enclosing , if it has one: the best fallback for a run whose ws is empty. - int enclosingWs = 0; + // The ws of the enclosing , if it has one, is the best fallback for a run whose ws is empty. It is + // resolved lazily, only when a fallback is needed: resolving it can create the ws or fail, and records + // whose runs all have a ws must load exactly as they always did. + string enclosingWsId = null; bool enclosingWsMissing = false; if (xml.Name.LocalName == "AStr") { XAttribute astrWs = xml.Attribute("ws"); if (astrWs != null && astrWs.Value.Length > 0) - enclosingWs = GetWsForId(astrWs.Value, lgwsf); + enclosingWsId = astrWs.Value; else enclosingWsMissing = true; } @@ -348,6 +350,18 @@ private static ITsString HandleComplexString(XElement xml, ILgWritingSystemFacto // it can be saved again. If the enclosing has no ws either, the run's ws would become the key // of the alternative (see MultiAccessor.ReadAstrElementOfMultiString), so do not invent one then: // that could overwrite a real alternative. + int enclosingWs = 0; + if (enclosingWsId != null) + { + try + { + enclosingWs = GetWsForId(enclosingWsId, lgwsf); + } + catch (XmlSchemaException) + { + // not usable as a fallback; try the next choice + } + } int fallbackWs = enclosingWs > 0 ? enclosingWs : previousWs > 0 ? previousWs : (enclosingWsMissing ? 0 : GetFallbackWs(lgwsf, 0, 0)); Trace.TraceWarning("TsStringSerializer: run has an empty ws; " + (fallbackWs > 0 @@ -424,6 +438,8 @@ internal static int GetFallbackWs(ILgWritingSystemFactory lgwsf, int enclosingWs if (candidate > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(candidate))) return candidate; } + // The user ws is the established default. In FieldWorks it is always set. A factory with no user ws + // creates one ("en") here; that is preferable to failing the save, which would block the whole project. int userWs = lgwsf.UserWs; return userWs > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(userWs)) ? userWs : 0; } diff --git a/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs index ee171b81b..c97167aba 100644 --- a/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs +++ b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs @@ -228,6 +228,19 @@ public void DeserializeString_EmptyWs_GetsFallbackWsAndResaves(string xml, strin AssertAllRunsHaveKnownWs(read2); } + [TestCase("123")] + [TestCase("en_US")] + [TestCase("qaa-x-")] + public void DeserializeString_AStrWsThatCannotBeCreated_StillLoadsWhenNoFallbackIsNeeded(string astrWs) + { + // The ws of the is only needed as a fallback for a run with an empty ws. Master ignores it when + // the runs have their own, so such records must keep loading. + string xml = "text"; + ITsString result = null; + Assert.That(() => result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), Throws.Nothing); + Assert.That(result.Text, Is.EqualTo("text")); + } + [Test] public void DeserializeString_EmptyRunWithEmptyWs_ResavesAndLoads() {