diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a172e5d..3bd74d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +- [SIL.LCModel.Core] A writing-system handle the factory does not know is no longer saved as `ws=""`/`wsBase=""`, which no reader could load. An unknown `ws`/`wsBase` on text properties is omitted; a text run with no usable ws is written with a fallback ws (the enclosing `` ws, else the previous run's, else the user ws) and a trace warning. Readers give a run with an existing empty `ws` the same fallback, so such records load and can be saved again. `MultiAccessor` and `MultiUnicodeAccessor` skip alternatives keyed by an unknown ws (#411) - [SIL.LCModel] BaseStyleInfo now loads default font features (ktptFontVariations) from style rules, fixing font features set as style defaults being dropped on reload (LT-22351) - [SIL.LCModel] BaseStyleInfo no longer clobbers ws-specific font overrides inherited from a based-on style with the default font info's merely-inherited values; only explicitly-set default properties are propagated to ws overrides (LT-22351) - [SIL.LCModel] BulletInfo now decodes all properties of an encoded bullet font info instead of stopping at the first string property, fixing bullet font features (ktptFontVariations) being lost when a bullet font name was also set (LT-22351) diff --git a/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs b/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs index 5b07b0e8..6503a705 100644 --- a/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsPropsSerializer.cs @@ -95,7 +95,13 @@ internal static void WriteIntProperty(XmlWriter writer, ILgWritingSystemFactory if (value > 0) { string id = lgwsf.GetStrFromWs(value); - writer.WriteAttributeString(tpt == FwTextPropType.ktptWs ? "ws" : "wsBase", Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); + string attrName = tpt == FwTextPropType.ktptWs ? "ws" : "wsBase"; + // An unknown handle has no id. Writing it as ws="" would produce XML that no reader accepts + // (the property is optional, so it is safer to leave it out). + if (string.IsNullOrEmpty(id)) + Trace.TraceWarning("TsPropsSerializer: omitting '" + attrName + "' attribute for unknown writing system handle " + value); + else + writer.WriteAttributeString(attrName, Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); } break; case FwTextPropType.ktptFontSize: @@ -863,6 +869,8 @@ private static void AddWsStyleInfoToBldr(ITsPropsBldr bldr, XElement wsStyleInfo if (wsAttrib == null) throw new XmlSchemaException("WsProp must contain a 'ws' attribute"); wsAttrib.Remove(); // Make sure we don't count it twice + if (wsAttrib.Value.Length == 0) + continue; // tolerate an empty ws written by older versions for an unknown handle WriteIntToStrBuilder(wsStyleProps, TsStringSerializer.GetWsForId(wsAttrib.Value, lgwsf)); // Get the font family for the prop override @@ -1138,12 +1146,15 @@ internal static ITsPropsBldr GetPropAttributesForElement(XElement xml, ILgWritin GetIntValueFromStr(attr.Value, false)); break; case "ws": - propsBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + // Older versions wrote ws="" for an unknown handle; drop it rather than failing to load. + if (attr.Value.Length > 0) + propsBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsBase": - propsBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + if (attr.Value.Length > 0) + propsBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsStyle": Debug.Fail("We don't support the old wsStyle attribute!"); @@ -1395,12 +1406,15 @@ internal static bool GetPropAttributesForElement(XElement xml, ILgWritingSystemF GetIntValueFromStr(attr.Value, false)); break; case "ws": - strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + // Older versions wrote ws="" for an unknown handle; drop it rather than failing to load. + if (attr.Value.Length > 0) + strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsBase": - strBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, - TsStringSerializer.GetWsForId(attr.Value, lgwsf)); + if (attr.Value.Length > 0) + strBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0, + TsStringSerializer.GetWsForId(attr.Value, lgwsf)); break; case "wsStyle": Debug.Fail("We don't support the old wsStyle attribute!"); diff --git a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs index 173b6a7a..efebc88c 100644 --- a/src/SIL.LCModel.Core/Text/TsStringSerializer.cs +++ b/src/SIL.LCModel.Core/Text/TsStringSerializer.cs @@ -9,6 +9,7 @@ // using System; +using System.Diagnostics; using System.Linq; using System.Text; using System.Text.RegularExpressions; @@ -51,6 +52,8 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto if (ws > 0) { string id = lgwsf.GetStrFromWs(ws); + if (string.IsNullOrEmpty(id)) + throw new ArgumentException("Unknown writing system handle " + ws + "; cannot write an without a ws.", "ws"); writer.WriteStartElement("AStr"); writer.WriteAttributeString("ws", Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC)); } @@ -61,6 +64,7 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto // Write the properties and text for each run string fieldName = null; + int previousWs = 0; for (int i = 0; i < normalizedTss.RunCount; i++) { TsRunInfo tri; @@ -152,6 +156,10 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto } } + // Work out the text that will actually be written, so that we can tell whether the reader + // will require a ws on this run. + string writtenText; + bool preserveSpace = false; if (pict != null) { // Write the bytes of the picture data @@ -162,21 +170,47 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto if (j % 32 == 31) sb.AppendLine(); } - writer.WriteString(sb.ToString()); + writtenText = sb.ToString(); } else if (hotGuid) { - writer.WriteString(string.Empty); + writtenText = string.Empty; } else { string runText = normalizedTss.get_RunText(i) ?? string.Empty; - if (runText != string.Empty && runText.All(char.IsWhiteSpace)) - writer.WriteAttributeString("xml", "space", "", "preserve"); + preserveSpace = runText != string.Empty && runText.All(char.IsWhiteSpace); // TODO: should we escape quotation marks? this is not necessary but different than the behavior of the C++ implementation - writer.WriteString(StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC))); + writtenText = StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC)); } + // A run that needs a ws but has none we can write (an unknown handle, or no ws at all) gets a + // fallback ws. Otherwise we would write a record that no reader can load, and throwing here + // would stop the whole project from being saved. + int runWs; + FwTextPropVar wsVar; + bool hasKnownWs = textProps.TryGetIntValue(FwTextPropType.ktptWs, out wsVar, out runWs) && runWs > 0 + && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(runWs)); + if (hasKnownWs) + { + previousWs = runWs; + } + else if (RunNeedsWs(writtenText)) + { + int fallbackWs = GetFallbackWs(lgwsf, ws, previousWs); + if (fallbackWs > 0) + { + Trace.TraceWarning("TsStringSerializer: run has no usable writing system (handle " + runWs + + "); writing it with fallback ws '" + lgwsf.GetStrFromWs(fallbackWs) + "'."); + writer.WriteAttributeString("ws", Normalizer.Normalize(lgwsf.GetStrFromWs(fallbackWs), Normalizer.UNormalizationMode.UNORM_NFC)); + previousWs = fallbackWs; + } + } + + if (preserveSpace) + writer.WriteAttributeString("xml", "space", "", "preserve"); + writer.WriteString(writtenText); + writer.WriteEndElement(); if (markItem) writer.WriteEndElement(); @@ -277,18 +311,68 @@ private static ITsString HandleComplexString(XElement xml, ILgWritingSystemFacto var strBldr = TsStringUtils.MakeIncStrBldr(); + // The ws of the enclosing , if it has one, is the best fallback for a run whose ws is empty. It is + // resolved lazily, only when a fallback is needed: resolving it can create the ws or fail, and records + // whose runs all have a ws must load exactly as they always did. + string enclosingWsId = null; + bool enclosingWsMissing = false; + if (xml.Name.LocalName == "AStr") + { + XAttribute astrWs = xml.Attribute("ws"); + if (astrWs != null && astrWs.Value.Length > 0) + enclosingWsId = astrWs.Value; + else + enclosingWsMissing = true; + } + int previousWs = 0; + foreach (XElement runElement in runs) { if (runElement == null) throw new XmlSchemaException("TsString XML must contain a element contained in a <" + xml.Name.LocalName + "> element"); string runText = runElement.Value; - if (runElement.Attribute("ws") == null && (runText.Length == 0 || runText[0] > 13)) + XAttribute runWsAttr = runElement.Attribute("ws"); + if (runWsAttr == null && RunNeedsWs(runText)) throw new XmlSchemaException("Run element must contain a ws attribute. Run text: " + runElement.Value); + bool needsFallbackWs = runWsAttr != null && runWsAttr.Value.Length == 0 && RunNeedsWs(runText); // Make sure the text is in the decomposed form (FWR-148) runText = Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFD); bool isOrcNeeded = TsPropsSerializer.GetPropAttributesForElement(runElement, lgwsf, strBldr); + if (runWsAttr != null && runWsAttr.Value.Length > 0) + { + previousWs = GetWsForId(runWsAttr.Value, lgwsf); + } + else if (needsFallbackWs) + { + // An empty ws was written by older versions for an unknown handle. Give the run a known ws so that + // it can be saved again. If the enclosing has no ws either, the run's ws would become the key + // of the alternative (see MultiAccessor.ReadAstrElementOfMultiString), so do not invent one then: + // that could overwrite a real alternative. + int enclosingWs = 0; + if (enclosingWsId != null) + { + try + { + enclosingWs = GetWsForId(enclosingWsId, lgwsf); + } + catch (XmlSchemaException) + { + // not usable as a fallback; try the next choice + } + } + int fallbackWs = enclosingWs > 0 ? enclosingWs : previousWs > 0 ? previousWs : + (enclosingWsMissing ? 0 : GetFallbackWs(lgwsf, 0, 0)); + Trace.TraceWarning("TsStringSerializer: run has an empty ws; " + (fallbackWs > 0 + ? "using fallback ws '" + lgwsf.GetStrFromWs(fallbackWs) + "'." : "leaving it without a ws.")); + if (fallbackWs > 0) + { + strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, fallbackWs); + previousWs = fallbackWs; + } + } + // Add an ORC character, if needed, for the run if (runText.Length == 0 && isOrcNeeded) runText = StringUtils.kszObject; @@ -323,8 +407,8 @@ private static ITsString HandleSimpleString(XElement rootXml, ILgWritingSystemFa return null; // Way too complex for this simple case XAttribute wsAttribute = textElement.Attributes().First(); - if (wsAttribute.Name.LocalName != "ws") - return null; // we handle only single runs with only the ws attribute. + if (wsAttribute.Name.LocalName != "ws" || wsAttribute.Value.Length == 0) + return null; // we handle only single runs with only a (non-empty) ws attribute; an empty ws, written by older versions for an unknown handle, is dropped by the general path. // Make sure the text is in the decomposed form (FWR-148) string runText = Normalizer.Normalize(textElement.Value, Normalizer.UNormalizationMode.UNORM_NFD); return TsStringUtils.MakeString(runText, GetWsForId(wsAttribute.Value, lgwsf)); @@ -333,6 +417,33 @@ private static ITsString HandleSimpleString(XElement rootXml, ILgWritingSystemFa #region Internal methods /// ------------------------------------------------------------------------------------ + /// + /// Whether the reader requires a ws attribute on a run whose text, as written to XML, is + /// . A run that is empty (e.g. an ORC or link run) or starts with an ordinary + /// character needs one; a run starting with a control character up to U+000D (tab, newline...) does not. + /// + internal static bool RunNeedsWs(string writtenText) + { + return writtenText.Length == 0 || writtenText[0] > 13; + } + + /// + /// Chooses a known ws for a run that has none: the enclosing ws, else the previous run's ws, else the user ws. + /// Returns 0 if none is known. + /// + internal static int GetFallbackWs(ILgWritingSystemFactory lgwsf, int enclosingWs, int previousWs) + { + foreach (int candidate in new[] { enclosingWs, previousWs }) + { + if (candidate > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(candidate))) + return candidate; + } + // The user ws is the established default. In FieldWorks it is always set. A factory with no user ws + // creates one ("en") here; that is preferable to failing the save, which would block the whole project. + int userWs = lgwsf.UserWs; + return userWs > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(userWs)) ? userWs : 0; + } + /// /// Gets the ws handle for specified writing system identifier. /// diff --git a/src/SIL.LCModel/DomainImpl/CmObject.cs b/src/SIL.LCModel/DomainImpl/CmObject.cs index 2a623fa9..045196ba 100644 --- a/src/SIL.LCModel/DomainImpl/CmObject.cs +++ b/src/SIL.LCModel/DomainImpl/CmObject.cs @@ -3114,6 +3114,10 @@ private void UserDefinedProperties(XmlWriter writer) var tsString = (ITsString)data; int temp; var wsHvo = tsString.get_Properties(0).GetIntPropValues((int)FwTextPropType.ktptWs, out temp); + // A first run in an unknown ws cannot be the ws of an ; write a and let each run + // get a fallback ws instead. + if (wsHvo > 0 && string.IsNullOrEmpty(wsf.GetStrFromWs(wsHvo))) + wsHvo = 0; writer.WriteRaw(TsStringUtils.GetXmlRep(tsString, wsf, wsHvo)); break; case CellarPropertyType.Binary: diff --git a/src/SIL.LCModel/DomainImpl/Strings.cs b/src/SIL.LCModel/DomainImpl/Strings.cs index c554f5e9..05b597d5 100644 --- a/src/SIL.LCModel/DomainImpl/Strings.cs +++ b/src/SIL.LCModel/DomainImpl/Strings.cs @@ -368,6 +368,14 @@ protected virtual void ToXml(XmlWriter writer, ILgWritingSystemFactory wsf, int if (string.IsNullOrEmpty(alternative.Text)) return; // Skip writing TsStrings with no content. + // An alternative whose ws the factory does not know has no id to write, and giving it another ws could + // overwrite a real alternative. Skip it rather than fail the whole save. + if (string.IsNullOrEmpty(wsf.GetStrFromWs(ws))) + { + Trace.TraceWarning("MultiAccessor: skipping alternative with unknown writing system handle " + ws); + return; + } + writer.WriteRaw(TsStringUtils.GetXmlRep(alternative, wsf, ws)); } @@ -898,6 +906,13 @@ protected override void ToXml(XmlWriter writer, ILgWritingSystemFactory wsf, int if (string.IsNullOrEmpty(text)) return; // Skip writing TsStrings with no content. + // An alternative whose ws the factory does not know has no id to write. Skip it rather than fail the whole save. + if (string.IsNullOrEmpty(wsf.GetStrFromWs(ws))) + { + Trace.TraceWarning("MultiUnicodeAccessor: skipping alternative with unknown writing system handle " + ws); + return; + } + writer.WriteStartElement("AUni"); writer.WriteAttributeString("ws", m_object.Services.WritingSystemManager.Get(ws).Id); text = TsStringSerializer.StripInvalidXmlChars(Normalizer.Normalize(text, Normalizer.UNormalizationMode.UNORM_NFC)); diff --git a/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs new file mode 100644 index 00000000..c97167ab --- /dev/null +++ b/tests/SIL.LCModel.Core.Tests/Text/TsUnknownWsHandleTests.cs @@ -0,0 +1,263 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using NUnit.Framework; +using SIL.LCModel.Core.KernelInterfaces; + +namespace SIL.LCModel.Core.Text +{ + /// + /// An unknown writing system handle used to be written as ws="", which no reader accepts. A run of text + /// cannot be written without a ws either, so such runs are written (and loaded) with a fallback ws. + /// + [TestFixture] + public class TsUnknownWsHandleTests : TsSerializerTestsBase + { + private const int UnknownWs = 987654; + + private ITsTextProps MakeProps(FwTextPropType wsType, int ws) + { + ITsPropsBldr bldr = TsStringUtils.MakePropsBldr(); + bldr.SetStrPropValue((int)FwTextPropType.ktptNamedStyle, "Emphasis"); + bldr.SetIntPropValues((int)wsType, 0, ws); + return bldr.GetTextProps(); + } + + private static int WsOfRun(ITsString tss, int irun) + { + int var; + return tss.get_Properties(irun).GetIntPropValues((int)FwTextPropType.ktptWs, out var); + } + + /// Asserts that every run has a ws the factory knows. + private void AssertAllRunsHaveKnownWs(ITsString tss) + { + for (int i = 0; i < tss.RunCount; i++) + { + int ws = WsOfRun(tss, i); + Assert.That(ws, Is.GreaterThan(0), "run " + i); + Assert.That(WritingSystemManager.GetStrFromWs(ws), Is.Not.Null.And.Not.Empty, "run " + i); + } + } + + /// + /// Serializes, reads, re-saves and reads again, checking that no step throws, that no empty ws is written + /// and that every run of the final string has a known ws. + /// + private ITsString SaveAndReloadTwice(ITsString tss, int astrWs = 0) + { + string xml1 = TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager, astrWs); + Assert.That(xml1, Does.Not.Contain("ws=\"\"")); + ITsString read1 = TsStringSerializer.DeserializeTsStringFromXml(xml1, WritingSystemManager); + string xml2 = TsStringSerializer.SerializeTsStringToXml(read1, WritingSystemManager, astrWs); + Assert.That(xml2, Does.Not.Contain("ws=\"\"")); + ITsString read2 = TsStringSerializer.DeserializeTsStringFromXml(xml2, WritingSystemManager); + Assert.That(read2.Text ?? string.Empty, Is.EqualTo(tss.Text ?? string.Empty).Or.EqualTo( + (tss.Text ?? string.Empty).Replace("\u0001", string.Empty))); + AssertAllRunsHaveKnownWs(read2); + return read2; + } + + #region Props + [TestCase(FwTextPropType.ktptWs, "ws")] + [TestCase(FwTextPropType.ktptBaseWs, "wsBase")] + public void SerializeProps_UnknownWs_OmitsAttributeAndRoundTrips(FwTextPropType wsType, string attrName) + { + string xml = TsPropsSerializer.SerializePropsToXml(MakeProps(wsType, UnknownWs), WritingSystemManager); + + Assert.That(xml, Does.Not.Contain(attrName + "=")); + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); + int var; + Assert.That(result.GetIntPropValues((int)wsType, out var), Is.EqualTo(-1)); + } + + [Test] + public void SerializeProps_KnownWs_StillWritten() + { + string xml = TsPropsSerializer.SerializePropsToXml(MakeProps(FwTextPropType.ktptWs, EnWS), WritingSystemManager); + Assert.That(xml, Does.Contain("ws=\"en\"")); + } + + [TestCase("")] + [TestCase("")] + [TestCase("")] + public void DeserializeProps_EmptyWs_IsDropped(string xml) + { + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + Assert.That(result.GetStrPropValue((int)FwTextPropType.ktptNamedStyle), Is.EqualTo("Emphasis")); + int var; + Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptWs, out var), Is.EqualTo(-1)); + Assert.That(result.GetIntPropValues((int)FwTextPropType.ktptBaseWs, out var), Is.EqualTo(-1)); + } + + [Test] + public void DeserializeProps_EmptyWsInWsProp_IsDropped() + { + string xml = "" + + ""; + ITsTextProps result = TsPropsSerializer.DeserializePropsFromXml(xml, WritingSystemManager); + string wsStyles = result.GetStrPropValue((int)FwTextPropType.ktptWsStyle); + Assert.That(wsStyles, Does.Contain("Times")); + Assert.That(wsStyles, Does.Not.Contain("Arial")); + } + #endregion + + #region Runs: writing + [Test] + public void SerializeString_RunWithUnknownBaseWs_RoundTrips() + { + ITsStrBldr bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "text", TsStringUtils.MakeProps("Emphasis", EnWS)); + bldr.SetIntPropValues(0, 4, (int)FwTextPropType.ktptBaseWs, 0, UnknownWs); + + string xml = TsStringSerializer.SerializeTsStringToXml(bldr.GetString(), WritingSystemManager); + + Assert.That(xml, Does.Not.Contain("wsBase")); + ITsString result = SaveAndReloadTwice(bldr.GetString()); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(EnWS)); + } + + [Test] + public void SerializeString_RunWithUnknownWs_UsesUserWs() + { + ITsString result = SaveAndReloadTwice(TsStringUtils.MakeString("text", UnknownWs)); + Assert.That(result.Text, Is.EqualTo("text")); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(WritingSystemManager.UserWs)); + } + + [Test] + public void SerializeString_RunWithUnknownWs_UsesPreviousRunWs() + { + ITsStrBldr bldr = TsStringUtils.MakeStrBldr(); + bldr.Replace(0, 0, "one", TsStringUtils.MakeProps(null, EsWS)); + bldr.Replace(3, 3, "two", TsStringUtils.MakeProps(null, UnknownWs)); + + ITsString result = SaveAndReloadTwice(bldr.GetString()); + + Assert.That(WsOfRun(result, result.RunCount - 1), Is.EqualTo(EsWS)); + Assert.That(result.Text, Is.EqualTo("onetwo")); + } + + [Test] + public void SerializeString_AStrRunWithUnknownWs_UsesAStrWs() + { + ITsString result = SaveAndReloadTwice(TsStringUtils.MakeString("text", UnknownWs), EsWS); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(EsWS)); + } + + [Test] + public void SerializeString_EmptyRunWithUnknownWs_Loads() + { + // The reader requires a ws on an empty run too. + string xml = TsStringSerializer.SerializeTsStringToXml(TsStringUtils.MakeString(string.Empty, UnknownWs), WritingSystemManager); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + Assert.That(() => TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), Throws.Nothing); + } + + [Test] + public void SerializeString_RunStartingWithStrippedControlCharAndUnknownWs_Loads() + { + // U+0001 is dropped when written, so the run text the reader sees starts with 'a' and needs a ws. + ITsString result = SaveAndReloadTwice(TsStringUtils.MakeString("\u0001abc", UnknownWs)); + Assert.That(result.Text, Is.EqualTo("abc")); + Assert.That(result.get_WritingSystem(0), Is.EqualTo(WritingSystemManager.UserWs)); + } + + [Test] + public void SerializeString_RunStartingWithTabAndUnknownWs_NeedsNoWs() + { + string xml = TsStringSerializer.SerializeTsStringToXml(TsStringUtils.MakeString("\tabc", UnknownWs), WritingSystemManager); + Assert.That(xml, Does.Not.Contain("ws=")); + Assert.That(TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager).Text, Is.EqualTo("\tabc")); + } + + [TestCase(FwObjDataTypes.kodtOwnNameGuidHot)] + [TestCase(FwObjDataTypes.kodtNameGuidHot)] + public void SerializeString_OrcRunWithUnknownWs_Loads(FwObjDataTypes type) + { + ITsString orc = TsStringUtils.CreateOrcFromGuid(Guid.NewGuid(), type, UnknownWs); + ITsString result = null; + string xml = TsStringSerializer.SerializeTsStringToXml(orc, WritingSystemManager); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + Assert.That(() => result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), Throws.Nothing); + AssertAllRunsHaveKnownWs(result); + } + + [Test] + public void SerializeString_KnownWs_IsUnchanged() + { + string xml = TsStringSerializer.SerializeTsStringToXml(TsStringUtils.MakeString("text", EsWS), WritingSystemManager); + Assert.That(xml, Does.Contain("text")); + } + + [Test] + public void SerializeString_AStrWithUnknownWs_Fails() + { + // There is no ws we could safely use as the key of the alternative. LCM skips such alternatives + // before they get here. + ITsString tss = TsStringUtils.MakeString("text", EnWS); + Assert.That(() => TsStringSerializer.SerializeTsStringToXml(tss, WritingSystemManager, UnknownWs), + Throws.TypeOf()); + } + #endregion + + #region Runs: reading legacy ws="" + [TestCase("text", "en")] + [TestCase("text", "en")] + [TestCase("text", "es")] + [TestCase("text", "es")] + [TestCase("text", "es")] + [TestCase("text", "en")] + public void DeserializeString_EmptyWs_GetsFallbackWsAndResaves(string xml, string expectedWsOfLastRun) + { + ITsString read1 = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager); + + Assert.That(read1.Text, Is.EqualTo("text")); + AssertAllRunsHaveKnownWs(read1); + Assert.That(WsOfRun(read1, read1.RunCount - 1), Is.EqualTo(WritingSystemManager.GetWsFromStr(expectedWsOfLastRun))); + // and the result can be saved and loaded again + bool isAStr = xml.StartsWith(" is only needed as a fallback for a run with an empty ws. Master ignores it when + // the runs have their own, so such records must keep loading. + string xml = "text"; + ITsString result = null; + Assert.That(() => result = TsStringSerializer.DeserializeTsStringFromXml(xml, WritingSystemManager), Throws.Nothing); + Assert.That(result.Text, Is.EqualTo("text")); + } + + [Test] + public void DeserializeString_EmptyRunWithEmptyWs_ResavesAndLoads() + { + ITsString read1 = TsStringSerializer.DeserializeTsStringFromXml("", WritingSystemManager); + string xml2 = TsStringSerializer.SerializeTsStringToXml(read1, WritingSystemManager); + Assert.That(xml2, Does.Not.Contain("ws=\"\"")); + Assert.That(() => TsStringSerializer.DeserializeTsStringFromXml(xml2, WritingSystemManager), Throws.Nothing); + } + + [Test] + public void DeserializeString_EmptyWsInAStrWithoutWs_IsNotGivenAWs() + { + // The run's ws is the key of the alternative when has no ws, so inventing one could replace a real + // alternative. Such a string stays without a ws, as before. + ITsString read = TsStringSerializer.DeserializeTsStringFromXml("text", WritingSystemManager); + Assert.That(read.get_WritingSystem(0), Is.LessThanOrEqualTo(0)); + } + #endregion + } +} diff --git a/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs b/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs index ee6360b8..aafbd4b9 100644 --- a/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs +++ b/tests/SIL.LCModel.Tests/DomainImpl/StringsTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Xml.Linq; using NUnit.Framework; using SIL.LCModel.Core.KernelInterfaces; using SIL.LCModel.Core.Text; @@ -305,6 +306,88 @@ public void ToXml_WithControlCharacterWorks() Assert.AreEqual("test", xml, "XML does not contain expected string element."); } + private const int UnknownWs = 987654; + + /// + /// An alternative keyed by a writing system handle the factory does not know cannot be written. It is skipped + /// (rather than failing the whole save), and the real alternatives are still written. + /// + [Test] + public void ToXml_AlternativeWithUnknownWs_IsSkipped() + { + var english = Cache.LangProject.CurrentAnalysisWritingSystems.First(); + var region = Cache.LangProject.Description; + region.set_String(english.Handle, TsStringUtils.MakeString("Real", english.Handle)); + ((MultiAccessor)region).Add(UnknownWs, TsStringUtils.MakeString("Stale", UnknownWs)); + + string xml = null; + Assert.That(() => xml = ToXml(region), Throws.Nothing); + + Assert.That(xml, Does.Contain("Real")); + Assert.That(xml, Does.Not.Contain("Stale")); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + } + + /// + /// The same for a MultiUnicode (<AUni>) alternative. + /// + [Test] + public void ToXml_UnicodeAlternativeWithUnknownWs_IsSkipped() + { + var english = Cache.LangProject.CurrentAnalysisWritingSystems.First(); + var country = Cache.LangProject.MainCountry; + country.set_String(english.Handle, TsStringUtils.MakeString("Real", english.Handle)); + ((MultiAccessor)country).Add(UnknownWs, TsStringUtils.MakeString("Stale", UnknownWs)); + + string xml = null; + Assert.That(() => xml = ToXml(country), Throws.Nothing); + + Assert.That(xml, Does.Contain("Real")); + Assert.That(xml, Does.Not.Contain("Stale")); + } + + /// + /// A run with an empty ws, written by older versions, is loaded with the ws of its alternative, and what is + /// saved from it can be loaded again. + /// + [Test] + public void FromXml_AStrWithEmptyRunWs_UsesAStrWsAndResaves() + { + var english = Cache.LangProject.CurrentAnalysisWritingSystems.First(); + var region = Cache.LangProject.Description; + var wsf = Cache.WritingSystemFactory; + + ((MultiAccessor)region).Clear(); + ((MultiAccessor)region).LoadFromDataStoreInternal( + XElement.Parse("legacy"), wsf); + + Assert.That(region.get_String(english.Handle).Text, Is.EqualTo("legacy")); + Assert.That(region.get_String(english.Handle).get_WritingSystem(0), Is.EqualTo(english.Handle)); + string xml = ToXml(region); + Assert.That(xml, Does.Not.Contain("ws=\"\"")); + ITsString tss; + Assert.That(MultiAccessor.ReadAstrElementOfMultiString(XElement.Parse("" + xml + "").Elements("AStr").First(a => a.Attribute("ws").Value == "en"), wsf, out tss), Is.EqualTo(english.Handle)); + Assert.That(tss.Text, Is.EqualTo("legacy")); + } + + /// + /// An alternative with no ws on either the AStr or its run cannot be placed. It must not be given the user + /// ws, which would overwrite the real alternative in that ws. + /// + [Test] + public void FromXml_AStrAndRunWithEmptyWs_DoesNotOverwriteRealAlternative() + { + var english = Cache.WritingSystemFactory.GetWsFromStr("en"); + var region = Cache.LangProject.Description; + var wsf = Cache.WritingSystemFactory; + Cache.ServiceLocator.WritingSystemManager.UserWs = english; + region.set_String(english, TsStringUtils.MakeString("Real", english)); + ((MultiAccessor)region).LoadFromDataStoreInternal( + XElement.Parse("lost"), wsf); + + Assert.That(region.get_String(english).Text, Is.EqualTo("Real")); + } + private string ToXml(ITsMultiString multiString) { using var ms = new MemoryStream();