Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).

### Fixed

- [SIL.LCModel.Core] A writing-system handle the factory does not know is no longer saved as `ws=""`/`wsBase=""`, which no reader could load. An unknown `ws`/`wsBase` on text properties is omitted; a text run with no usable ws is written with a fallback ws (the enclosing `<AStr>` ws, else the previous run's, else the user ws) and a trace warning. Readers give a run with an existing empty `ws` the same fallback, so such records load and can be saved again. `MultiAccessor` and `MultiUnicodeAccessor` skip alternatives keyed by an unknown ws (#411)
- [SIL.LCModel] BaseStyleInfo now loads default font features (ktptFontVariations) from style rules, fixing font features set as style defaults being dropped on reload (LT-22351)
- [SIL.LCModel] BaseStyleInfo no longer clobbers ws-specific font overrides inherited from a based-on style with the default font info's merely-inherited values; only explicitly-set default properties are propagated to ws overrides (LT-22351)
- [SIL.LCModel] BulletInfo now decodes all properties of an encoded bullet font info instead of stopping at the first string property, fixing bullet font features (ktptFontVariations) being lost when a bullet font name was also set (LT-22351)
Expand Down
32 changes: 23 additions & 9 deletions src/SIL.LCModel.Core/Text/TsPropsSerializer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,13 @@ internal static void WriteIntProperty(XmlWriter writer, ILgWritingSystemFactory
if (value > 0)
{
string id = lgwsf.GetStrFromWs(value);
writer.WriteAttributeString(tpt == FwTextPropType.ktptWs ? "ws" : "wsBase", Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC));
string attrName = tpt == FwTextPropType.ktptWs ? "ws" : "wsBase";
// An unknown handle has no id. Writing it as ws="" would produce XML that no reader accepts
// (the property is optional, so it is safer to leave it out).
if (string.IsNullOrEmpty(id))
Trace.TraceWarning("TsPropsSerializer: omitting '" + attrName + "' attribute for unknown writing system handle " + value);
else
writer.WriteAttributeString(attrName, Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC));
}
break;
case FwTextPropType.ktptFontSize:
Expand Down Expand Up @@ -863,6 +869,8 @@ private static void AddWsStyleInfoToBldr(ITsPropsBldr bldr, XElement wsStyleInfo
if (wsAttrib == null)
throw new XmlSchemaException("WsProp must contain a 'ws' attribute");
wsAttrib.Remove(); // Make sure we don't count it twice
if (wsAttrib.Value.Length == 0)
continue; // tolerate an empty ws written by older versions for an unknown handle
WriteIntToStrBuilder(wsStyleProps, TsStringSerializer.GetWsForId(wsAttrib.Value, lgwsf));

// Get the font family for the prop override
Expand Down Expand Up @@ -1138,12 +1146,15 @@ internal static ITsPropsBldr GetPropAttributesForElement(XElement xml, ILgWritin
GetIntValueFromStr(attr.Value, false));
break;
case "ws":
propsBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
// Older versions wrote ws="" for an unknown handle; drop it rather than failing to load.
if (attr.Value.Length > 0)
propsBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
break;
case "wsBase":
propsBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
if (attr.Value.Length > 0)
propsBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
break;
case "wsStyle":
Debug.Fail("We don't support the old wsStyle attribute!");
Expand Down Expand Up @@ -1395,12 +1406,15 @@ internal static bool GetPropAttributesForElement(XElement xml, ILgWritingSystemF
GetIntValueFromStr(attr.Value, false));
break;
case "ws":
strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
// Older versions wrote ws="" for an unknown handle; drop it rather than failing to load.
if (attr.Value.Length > 0)
strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
break;
case "wsBase":
strBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
if (attr.Value.Length > 0)
strBldr.SetIntPropValues((int)FwTextPropType.ktptBaseWs, 0,
TsStringSerializer.GetWsForId(attr.Value, lgwsf));
break;
case "wsStyle":
Debug.Fail("We don't support the old wsStyle attribute!");
Expand Down
127 changes: 119 additions & 8 deletions src/SIL.LCModel.Core/Text/TsStringSerializer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
// </remarks>

using System;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -51,6 +52,8 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto
if (ws > 0)
{
string id = lgwsf.GetStrFromWs(ws);
if (string.IsNullOrEmpty(id))
throw new ArgumentException("Unknown writing system handle " + ws + "; cannot write an <AStr> without a ws.", "ws");
writer.WriteStartElement("AStr");
writer.WriteAttributeString("ws", Normalizer.Normalize(id, Normalizer.UNormalizationMode.UNORM_NFC));
}
Expand All @@ -61,6 +64,7 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto

// Write the properties and text for each run
string fieldName = null;
int previousWs = 0;
for (int i = 0; i < normalizedTss.RunCount; i++)
{
TsRunInfo tri;
Expand Down Expand Up @@ -152,6 +156,10 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto
}
}

// Work out the text that will actually be written, so that we can tell whether the reader
// will require a ws on this run.
string writtenText;
bool preserveSpace = false;
if (pict != null)
{
// Write the bytes of the picture data
Expand All @@ -162,21 +170,47 @@ public static string SerializeTsStringToXml(ITsString tss, ILgWritingSystemFacto
if (j % 32 == 31)
sb.AppendLine();
}
writer.WriteString(sb.ToString());
writtenText = sb.ToString();
}
else if (hotGuid)
{
writer.WriteString(string.Empty);
writtenText = string.Empty;
}
else
{
string runText = normalizedTss.get_RunText(i) ?? string.Empty;
if (runText != string.Empty && runText.All(char.IsWhiteSpace))
writer.WriteAttributeString("xml", "space", "", "preserve");
preserveSpace = runText != string.Empty && runText.All(char.IsWhiteSpace);
// TODO: should we escape quotation marks? this is not necessary but different than the behavior of the C++ implementation
writer.WriteString(StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC)));
writtenText = StripInvalidXmlChars(Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFC));
}

// A run that needs a ws but has none we can write (an unknown handle, or no ws at all) gets a
// fallback ws. Otherwise we would write a record that no reader can load, and throwing here
// would stop the whole project from being saved.
int runWs;
FwTextPropVar wsVar;
bool hasKnownWs = textProps.TryGetIntValue(FwTextPropType.ktptWs, out wsVar, out runWs) && runWs > 0
&& !string.IsNullOrEmpty(lgwsf.GetStrFromWs(runWs));
if (hasKnownWs)
{
previousWs = runWs;
}
else if (RunNeedsWs(writtenText))
{
int fallbackWs = GetFallbackWs(lgwsf, ws, previousWs);
if (fallbackWs > 0)
{
Trace.TraceWarning("TsStringSerializer: run has no usable writing system (handle " + runWs +
"); writing it with fallback ws '" + lgwsf.GetStrFromWs(fallbackWs) + "'.");
writer.WriteAttributeString("ws", Normalizer.Normalize(lgwsf.GetStrFromWs(fallbackWs), Normalizer.UNormalizationMode.UNORM_NFC));
previousWs = fallbackWs;
}
}

if (preserveSpace)
writer.WriteAttributeString("xml", "space", "", "preserve");
writer.WriteString(writtenText);

writer.WriteEndElement();
if (markItem)
writer.WriteEndElement();
Expand Down Expand Up @@ -277,18 +311,68 @@ private static ITsString HandleComplexString(XElement xml, ILgWritingSystemFacto

var strBldr = TsStringUtils.MakeIncStrBldr();

// The ws of the enclosing <AStr>, if it has one, is the best fallback for a run whose ws is empty. It is
// resolved lazily, only when a fallback is needed: resolving it can create the ws or fail, and records
// whose runs all have a ws must load exactly as they always did.
string enclosingWsId = null;
bool enclosingWsMissing = false;
if (xml.Name.LocalName == "AStr")
{
XAttribute astrWs = xml.Attribute("ws");
if (astrWs != null && astrWs.Value.Length > 0)
enclosingWsId = astrWs.Value;
else
enclosingWsMissing = true;
}
int previousWs = 0;

foreach (XElement runElement in runs)
{
if (runElement == null)
throw new XmlSchemaException("TsString XML must contain a <Run> element contained in a <" + xml.Name.LocalName + "> element");
string runText = runElement.Value;
if (runElement.Attribute("ws") == null && (runText.Length == 0 || runText[0] > 13))
XAttribute runWsAttr = runElement.Attribute("ws");
if (runWsAttr == null && RunNeedsWs(runText))
throw new XmlSchemaException("Run element must contain a ws attribute. Run text: " + runElement.Value);

bool needsFallbackWs = runWsAttr != null && runWsAttr.Value.Length == 0 && RunNeedsWs(runText);
// Make sure the text is in the decomposed form (FWR-148)
runText = Normalizer.Normalize(runText, Normalizer.UNormalizationMode.UNORM_NFD);
bool isOrcNeeded = TsPropsSerializer.GetPropAttributesForElement(runElement, lgwsf, strBldr);

if (runWsAttr != null && runWsAttr.Value.Length > 0)
{
previousWs = GetWsForId(runWsAttr.Value, lgwsf);
}
else if (needsFallbackWs)
{
// An empty ws was written by older versions for an unknown handle. Give the run a known ws so that
// it can be saved again. If the enclosing <AStr> has no ws either, the run's ws would become the key
// of the alternative (see MultiAccessor.ReadAstrElementOfMultiString), so do not invent one then:
// that could overwrite a real alternative.
int enclosingWs = 0;
if (enclosingWsId != null)
{
try
{
enclosingWs = GetWsForId(enclosingWsId, lgwsf);
}
catch (XmlSchemaException)
{
// not usable as a fallback; try the next choice
}
}
int fallbackWs = enclosingWs > 0 ? enclosingWs : previousWs > 0 ? previousWs :
(enclosingWsMissing ? 0 : GetFallbackWs(lgwsf, 0, 0));
Trace.TraceWarning("TsStringSerializer: run has an empty ws; " + (fallbackWs > 0
? "using fallback ws '" + lgwsf.GetStrFromWs(fallbackWs) + "'." : "leaving it without a ws."));
if (fallbackWs > 0)
{
strBldr.SetIntPropValues((int)FwTextPropType.ktptWs, 0, fallbackWs);
previousWs = fallbackWs;
}
}

// Add an ORC character, if needed, for the run
if (runText.Length == 0 && isOrcNeeded)
runText = StringUtils.kszObject;
Expand Down Expand Up @@ -323,8 +407,8 @@ private static ITsString HandleSimpleString(XElement rootXml, ILgWritingSystemFa
return null; // Way too complex for this simple case

XAttribute wsAttribute = textElement.Attributes().First();
if (wsAttribute.Name.LocalName != "ws")
return null; // we handle only single runs with only the ws attribute.
if (wsAttribute.Name.LocalName != "ws" || wsAttribute.Value.Length == 0)
return null; // we handle only single runs with only a (non-empty) ws attribute; an empty ws, written by older versions for an unknown handle, is dropped by the general path.
// Make sure the text is in the decomposed form (FWR-148)
string runText = Normalizer.Normalize(textElement.Value, Normalizer.UNormalizationMode.UNORM_NFD);
return TsStringUtils.MakeString(runText, GetWsForId(wsAttribute.Value, lgwsf));
Expand All @@ -333,6 +417,33 @@ private static ITsString HandleSimpleString(XElement rootXml, ILgWritingSystemFa

#region Internal methods
/// ------------------------------------------------------------------------------------
/// <summary>
/// Whether the reader requires a ws attribute on a run whose text, as written to XML, is
/// <paramref name="writtenText"/>. A run that is empty (e.g. an ORC or link run) or starts with an ordinary
/// character needs one; a run starting with a control character up to U+000D (tab, newline...) does not.
/// </summary>
internal static bool RunNeedsWs(string writtenText)
{
return writtenText.Length == 0 || writtenText[0] > 13;
}

/// <summary>
/// Chooses a known ws for a run that has none: the enclosing ws, else the previous run's ws, else the user ws.
/// Returns 0 if none is known.
/// </summary>
internal static int GetFallbackWs(ILgWritingSystemFactory lgwsf, int enclosingWs, int previousWs)
{
foreach (int candidate in new[] { enclosingWs, previousWs })
{
if (candidate > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(candidate)))
return candidate;
}
// The user ws is the established default. In FieldWorks it is always set. A factory with no user ws
// creates one ("en") here; that is preferable to failing the save, which would block the whole project.
int userWs = lgwsf.UserWs;
return userWs > 0 && !string.IsNullOrEmpty(lgwsf.GetStrFromWs(userWs)) ? userWs : 0;
}

/// <summary>
/// Gets the ws handle for specified writing system identifier.
/// </summary>
Expand Down
4 changes: 4 additions & 0 deletions src/SIL.LCModel/DomainImpl/CmObject.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3114,6 +3114,10 @@ private void UserDefinedProperties(XmlWriter writer)
var tsString = (ITsString)data;
int temp;
var wsHvo = tsString.get_Properties(0).GetIntPropValues((int)FwTextPropType.ktptWs, out temp);
// A first run in an unknown ws cannot be the ws of an <AStr>; write a <Str> and let each run
// get a fallback ws instead.
if (wsHvo > 0 && string.IsNullOrEmpty(wsf.GetStrFromWs(wsHvo)))
wsHvo = 0;
writer.WriteRaw(TsStringUtils.GetXmlRep(tsString, wsf, wsHvo));
break;
case CellarPropertyType.Binary:
Expand Down
15 changes: 15 additions & 0 deletions src/SIL.LCModel/DomainImpl/Strings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,14 @@ protected virtual void ToXml(XmlWriter writer, ILgWritingSystemFactory wsf, int
if (string.IsNullOrEmpty(alternative.Text))
return; // Skip writing TsStrings with no content.

// An alternative whose ws the factory does not know has no id to write, and giving it another ws could
// overwrite a real alternative. Skip it rather than fail the whole save.
if (string.IsNullOrEmpty(wsf.GetStrFromWs(ws)))
{
Trace.TraceWarning("MultiAccessor: skipping alternative with unknown writing system handle " + ws);
return;
}

writer.WriteRaw(TsStringUtils.GetXmlRep(alternative, wsf, ws));
}

Expand Down Expand Up @@ -898,6 +906,13 @@ protected override void ToXml(XmlWriter writer, ILgWritingSystemFactory wsf, int
if (string.IsNullOrEmpty(text))
return; // Skip writing TsStrings with no content.

// An alternative whose ws the factory does not know has no id to write. Skip it rather than fail the whole save.
if (string.IsNullOrEmpty(wsf.GetStrFromWs(ws)))
{
Trace.TraceWarning("MultiUnicodeAccessor: skipping alternative with unknown writing system handle " + ws);
return;
}

writer.WriteStartElement("AUni");
writer.WriteAttributeString("ws", m_object.Services.WritingSystemManager.Get(ws).Id);
text = TsStringSerializer.StripInvalidXmlChars(Normalizer.Normalize(text, Normalizer.UNormalizationMode.UNORM_NFC));
Expand Down
Loading
Loading