diff --git a/.github/workflows/check-grammars.yml b/.github/workflows/check-grammars.yml index 677d970..eff43c7 100644 --- a/.github/workflows/check-grammars.yml +++ b/.github/workflows/check-grammars.yml @@ -47,12 +47,16 @@ jobs: g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt + # The CP1251 locale stands in for a Windows build, where Boost folds case by the ANSI + # code page: there, bytes above 0x7F are letters with a case. - name: Check Unicode highlighting and recursion run: | g++ -std=c++20 -O0 -g -Ilibprisma -o tools/check-unicode \ tools/check-unicode.cpp libprisma/Highlight.cpp libprisma/LanguageTree.cpp \ libprisma/SyntaxHighlighter.cpp libprisma/TokenList.cpp -lboost_regex - ./tools/check-unicode libprisma/grammars.dat + mkdir -p "$RUNNER_TEMP/locale" + localedef -i ru_RU -f CP1251 "$RUNNER_TEMP/locale/ru_RU.CP1251" + LOCPATH="$RUNNER_TEMP/locale" ./tools/check-unicode libprisma/grammars.dat ru_RU.CP1251 - name: Compare what the two engines matched run: | diff --git a/README.md b/README.md index 8fb403c..2a4e26e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`. |`Brainfuck`|`brainfuck`| |`BrightScript`|`brightscript`| |`Bro`|`bro`| +|`BSL (1C:Enterprise)`|`bsl`,`oscript`| |`CFScript`|`cfscript`,`cfc`| |`ChaiScript`|`chaiscript`| |`CIL`|`cil`| diff --git a/generate.js b/generate.js index 92a6086..cf1c810 100644 --- a/generate.js +++ b/generate.js @@ -125,6 +125,78 @@ function escapeNonAscii(pattern) { return result } +// What JavaScript's /i compares a character as, without the u flag (ECMAScript's +// Canonicalize): its upper case, unless that is more than one character, or is ASCII while +// the character is not. So /ſ/i matches only ſ, though "ſ".toUpperCase() is "S". +function canonicalize(c) { + const upper = c.toUpperCase() + if (upper.length !== 1) { + return c + } + return c.charCodeAt(0) >= 0x80 && upper.charCodeAt(0) < 0x80 ? c : upper +} + +// every UTF-16 code unit, grouped by what /i compares it as +let caseGroups = null + +function sameCase(c) { + if (!caseGroups) { + caseGroups = new Map() + for (let i = 0; i < 0x10000; i++) { + const x = String.fromCharCode(i) + const key = canonicalize(x) + caseGroups.set(key, (caseGroups.get(key) || '') + x) + } + } + return caseGroups.get(canonicalize(c)) +} + +// Boost compares bytes, and java.util.regex folds only ASCII without UNICODE_CASE, so /и/i +// would not match "И". Each non-ASCII letter of a case-insensitive pattern is spelled out as +// the class of characters JavaScript's /i treats as equal to it: /µ/i becomes /[µΜμ]/i. +function foldCase(pattern) { + const flags = pattern.slice(pattern.lastIndexOf('/') + 1) + if (!flags.includes('i') || !/[^\x00-\x7F]/.test(pattern)) { + return pattern + } + + const cases = c => c > '\x7F' && sameCase(c).length > 1 ? sameCase(c) : null + + let result = '' + let inClass = false + + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i] + + if (c === '\\') { + result += pattern.substr(i, 2) + i++ + continue + } + + if (inClass) { + // A letter inside a class, such as [а-я], would need its other case added to the + // class as well. No grammar has one yet, so stop rather than emit a pattern that + // silently stays case-sensitive. + if (cases(c)) { + throw new Error(`foldCase does not handle the letter ${c} inside a character class: ${pattern}`) + } + if (c === ']') { + inClass = false + } + } else if (c === '[') { + inClass = true + } else if (cases(c)) { + result += '[' + cases(c) + ']' + continue + } + + result += c + } + + return result +} + function unique(a, fn) { if (a.length === 0 || a.length === 1) { return a; @@ -260,14 +332,9 @@ async function generate() { function sanitize(pattern) { // Unsupported: - // UTF-16 ranges // [^] => [\s\S] <- matches any character, including new line // [] => ?? <- matches _empty_ string - // All the whitelisted languages have 0xFFFF as maximum range - // This is not the case for all the grammars supported by Prisma. - - pattern = pattern.replaceAll("\\uFFFF", "\\xFF"); pattern = pattern.replaceAll("[^]", "[\\s\\S]"); // TODO: This just bruteforces the regex to work, but of course @@ -280,7 +347,7 @@ async function generate() { pattern = pattern.replaceAll("|[])", ")"); pattern = pattern.replaceAll(":[]", ":"); - return escapeNonAscii(normalizeEscapes(pattern)) + return escapeNonAscii(foldCase(normalizeEscapes(pattern))) } for (var token in copy) { @@ -347,7 +414,6 @@ async function generate() { } var unsupported = [ - "bsl", "coq", "gherkin", "jexl", diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index 6b8b6fc..17f3150 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -7,10 +7,11 @@ namespace libprisma { // Boost matches over bytes and does not read a \uXXXX escape at all: a pattern holding one - // simply never matches. Spell every escape as the bytes of its UTF-8 encoding instead, and - // where one sits inside a character class - whose members have to be a single byte each - - // lift it out into an alternation. This belongs here rather than in grammars.dat, which - // keeps the \u spelling for the clients that match over UTF-16 and read it correctly. + // simply never matches. It also reads \xHH as a byte, where JavaScript and java.util.regex + // read the code point. Spell every escape above ASCII as the bytes of its UTF-8 encoding + // instead, and where one sits inside a character class - whose members have to be a single + // byte each - lift it out into an alternation. This belongs here rather than in + // grammars.dat, which keeps the original spelling for the clients that match over UTF-16. inline bool isHexDigit(char c) { @@ -44,6 +45,18 @@ namespace libprisma return value; } + // the code point of the \xHH escape at begin, or -1 if there is not one there + inline int byteEscapeAt(std::string_view pattern, size_t begin) + { + if (begin + 4 > pattern.size() || pattern[begin] != '\\' || pattern[begin + 1] != 'x' + || !isHexDigit(pattern[begin + 2]) || !isHexDigit(pattern[begin + 3])) + { + return -1; + } + + return hexValue(pattern[begin + 2]) * 16 + hexValue(pattern[begin + 3]); + } + inline void appendByte(std::string& out, unsigned char byte) { static const char digits[] = "0123456789ABCDEF"; @@ -53,37 +66,138 @@ namespace libprisma out += digits[byte & 0x0F]; } - inline void appendUtf8(std::string& out, int codePoint) + // Writes the UTF-8 encoding of codePoint into bytes and returns the number of bytes used. + inline size_t encodeUtf8(int codePoint, unsigned char (&bytes)[4]) { if (codePoint < 0x80) { - appendByte(out, static_cast(codePoint)); + bytes[0] = static_cast(codePoint); + return 1; } - else if (codePoint < 0x800) + if (codePoint < 0x800) { - appendByte(out, static_cast(0xC0 | (codePoint >> 6))); - appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + bytes[0] = static_cast(0xC0 | (codePoint >> 6)); + bytes[1] = static_cast(0x80 | (codePoint & 0x3F)); + return 2; } - else if (codePoint < 0x10000) + if (codePoint < 0x10000) { - appendByte(out, static_cast(0xE0 | (codePoint >> 12))); - appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F))); - appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + bytes[0] = static_cast(0xE0 | (codePoint >> 12)); + bytes[1] = static_cast(0x80 | ((codePoint >> 6) & 0x3F)); + bytes[2] = static_cast(0x80 | (codePoint & 0x3F)); + return 3; } - else + bytes[0] = static_cast(0xF0 | (codePoint >> 18)); + bytes[1] = static_cast(0x80 | ((codePoint >> 12) & 0x3F)); + bytes[2] = static_cast(0x80 | ((codePoint >> 6) & 0x3F)); + bytes[3] = static_cast(0x80 | (codePoint & 0x3F)); + return 4; + } + + inline void appendUtf8(std::string& out, int codePoint) + { + unsigned char bytes[4]; + const size_t length = encodeUtf8(codePoint, bytes); + for (size_t i = 0; i < length; ++i) { - appendByte(out, static_cast(0xF0 | (codePoint >> 18))); - appendByte(out, static_cast(0x80 | ((codePoint >> 12) & 0x3F))); - appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F))); - appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + appendByte(out, bytes[i]); } } - // The code point at begin, joining a surrogate pair into the one character it spells, with - // the characters it occupies in length. A lone \uXXXX is left to Boost, which expands it - // to UTF-8 itself; a pair it would expand to two three byte sequences instead. + // Adds to alternatives the patterns that match the UTF-8 encoding of every code point from + // low to high. Each pattern is a sequence of byte ranges: for example, U+0400 to U+04FF + // becomes [\xD0-\xD3][\x80-\xBF]. + // + // Not every range can be written as one such sequence, so it is split into pieces that + // can: first where the length of the encoding changes (at U+0080, U+0800 and U+10000), + // then wherever the continuation bytes would not cover their full range \x80-\xBF. This + // is the usual approach; Rust's regex-syntax, for example, does the same in Utf8Sequences. + inline void appendUtf8Range(std::vector& alternatives, int low, int high) + { + for (const int last : { 0x7F, 0x7FF, 0xFFFF }) + { + if (low <= last && high > last) + { + appendUtf8Range(alternatives, low, last); + appendUtf8Range(alternatives, last + 1, high); + return; + } + } + + for (int i = 1; i < 4; ++i) + { + const int tail = (1 << (6 * i)) - 1; + if ((low & ~tail) == (high & ~tail)) + { + continue; + } + if ((low & tail) != 0) + { + appendUtf8Range(alternatives, low, low | tail); + appendUtf8Range(alternatives, (low | tail) + 1, high); + return; + } + if ((high & tail) != tail) + { + appendUtf8Range(alternatives, low, (high & ~tail) - 1); + appendUtf8Range(alternatives, high & ~tail, high); + return; + } + } + + unsigned char lows[4]; + unsigned char highs[4]; + const size_t length = encodeUtf8(low, lows); + encodeUtf8(high, highs); + + std::string sequence; + for (size_t i = 0; i < length; ++i) + { + if (lows[i] == highs[i]) + { + appendByte(sequence, lows[i]); + continue; + } + sequence += '['; + appendByte(sequence, lows[i]); + sequence += '-'; + appendByte(sequence, highs[i]); + sequence += ']'; + } + alternatives.push_back(std::move(sequence)); + } + + // Same as appendUtf8Range, but skips the surrogates U+D800 to U+DFFF: they have no UTF-8 + // encoding, so no text can contain them. + inline void appendCodePoints(std::vector& alternatives, int low, int high) + { + // In JavaScript a range over every surrogate takes both halves of each pair, so it + // takes every character above U+FFFF too. + if (low <= 0xD800 && high >= 0xDFFF) + { + appendUtf8Range(alternatives, 0x10000, 0x10FFFF); + } + + if (low < 0xD800) + { + appendUtf8Range(alternatives, low, high < 0xD7FF ? high : 0xD7FF); + } + if (high > 0xDFFF) + { + appendUtf8Range(alternatives, low > 0xE000 ? low : 0xE000, high); + } + } + + // The code point of the \u or \x escape at begin, joining a surrogate pair into the one + // character it spells, with the characters it occupies in length; -1 if there is none. inline int codePointAt(std::string_view pattern, size_t begin, size_t& length) { + if (const int byte = byteEscapeAt(pattern, begin); byte >= 0) + { + length = 4; + return byte; + } + const int value = unicodeEscapeAt(pattern, begin); length = value < 0 ? 0 : 6; @@ -103,8 +217,9 @@ namespace libprisma struct ClassMember { std::string_view text; // as spelled in the pattern, "a", "\\n", "\\u2227", "a-z" - int low = -1; // code point, for a member that is a lone \u escape - int high = -1; // and the other end, when the member is a range of them + bool range = false; // "a-z" rather than "a" + int low = -1; // code point of the character, or of the start of the range; + int high = -1; // of its end; -1 for a set such as \w, or no end at all }; struct CharacterClass @@ -115,17 +230,47 @@ namespace libprisma std::vector members; }; + // The character at begin inside a class, as its code point, with the characters it occupies; + // -1 for a set such as \w or \d, and for an escape whose meaning differs between engines. + inline size_t classCharacterAt(std::string_view pattern, size_t at, int& codePoint) + { + size_t length = 0; + codePoint = codePointAt(pattern, at, length); + if (codePoint >= 0) + { + return length; + } + + const unsigned char c = static_cast(pattern[at]); + if (c != '\\' || at + 1 >= pattern.size()) + { + codePoint = c < 0x80 ? c : -1; + return 1; + } + + const unsigned char escaped = static_cast(pattern[at + 1]); + switch (escaped) + { + case 'n': codePoint = '\n'; break; + case 'r': codePoint = '\r'; break; + case 't': codePoint = '\t'; break; + case 'f': codePoint = '\f'; break; + case 'b': codePoint = '\b'; break; + default: + // \w, \d, \s and the rest are sets; an escaped punctuation mark is itself + const bool letterOrDigit = (escaped >= '0' && escaped <= '9') + || ((escaped | 0x20) >= 'a' && (escaped | 0x20) <= 'z'); + codePoint = letterOrDigit || escaped >= 0x80 ? -1 : escaped; + break; + } + return 2; + } + // one class member starting at begin, or 0 if the class ends there inline size_t readMember(std::string_view pattern, size_t begin, ClassMember& member) { auto unit = [&](size_t at, int& codePoint) -> size_t { - size_t length = 0; - codePoint = codePointAt(pattern, at, length); - if (codePoint >= 0) - { - return length; - } - return pattern[at] == '\\' && at + 1 < pattern.size() ? 2 : 1; + return classCharacterAt(pattern, at, codePoint); }; int low = -1; @@ -137,6 +282,7 @@ namespace libprisma { int high = -1; size_t tail = unit(at + 1, high); + member.range = true; member.low = low; member.high = high; member.text = pattern.substr(begin, length + 1 + tail); @@ -220,7 +366,7 @@ namespace libprisma return 0; } - out += "(?:[\\xF0-\\xF4][\\x80-\\xBF]{3})"; + out += "(?-i:[\\xF0-\\xF4][\\x80-\\xBF]{3})"; return end - begin; } @@ -233,68 +379,109 @@ namespace libprisma return 0; } - std::vector wide; + std::vector alternatives; std::string narrow; for (const auto& member : characters.members) { - if (member.low <= 0x7F && member.high <= 0x7F) + if (!member.range) + { + if (member.low > 0x7F) + { + appendCodePoints(alternatives, member.low, member.low); + } + else + { + narrow += member.text; + } + } + else if (member.low < 0 || member.high < 0 || member.low > member.high) { + // a range this code cannot read; leave a class above ASCII to Boost as it is + if (member.low > 0x7F || member.high > 0x7F) + { + return 0; + } narrow += member.text; } - else if (member.high < 0) + else if (member.high <= 0x7F) { - wide.push_back(member.low); + narrow += member.text; } - else if (!characters.negated) + else { - // a range of code points that is not the surrogate idiom; leave it be rather - // than expand what could be the whole of Unicode - return 0; + // the ASCII part of the range stays in the class, where the i flag applies to it + int low = member.low; + if (low <= 0x7F) + { + appendByte(narrow, static_cast(low)); + narrow += '-'; + appendByte(narrow, 0x7F); + low = 0x80; + } + appendCodePoints(alternatives, low, member.high); } } - if (wide.empty()) + std::string any; + if (!narrow.empty()) { - out += '['; - if (characters.negated) + any += '['; + if (narrow.front() == '^') { - out += '^'; + any += '\\'; } - out += narrow; - out += ']'; - return characters.end - begin; + any += narrow; + any += ']'; } - // Exclude whole UTF-8 sequences without rejecting bytes shared by other characters. - out += characters.negated ? "(?:(?!" : "(?:"; - if (!narrow.empty()) + // The UTF-8 bytes go in (?-i:), so that icase does not fold them by the locale's code + // page; the narrow members stay outside it, where icase applies to them. + if (!alternatives.empty()) { - out += '['; - if (narrow.front() == '^') + if (!any.empty()) + { + any += '|'; + } + any += "(?-i:"; + for (size_t i = 0; i < alternatives.size(); ++i) { - out += '\\'; + if (i > 0) + { + any += '|'; + } + any += alternatives[i]; } - out += narrow; - out += ']'; - out += '|'; + any += ')'; } - for (size_t i = 0; i < wide.size(); ++i) + + if (characters.negated) { - if (i > 0) + // One whole UTF-8 character that the lookahead does not exclude. Each branch has a + // fixed length, so backtracking cannot give up part of the character. + out += "(?:"; + if (!any.empty()) { - out += '|'; + out += "(?!" + any + ")"; } - appendUtf8(out, wide[i]); + out += "(?-i:[\\x00-\\x7F]|[\\xC0-\\xDF][\\x80-\\xBF]|[\\xE0-\\xEF][\\x80-\\xBF]{2}|[\\xF0-\\xF7][\\x80-\\xBF]{3}))"; + } + else + { + out += any.empty() ? "(?!)" : "(?:" + any + ")"; } - out += characters.negated ? ")[\\s\\S])" : ")"; return characters.end - begin; } inline std::string widenUnicodeClasses(std::string_view pattern) { - if (pattern.find("\\u") == std::string_view::npos) + bool wide = pattern.find("\\u") != std::string_view::npos; + for (size_t at = pattern.find("\\x"); !wide && at != std::string_view::npos; at = pattern.find("\\x", at + 2)) + { + wide = byteEscapeAt(pattern, at) > 0x7F; + } + if (!wide) { return std::string{ pattern }; } @@ -312,11 +499,12 @@ namespace libprisma if (codePoint >= 0) { // grouped, because a quantifier after it has to apply to the whole - // character and not to the last byte of its encoding + // character and not to the last byte of its encoding, and with icase off, + // so that the bytes are not folded by the locale's code page const bool multiByte = codePoint > 0x7F; if (multiByte) { - out += "(?:"; + out += "(?-i:"; } appendUtf8(out, codePoint); if (multiByte) diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index 67c10cb..43fcd31 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -182,16 +183,27 @@ bool CheckAliases( bool CheckPattern( const std::string& pattern, const std::string& text, - bool expected) + bool expected, + bool icase = false) { - const auto regex = boost::regex( - libprisma::widenUnicodeClasses(pattern), - boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m); - if (boost::regex_match(text, regex) == expected) + try { - return true; + auto flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m; + if (icase) + { + flags |= boost::regex_constants::icase; + } + const auto regex = boost::regex(libprisma::widenUnicodeClasses(pattern), flags); + if (boost::regex_match(text, regex) == expected) + { + return true; + } + std::cerr << "Unexpected match for " << pattern << ": " << text << '\n'; + } + catch (const std::exception& e) + { + std::cerr << "Boost will not take " << pattern << ": " << e.what() << '\n'; } - std::cerr << "Unexpected match for " << pattern << ": " << text << '\n'; return false; } @@ -247,9 +259,9 @@ bool CheckThreads(const std::string& grammars, const SyntaxHighlighter& referenc int main(int argc, char** argv) { - if (argc != 2) + if (argc != 2 && argc != 3) { - std::cerr << "usage: check-unicode \n"; + std::cerr << "usage: check-unicode []\n"; return 2; } auto input = std::ifstream(argv[1], std::ios::binary); @@ -313,6 +325,48 @@ int main(int argc, char** argv) failed += !CheckPattern(R"([\u201c\u201d]+)", "\u201c\u201d", true); failed += !CheckPattern(R"([\uD800-\uDBFF][\uDC00-\uDFFF])", "\U0001F600", true); + // A range of code points inside a class must match whole UTF-8 characters, whatever the + // length of their encoding, and a negated class must consume one whole character, not one + // byte. These patterns are plain string literals: "\\u0400" is the escape a pattern holds. + failed += !CheckPattern("[\\u0400-\\u04ff]+", "\U0000041F\U00000440\U00000438\U00000432\U00000435\U00000442", true); + failed += !CheckPattern("[\\u0400-\\u04ff]+", "Hi", false); + failed += !CheckPattern("[\\w\\u0400-\\u0484\\u0487-\\u052f]+", "a_1\U00000401\U00000436\U00000487\U0000052F", true); + failed += !CheckPattern("[\\w\\u0400-\\u0484\\u0487-\\u052f]", "\U00000485", false); + failed += !CheckPattern("[\\w\\u0400-\\u0484\\u0487-\\u052f]", "\U00000530", false); + failed += !CheckPattern("[\\u0061-\\u0800]+", "az\U000000E9\U000007FF\U00000800", true); + failed += !CheckPattern("[\\u0061-\\u0800]", "`", false); + failed += !CheckPattern("[\\u0061-\\u0800]", "\U00000801", false); + failed += !CheckPattern("[\\ua640-\\ua69f\\ufe2e]{2}", "\U0000A640\U0000FE2E", true); + failed += !CheckPattern("[\\ud7ff-\\ue000]+", "\U0000D7FF\U0000E000", true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00000416", false); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "_", false); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", " ", true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00002014", true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]{2}", "\U00002014\U0001F600", true); + failed += !CheckPattern("[^\\u2019]", "\U00002014", true); + // What follows a negated class cannot match the rest of the character the class took. + failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U00000416", false); + failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U0001F600", false); + failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U00000416a", true); + + // \xHH is a code point, as JavaScript and java.util.regex read it, not a byte: inside a + // class, at either end of a range, and outside a class. + failed += !CheckPattern("[\\u00e0-\\xff]+", "\U000000E0\U000000F0\U000000FF", true); + failed += !CheckPattern("[\\u00e0-\\xff]", "f", false); + failed += !CheckPattern("[\\xa0-\\u00ff]+", "\U000000A0\U000000E9", true); + failed += !CheckPattern("[\\xa0-\\xff]", "\U000000E9", true); + failed += !CheckPattern("[^\\x85\\u2019]", "\U00000085", false); + failed += !CheckPattern("[^\\x85\\u2019]", "\U0000041F", true); + failed += !CheckPattern("a\\xe9+", "a\U000000E9\U000000E9", true); + failed += !CheckPattern("[a-\\u00ff]+", "az\U000000E9", true); + + // A range over every surrogate takes both halves of each pair in JavaScript, so in UTF-8 it + // takes the characters above U+FFFF as well: Prism's "any but ASCII" is \x80 to U+FFFF. + failed += !CheckPattern("[\\x80-\\uffff]+", "\U000000E9\U0001F600", true); + failed += !CheckPattern("[^\\ud800-\\udfff]", "\U0001F600", false); + failed += !CheckPattern("[^\\ud800-\\udfff]", "\U0000FFFF", true); + failed += !CheckPattern("[\\x80-\\ud7ff]", "\U0001F600", false); + const auto text = std::string("#iFclude "); const auto tokens = highlighter.tokenize(text, "brightscript"); auto result = Inspection(); @@ -530,8 +584,75 @@ int main(int argc, char** argv) { "string", "\"a\"" }, }); + // BSL keywords can be Cyrillic or Latin, in any letter case. A keyword must be a whole word: + // in "ПеременнаяЕсли" the trailing "Если" is part of a name, not a keyword. + failed += !CheckTokens( + highlighter, + "bsl", + "Процедура Тест() Экспорт\n" + "\tЕСЛИ НЕ а Тогда КонецЕсли;\n" + "\tПеременнаяЕсли = 1;\n" + "КонецПроцедуры", + { + { "keyword", "Процедура" }, + { "punctuation", "(" }, + { "punctuation", ")" }, + { "keyword", "Экспорт" }, + { "keyword", "ЕСЛИ" }, + { "operator", "НЕ" }, + { "keyword", "Тогда" }, + { "keyword", "КонецЕсли" }, + { "punctuation", ";" }, + { "operator", "=" }, + { "number", "1" }, + { "punctuation", ";" }, + { "keyword", "КонецПроцедуры" }, + }); + failed += !CheckTokens( + highlighter, + "oscript", + "Сообщить(\"Привет\");", + { + { "punctuation", "(" }, + { "string", "\"Привет\"" }, + { "punctuation", ")" }, + { "punctuation", ";" }, + }); + + // A letter in a case-insensitive pattern matches what JavaScript's /i matches: every + // character with the same upper case. OpenQASM's "µs" is micro sign, Greek mu or capital Mu. + for (const auto& micro : { "\U000000B5", "\U000003BC", "\U0000039C" }) + { + failed += !CheckTokens( + highlighter, + "openqasm", + std::string("delay[5") + micro + "s] q;", + { + { "keyword", "delay" }, + { "punctuation", "[" }, + { "number", std::string("5") + micro + "s" }, + { "punctuation", "]" }, + { "punctuation", ";" }, + }); + } + + // Prism spells "any character but ASCII" as \xA0 to U+FFFF, as in JavaScript's function names. + failed += !CheckTokens( + highlighter, + "javascript", + "функция(1);", + { + { "function", "функция" }, + { "punctuation", "(" }, + { "number", "1" }, + { "punctuation", ")" }, + { "punctuation", ";" }, + }); + // A file extension that is no language name reads as itself, as an unknown name does. for (const auto& [language, name] : std::vector>{ + { "bsl", "BSL (1C:Enterprise)" }, + { "oscript", "OneScript" }, { "rs", "rs" }, { "rust", "Rust" }, { "nope", "nope" }, @@ -572,6 +693,38 @@ int main(int argc, char** argv) failed += !CheckThreads(grammars, highlighter); + // With icase, Boost folds each byte by the locale it compiles a pattern under: on Windows + // the ANSI code page, where bytes above 0x7F are letters with a case. The UTF-8 bytes this + // library writes must not be folded: in CP1251, D0 B2 ("в") would equal D0 B3 ("г"), and + // the range 80-84 would turn into 90-84, which Boost rejects. + if (argc == 3) + { + std::locale::global(std::locale(argv[2])); + + failed += !CheckPattern("[\\u0432\\u0412]", "\U00000433", false, true); + failed += !CheckPattern("[\\u0438\\u0418]", "\U00000428", false, true); + failed += !CheckPattern("[\\u0432\\u0412]", "\U00000432", true, true); + failed += !CheckPattern("[\\u0400-\\u0484]", "\U00000480", true, true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00000416", false, true); + + const auto code = std::string( + "Процедура Тест() Экспорт\n" + "\tЕСЛИ НЕ а Тогда КонецЕсли;\n" + "\tПеременнаяЕсли = 1;\n" + "КонецПроцедуры"); + auto expected = Inspection(); + Inspect(highlighter.tokenize(code, "bsl"), expected); + auto result = Inspection(); + Inspect(SyntaxHighlighter(grammars).tokenize(code, "bsl"), result); + if (result.tokens != expected.tokens) + { + std::cerr << "BSL tokens differ under " << argv[2] << '\n'; + ++failed; + } + + std::locale::global(std::locale::classic()); + } + std::cout << failed << " Unicode highlighting checks failed\n"; return failed ? 1 : 0; }