From 39d0f918b82f9199ef14bbf1d6a04242cc887b85 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:57:34 +0000 Subject: [PATCH 1/7] fix: support ranges of non-ASCII characters in character classes widenUnicodeClasses converted single \u escapes to UTF-8, but left ranges such as U+0400 to U+04FF unchanged. Boost cannot compile a class with such a range and throws an exception. tokenize catches it and stops, so the rest of the text is returned without highlighting. In a negated class the same ranges were dropped instead, so the class did not exclude those characters. Each range is now converted into the UTF-8 byte sequences that cover it. Surrogates are skipped, because valid UTF-8 never contains them. A negated class now consumes a whole UTF-8 character instead of a single byte, so it can no longer match in the middle of a character. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- libprisma/UnicodeEscapes.h | 187 +++++++++++++++++++++++++++++-------- tools/check-unicode.cpp | 39 ++++++-- 2 files changed, 180 insertions(+), 46 deletions(-) diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index 6b8b6fc..59055cf 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -53,29 +53,127 @@ namespace libprisma out += digits[byte & 0x0F]; } - inline void appendUtf8(std::string& out, int codePoint) + // Writes the UTF-8 encoding of codePoint into bytes and returns the number of bytes used. + inline size_t encodeUtf8(int codePoint, unsigned char (&bytes)[4]) { if (codePoint < 0x80) { - appendByte(out, static_cast(codePoint)); + bytes[0] = static_cast(codePoint); + return 1; } - else if (codePoint < 0x800) + if (codePoint < 0x800) { - appendByte(out, static_cast(0xC0 | (codePoint >> 6))); - appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + bytes[0] = static_cast(0xC0 | (codePoint >> 6)); + bytes[1] = static_cast(0x80 | (codePoint & 0x3F)); + return 2; } - else if (codePoint < 0x10000) + if (codePoint < 0x10000) { - appendByte(out, static_cast(0xE0 | (codePoint >> 12))); - appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F))); - appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + bytes[0] = static_cast(0xE0 | (codePoint >> 12)); + bytes[1] = static_cast(0x80 | ((codePoint >> 6) & 0x3F)); + bytes[2] = static_cast(0x80 | (codePoint & 0x3F)); + return 3; } - else + bytes[0] = static_cast(0xF0 | (codePoint >> 18)); + bytes[1] = static_cast(0x80 | ((codePoint >> 12) & 0x3F)); + bytes[2] = static_cast(0x80 | ((codePoint >> 6) & 0x3F)); + bytes[3] = static_cast(0x80 | (codePoint & 0x3F)); + return 4; + } + + inline void appendUtf8(std::string& out, int codePoint) + { + unsigned char bytes[4]; + const size_t length = encodeUtf8(codePoint, bytes); + for (size_t i = 0; i < length; ++i) { - appendByte(out, static_cast(0xF0 | (codePoint >> 18))); - appendByte(out, static_cast(0x80 | ((codePoint >> 12) & 0x3F))); - appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F))); - appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + appendByte(out, bytes[i]); + } + } + + // Adds to alternatives the patterns that match the UTF-8 encoding of every code point from + // low to high. Each pattern is a sequence of byte ranges: for example, U+0400 to U+04FF + // becomes [\xD0-\xD3][\x80-\xBF]. + // + // Not every range can be written as one such sequence, so it is split into pieces that + // can: first where the length of the encoding changes (at U+0080, U+0800 and U+10000), + // then wherever the continuation bytes would not cover their full range \x80-\xBF. This + // is the usual approach; Rust's regex-syntax, for example, does the same in Utf8Sequences. + inline void appendUtf8Range(std::vector& alternatives, int low, int high) + { + for (const int last : { 0x7F, 0x7FF, 0xFFFF }) + { + if (low <= last && high > last) + { + appendUtf8Range(alternatives, low, last); + appendUtf8Range(alternatives, last + 1, high); + return; + } + } + + for (int i = 1; i < 4; ++i) + { + const int tail = (1 << (6 * i)) - 1; + if ((low & ~tail) == (high & ~tail)) + { + continue; + } + if ((low & tail) != 0) + { + appendUtf8Range(alternatives, low, low | tail); + appendUtf8Range(alternatives, (low | tail) + 1, high); + return; + } + if ((high & tail) != tail) + { + appendUtf8Range(alternatives, low, (high & ~tail) - 1); + appendUtf8Range(alternatives, high & ~tail, high); + return; + } + } + + unsigned char lows[4]; + unsigned char highs[4]; + const size_t length = encodeUtf8(low, lows); + encodeUtf8(high, highs); + + std::string sequence; + for (size_t i = 0; i < length; ++i) + { + if (lows[i] == highs[i]) + { + appendByte(sequence, lows[i]); + continue; + } + sequence += '['; + appendByte(sequence, lows[i]); + sequence += '-'; + appendByte(sequence, highs[i]); + sequence += ']'; + } + alternatives.push_back(std::move(sequence)); + } + + // Same as appendUtf8Range, but skips the surrogates U+D800 to U+DFFF: they have no UTF-8 + // encoding, so no text can contain them. + inline void appendCodePoints(std::vector& alternatives, int low, int high) + { + if (low < 0xD800 && high > 0xDFFF) + { + appendUtf8Range(alternatives, low, 0xD7FF); + appendUtf8Range(alternatives, 0xE000, high); + } + else if (high < 0xD800 || low > 0xDFFF) + { + appendUtf8Range(alternatives, low, high); + } + else if (low < 0xD800) + { + appendUtf8Range(alternatives, low, 0xD7FF); + } + else if (high > 0xDFFF) + { + appendUtf8Range(alternatives, 0xE000, high); } } @@ -233,7 +331,7 @@ namespace libprisma return 0; } - std::vector wide; + std::vector alternatives; std::string narrow; for (const auto& member : characters.members) @@ -244,50 +342,59 @@ namespace libprisma } else if (member.high < 0) { - wide.push_back(member.low); + appendCodePoints(alternatives, member.low, member.low); + } + else if (member.low >= 0 && member.low <= member.high) + { + appendCodePoints(alternatives, member.low, member.high); } - else if (!characters.negated) + else { - // a range of code points that is not the surrogate idiom; leave it be rather - // than expand what could be the whole of Unicode + // A range whose end is above ASCII but whose start is not a \u escape. This + // code does not convert such ranges, so leave the whole class unchanged. return 0; } } - if (wide.empty()) + if (!narrow.empty()) { - out += '['; - if (characters.negated) + std::string members = "["; + if (narrow.front() == '^') { - out += '^'; + members += '\\'; } - out += narrow; - out += ']'; - return characters.end - begin; + members += narrow; + members += ']'; + alternatives.insert(alternatives.begin(), std::move(members)); } - // Exclude whole UTF-8 sequences without rejecting bytes shared by other characters. - out += characters.negated ? "(?:(?!" : "(?:"; - if (!narrow.empty()) + std::string any; + for (size_t i = 0; i < alternatives.size(); ++i) { - out += '['; - if (narrow.front() == '^') + if (i > 0) { - out += '\\'; + any += '|'; } - out += narrow; - out += ']'; - out += '|'; + any += alternatives[i]; } - for (size_t i = 0; i < wide.size(); ++i) + + if (characters.negated) { - if (i > 0) + // Match one whole UTF-8 character that is not in the class. The lookahead rejects + // the excluded characters; then the whole character is consumed, not just its first + // byte. Consuming a single byte would let the class match the last byte of an + // excluded character, or start matching in the middle of one. + out += "(?:"; + if (!any.empty()) { - out += '|'; + out += "(?!" + any + ")"; } - appendUtf8(out, wide[i]); + out += "[\\x00-\\x7F\\xC0-\\xFF][\\x80-\\xBF]{0,3})"; + } + else + { + out += any.empty() ? "(?!)" : "(?:" + any + ")"; } - out += characters.negated ? ")[\\s\\S])" : ")"; return characters.end - begin; } diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index 67c10cb..7cce43c 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -184,14 +184,21 @@ bool CheckPattern( const std::string& text, bool expected) { - const auto regex = boost::regex( - libprisma::widenUnicodeClasses(pattern), - boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m); - if (boost::regex_match(text, regex) == expected) + try { - return true; + const auto regex = boost::regex( + libprisma::widenUnicodeClasses(pattern), + boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m); + if (boost::regex_match(text, regex) == expected) + { + return true; + } + std::cerr << "Unexpected match for " << pattern << ": " << text << '\n'; + } + catch (const std::exception& e) + { + std::cerr << "Boost will not take " << pattern << ": " << e.what() << '\n'; } - std::cerr << "Unexpected match for " << pattern << ": " << text << '\n'; return false; } @@ -313,6 +320,26 @@ int main(int argc, char** argv) failed += !CheckPattern(R"([\u201c\u201d]+)", "\u201c\u201d", true); failed += !CheckPattern(R"([\uD800-\uDBFF][\uDC00-\uDFFF])", "\U0001F600", true); + // A range of code points inside a class must match whole UTF-8 characters, whatever the + // length of their encoding, and a negated class must consume one whole character, not one + // byte. These patterns are plain string literals: "\\u0400" is the escape a pattern holds. + failed += !CheckPattern("[\\u0400-\\u04ff]+", "\U0000041F\U00000440\U00000438\U00000432\U00000435\U00000442", true); + failed += !CheckPattern("[\\u0400-\\u04ff]+", "Hi", false); + failed += !CheckPattern("[\\w\\u0400-\\u0484\\u0487-\\u052f]+", "a_1\U00000401\U00000436\U00000487\U0000052F", true); + failed += !CheckPattern("[\\w\\u0400-\\u0484\\u0487-\\u052f]", "\U00000485", false); + failed += !CheckPattern("[\\w\\u0400-\\u0484\\u0487-\\u052f]", "\U00000530", false); + failed += !CheckPattern("[\\u0061-\\u0800]+", "az\U000000E9\U000007FF\U00000800", true); + failed += !CheckPattern("[\\u0061-\\u0800]", "`", false); + failed += !CheckPattern("[\\u0061-\\u0800]", "\U00000801", false); + failed += !CheckPattern("[\\ua640-\\ua69f\\ufe2e]{2}", "\U0000A640\U0000FE2E", true); + failed += !CheckPattern("[\\ud7ff-\\ue000]+", "\U0000D7FF\U0000E000", true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00000416", false); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "_", false); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", " ", true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00002014", true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]{2}", "\U00002014\U0001F600", true); + failed += !CheckPattern("[^\\u2019]", "\U00002014", true); + const auto text = std::string("#iFclude "); const auto tokens = highlighter.tokenize(text, "brightscript"); auto result = Inspection(); From 98d178248eb2819e8f8f8f39387b71a638d6a3e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:58:28 +0000 Subject: [PATCH 2/7] feat: support BSL (1C:Enterprise) and OneScript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BSL was excluded in generate.js because Boost could not compile its patterns. The previous commit fixes that. One more problem remained. BSL keywords use the i flag, but neither Boost, which compares bytes, nor java.util.regex without UNICODE_CASE ignores the case of Cyrillic letters. Only lower-case keywords such as "если" were found, not "Если". generate.js now replaces each non-ASCII letter in a pattern with the i flag by a class with both of its cases: "если" becomes "[еЕ][сС][лЛ][иИ]". If such a letter is inside a character class, the generator stops with an error, because no grammar needs that yet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- README.md | 1 + generate.js | 59 +++++++++++++++++++++++++++++++++++++++-- tools/check-unicode.cpp | 37 ++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8fb403c..2a4e26e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`. |`Brainfuck`|`brainfuck`| |`BrightScript`|`brightscript`| |`Bro`|`bro`| +|`BSL (1C:Enterprise)`|`bsl`,`oscript`| |`CFScript`|`cfscript`,`cfc`| |`ChaiScript`|`chaiscript`| |`CIL`|`cil`| diff --git a/generate.js b/generate.js index 92a6086..9a9a6b3 100644 --- a/generate.js +++ b/generate.js @@ -125,6 +125,62 @@ function escapeNonAscii(pattern) { return result } +// The i flag makes a pattern case-insensitive only for ASCII letters in the engines that read +// grammars.dat: Boost compares bytes, and java.util.regex ignores the case of other letters +// unless UNICODE_CASE is set. So /если/i would not match "Если". +// +// To fix that, every non-ASCII letter in a pattern with the i flag is replaced by a character +// class holding its lower and upper case: /если/i becomes /[еЕ][сС][лЛ][иИ]/i. Both engines +// match that as intended. BSL needs this for its Cyrillic keywords. +function foldCase(pattern) { + const flags = pattern.slice(pattern.lastIndexOf('/') + 1) + if (!flags.includes('i') || !/[^\x00-\x7F]/.test(pattern)) { + return pattern + } + + const cases = c => { + const lower = c.toLowerCase() + const upper = c.toUpperCase() + return c > '\x7F' && lower !== upper && lower.length === 1 && upper.length === 1 + ? lower + upper + : null + } + + let result = '' + let inClass = false + + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i] + + if (c === '\\') { + result += pattern.substr(i, 2) + i++ + continue + } + + if (inClass) { + // A letter inside a class, such as [а-я], would need its other case added to the + // class as well. No grammar has one yet, so stop rather than emit a pattern that + // silently stays case-sensitive. + if (cases(c)) { + throw new Error(`foldCase does not handle the letter ${c} inside a character class: ${pattern}`) + } + if (c === ']') { + inClass = false + } + } else if (c === '[') { + inClass = true + } else if (cases(c)) { + result += '[' + cases(c) + ']' + continue + } + + result += c + } + + return result +} + function unique(a, fn) { if (a.length === 0 || a.length === 1) { return a; @@ -280,7 +336,7 @@ async function generate() { pattern = pattern.replaceAll("|[])", ")"); pattern = pattern.replaceAll(":[]", ":"); - return escapeNonAscii(normalizeEscapes(pattern)) + return escapeNonAscii(foldCase(normalizeEscapes(pattern))) } for (var token in copy) { @@ -347,7 +403,6 @@ async function generate() { } var unsupported = [ - "bsl", "coq", "gherkin", "jexl", diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index 7cce43c..209e9a9 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -557,8 +557,45 @@ int main(int argc, char** argv) { "string", "\"a\"" }, }); + // BSL keywords can be Cyrillic or Latin, in any letter case. A keyword must be a whole word: + // in "ПеременнаяЕсли" the trailing "Если" is part of a name, not a keyword. + failed += !CheckTokens( + highlighter, + "bsl", + "Процедура Тест() Экспорт\n" + "\tЕСЛИ НЕ а Тогда КонецЕсли;\n" + "\tПеременнаяЕсли = 1;\n" + "КонецПроцедуры", + { + { "keyword", "Процедура" }, + { "punctuation", "(" }, + { "punctuation", ")" }, + { "keyword", "Экспорт" }, + { "keyword", "ЕСЛИ" }, + { "operator", "НЕ" }, + { "keyword", "Тогда" }, + { "keyword", "КонецЕсли" }, + { "punctuation", ";" }, + { "operator", "=" }, + { "number", "1" }, + { "punctuation", ";" }, + { "keyword", "КонецПроцедуры" }, + }); + failed += !CheckTokens( + highlighter, + "oscript", + "Сообщить(\"Привет\");", + { + { "punctuation", "(" }, + { "string", "\"Привет\"" }, + { "punctuation", ")" }, + { "punctuation", ";" }, + }); + // A file extension that is no language name reads as itself, as an unknown name does. for (const auto& [language, name] : std::vector>{ + { "bsl", "BSL (1C:Enterprise)" }, + { "oscript", "OneScript" }, { "rs", "rs" }, { "rust", "Rust" }, { "nope", "nope" }, From 21a3de583a205e8df6db766829493eda7fa960a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 20 Nov 2026 16:02:35 +0000 Subject: [PATCH 3/7] fix: keep a negated class from matching part of a character The negated class took a whole character as a lead byte followed by up to three continuation bytes. The quantifier could backtrack and give up continuation bytes, so when what followed could match one, such as [\s\S], a match could end in the middle of a character. Match one branch per UTF-8 encoding length instead. Each branch has a fixed length, so there is nothing to give up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- libprisma/UnicodeEscapes.h | 8 +++----- tools/check-unicode.cpp | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index 59055cf..2594502 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -380,16 +380,14 @@ namespace libprisma if (characters.negated) { - // Match one whole UTF-8 character that is not in the class. The lookahead rejects - // the excluded characters; then the whole character is consumed, not just its first - // byte. Consuming a single byte would let the class match the last byte of an - // excluded character, or start matching in the middle of one. + // One whole UTF-8 character that the lookahead does not exclude. Each branch has a + // fixed length, so backtracking cannot give up part of the character. out += "(?:"; if (!any.empty()) { out += "(?!" + any + ")"; } - out += "[\\x00-\\x7F\\xC0-\\xFF][\\x80-\\xBF]{0,3})"; + out += "(?:[\\x00-\\x7F]|[\\xC0-\\xDF][\\x80-\\xBF]|[\\xE0-\\xEF][\\x80-\\xBF]{2}|[\\xF0-\\xF7][\\x80-\\xBF]{3}))"; } else { diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index 209e9a9..ad70e40 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -339,6 +339,10 @@ int main(int argc, char** argv) failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00002014", true); failed += !CheckPattern("[^\\w\\u0400-\\u04ff]{2}", "\U00002014\U0001F600", true); failed += !CheckPattern("[^\\u2019]", "\U00002014", true); + // What follows a negated class cannot match the rest of the character the class took. + failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U00000416", false); + failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U0001F600", false); + failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U00000416a", true); const auto text = std::string("#iFclude "); const auto tokens = highlighter.tokenize(text, "brightscript"); From cfd441894862ea6eb66d542ce1b634f897887882 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 20 Nov 2026 16:04:46 +0000 Subject: [PATCH 4/7] fix: fold case in generate.js the way JavaScript's /i does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit foldCase spelled out a letter as its lower and upper case. Without the u flag, JavaScript's /i treats two characters as equal when their canonical upper cases are, so /µ/i also matches Greek small mu, /σ/i matches final sigma, and a titlecase letter such as Dž matches itself. The old class dropped those, and it added k for the Kelvin sign, which /i does not match. Build the class from every UTF-16 code unit with the same canonical form, as ECMAScript's Canonicalize defines it. The only existing pattern this changes outside BSL is OpenQASM's duration unit µs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- generate.js | 44 ++++++++++++++++++++++++++++------------- tools/check-unicode.cpp | 17 ++++++++++++++++ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/generate.js b/generate.js index 9a9a6b3..770224e 100644 --- a/generate.js +++ b/generate.js @@ -125,26 +125,42 @@ function escapeNonAscii(pattern) { return result } -// The i flag makes a pattern case-insensitive only for ASCII letters in the engines that read -// grammars.dat: Boost compares bytes, and java.util.regex ignores the case of other letters -// unless UNICODE_CASE is set. So /если/i would not match "Если". -// -// To fix that, every non-ASCII letter in a pattern with the i flag is replaced by a character -// class holding its lower and upper case: /если/i becomes /[еЕ][сС][лЛ][иИ]/i. Both engines -// match that as intended. BSL needs this for its Cyrillic keywords. +// What JavaScript's /i compares a character as, without the u flag (ECMAScript's +// Canonicalize): its upper case, unless that is more than one character, or is ASCII while +// the character is not. So /ſ/i matches only ſ, though "ſ".toUpperCase() is "S". +function canonicalize(c) { + const upper = c.toUpperCase() + if (upper.length !== 1) { + return c + } + return c.charCodeAt(0) >= 0x80 && upper.charCodeAt(0) < 0x80 ? c : upper +} + +// every UTF-16 code unit, grouped by what /i compares it as +let caseGroups = null + +function sameCase(c) { + if (!caseGroups) { + caseGroups = new Map() + for (let i = 0; i < 0x10000; i++) { + const x = String.fromCharCode(i) + const key = canonicalize(x) + caseGroups.set(key, (caseGroups.get(key) || '') + x) + } + } + return caseGroups.get(canonicalize(c)) +} + +// Boost compares bytes, and java.util.regex folds only ASCII without UNICODE_CASE, so /и/i +// would not match "И". Each non-ASCII letter of a case-insensitive pattern is spelled out as +// the class of characters JavaScript's /i treats as equal to it: /µ/i becomes /[µΜμ]/i. function foldCase(pattern) { const flags = pattern.slice(pattern.lastIndexOf('/') + 1) if (!flags.includes('i') || !/[^\x00-\x7F]/.test(pattern)) { return pattern } - const cases = c => { - const lower = c.toLowerCase() - const upper = c.toUpperCase() - return c > '\x7F' && lower !== upper && lower.length === 1 && upper.length === 1 - ? lower + upper - : null - } + const cases = c => c > '\x7F' && sameCase(c).length > 1 ? sameCase(c) : null let result = '' let inClass = false diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index ad70e40..7b5589e 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -596,6 +596,23 @@ int main(int argc, char** argv) { "punctuation", ";" }, }); + // A letter in a case-insensitive pattern matches what JavaScript's /i matches: every + // character with the same upper case. OpenQASM's "µs" is micro sign, Greek mu or capital Mu. + for (const auto& micro : { "\U000000B5", "\U000003BC", "\U0000039C" }) + { + failed += !CheckTokens( + highlighter, + "openqasm", + std::string("delay[5") + micro + "s] q;", + { + { "keyword", "delay" }, + { "punctuation", "[" }, + { "number", std::string("5") + micro + "s" }, + { "punctuation", "]" }, + { "punctuation", ";" }, + }); + } + // A file extension that is no language name reads as itself, as an unknown name does. for (const auto& [language, name] : std::vector>{ { "bsl", "BSL (1C:Enterprise)" }, From c10cb03d0692993ba4dda12ac78d7c4f40fe1015 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 20 Nov 2026 16:13:04 +0000 Subject: [PATCH 5/7] fix: read \xHH as a code point and drop the U+FFFF replacement Boost reads \xHH as a byte, where JavaScript and java.util.regex read the code point. Inside a converted class this kept \x85 from excluding U+0085, and a range whose end was an \xHH escape was read as its start alone, with the hex digits taken as letters: a class from U+00E0 to \xff became "U+00E0 or f". To make Prism's ranges from \xA0 to U+FFFF compile at all, sanitize rewrote the U+FFFF escape as \xFF. Boost then matched raw bytes, so a JavaScript function name in Cyrillic was cut at its first continuation byte, and java.util.regex matched only up to U+00FF. Read \xHH as a code point wherever \u escapes are read, give ClassMember an explicit range flag, keep the ASCII part of a range in the class where the i flag applies to it, and remove the replacement. A range over every surrogate also takes the characters above U+FFFF, as JavaScript takes both halves of each pair. Tokenizing one text with Cyrillic, accented and astral characters in 279 languages, 257 now match Prism against 246 before, and none that matched before differ. Tokenizing is 10-25% slower, measured on 100 KB of JavaScript and on 125 KB of mostly Cyrillic text. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- generate.js | 5 -- libprisma/UnicodeEscapes.h | 133 +++++++++++++++++++++++++++++-------- tools/check-unicode.cpp | 31 +++++++++ 3 files changed, 138 insertions(+), 31 deletions(-) diff --git a/generate.js b/generate.js index 770224e..cf1c810 100644 --- a/generate.js +++ b/generate.js @@ -332,14 +332,9 @@ async function generate() { function sanitize(pattern) { // Unsupported: - // UTF-16 ranges // [^] => [\s\S] <- matches any character, including new line // [] => ?? <- matches _empty_ string - // All the whitelisted languages have 0xFFFF as maximum range - // This is not the case for all the grammars supported by Prisma. - - pattern = pattern.replaceAll("\\uFFFF", "\\xFF"); pattern = pattern.replaceAll("[^]", "[\\s\\S]"); // TODO: This just bruteforces the regex to work, but of course diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index 2594502..e26c99f 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -7,10 +7,11 @@ namespace libprisma { // Boost matches over bytes and does not read a \uXXXX escape at all: a pattern holding one - // simply never matches. Spell every escape as the bytes of its UTF-8 encoding instead, and - // where one sits inside a character class - whose members have to be a single byte each - - // lift it out into an alternation. This belongs here rather than in grammars.dat, which - // keeps the \u spelling for the clients that match over UTF-16 and read it correctly. + // simply never matches. It also reads \xHH as a byte, where JavaScript and java.util.regex + // read the code point. Spell every escape above ASCII as the bytes of its UTF-8 encoding + // instead, and where one sits inside a character class - whose members have to be a single + // byte each - lift it out into an alternation. This belongs here rather than in + // grammars.dat, which keeps the original spelling for the clients that match over UTF-16. inline bool isHexDigit(char c) { @@ -44,6 +45,18 @@ namespace libprisma return value; } + // the code point of the \xHH escape at begin, or -1 if there is not one there + inline int byteEscapeAt(std::string_view pattern, size_t begin) + { + if (begin + 4 > pattern.size() || pattern[begin] != '\\' || pattern[begin + 1] != 'x' + || !isHexDigit(pattern[begin + 2]) || !isHexDigit(pattern[begin + 3])) + { + return -1; + } + + return hexValue(pattern[begin + 2]) * 16 + hexValue(pattern[begin + 3]); + } + inline void appendByte(std::string& out, unsigned char byte) { static const char digits[] = "0123456789ABCDEF"; @@ -158,6 +171,13 @@ namespace libprisma // encoding, so no text can contain them. inline void appendCodePoints(std::vector& alternatives, int low, int high) { + // In JavaScript a range over every surrogate takes both halves of each pair, so it + // takes every character above U+FFFF too. + if (low <= 0xD800 && high >= 0xDFFF) + { + appendUtf8Range(alternatives, 0x10000, 0x10FFFF); + } + if (low < 0xD800 && high > 0xDFFF) { appendUtf8Range(alternatives, low, 0xD7FF); @@ -177,11 +197,16 @@ namespace libprisma } } - // The code point at begin, joining a surrogate pair into the one character it spells, with - // the characters it occupies in length. A lone \uXXXX is left to Boost, which expands it - // to UTF-8 itself; a pair it would expand to two three byte sequences instead. + // The code point of the \u or \x escape at begin, joining a surrogate pair into the one + // character it spells, with the characters it occupies in length; -1 if there is none. inline int codePointAt(std::string_view pattern, size_t begin, size_t& length) { + if (const int byte = byteEscapeAt(pattern, begin); byte >= 0) + { + length = 4; + return byte; + } + const int value = unicodeEscapeAt(pattern, begin); length = value < 0 ? 0 : 6; @@ -201,8 +226,9 @@ namespace libprisma struct ClassMember { std::string_view text; // as spelled in the pattern, "a", "\\n", "\\u2227", "a-z" - int low = -1; // code point, for a member that is a lone \u escape - int high = -1; // and the other end, when the member is a range of them + bool range = false; // "a-z" rather than "a" + int low = -1; // code point of the character, or of the start of the range; + int high = -1; // of its end; -1 for a set such as \w, or no end at all }; struct CharacterClass @@ -213,17 +239,47 @@ namespace libprisma std::vector members; }; + // The character at begin inside a class, as its code point, with the characters it occupies; + // -1 for a set such as \w or \d, and for an escape whose meaning differs between engines. + inline size_t classCharacterAt(std::string_view pattern, size_t at, int& codePoint) + { + size_t length = 0; + codePoint = codePointAt(pattern, at, length); + if (codePoint >= 0) + { + return length; + } + + const unsigned char c = static_cast(pattern[at]); + if (c != '\\' || at + 1 >= pattern.size()) + { + codePoint = c < 0x80 ? c : -1; + return 1; + } + + const unsigned char escaped = static_cast(pattern[at + 1]); + switch (escaped) + { + case 'n': codePoint = '\n'; break; + case 'r': codePoint = '\r'; break; + case 't': codePoint = '\t'; break; + case 'f': codePoint = '\f'; break; + case 'b': codePoint = '\b'; break; + default: + // \w, \d, \s and the rest are sets; an escaped punctuation mark is itself + const bool letterOrDigit = (escaped >= '0' && escaped <= '9') + || ((escaped | 0x20) >= 'a' && (escaped | 0x20) <= 'z'); + codePoint = letterOrDigit || escaped >= 0x80 ? -1 : escaped; + break; + } + return 2; + } + // one class member starting at begin, or 0 if the class ends there inline size_t readMember(std::string_view pattern, size_t begin, ClassMember& member) { auto unit = [&](size_t at, int& codePoint) -> size_t { - size_t length = 0; - codePoint = codePointAt(pattern, at, length); - if (codePoint >= 0) - { - return length; - } - return pattern[at] == '\\' && at + 1 < pattern.size() ? 2 : 1; + return classCharacterAt(pattern, at, codePoint); }; int low = -1; @@ -235,6 +291,7 @@ namespace libprisma { int high = -1; size_t tail = unit(at + 1, high); + member.range = true; member.low = low; member.high = high; member.text = pattern.substr(begin, length + 1 + tail); @@ -336,23 +393,42 @@ namespace libprisma for (const auto& member : characters.members) { - if (member.low <= 0x7F && member.high <= 0x7F) + if (!member.range) { - narrow += member.text; + if (member.low > 0x7F) + { + appendCodePoints(alternatives, member.low, member.low); + } + else + { + narrow += member.text; + } } - else if (member.high < 0) + else if (member.low < 0 || member.high < 0 || member.low > member.high) { - appendCodePoints(alternatives, member.low, member.low); + // a range this code cannot read; leave a class above ASCII to Boost as it is + if (member.low > 0x7F || member.high > 0x7F) + { + return 0; + } + narrow += member.text; } - else if (member.low >= 0 && member.low <= member.high) + else if (member.high <= 0x7F) { - appendCodePoints(alternatives, member.low, member.high); + narrow += member.text; } else { - // A range whose end is above ASCII but whose start is not a \u escape. This - // code does not convert such ranges, so leave the whole class unchanged. - return 0; + // the ASCII part of the range stays in the class, where the i flag applies to it + int low = member.low; + if (low <= 0x7F) + { + appendByte(narrow, static_cast(low)); + narrow += '-'; + appendByte(narrow, 0x7F); + low = 0x80; + } + appendCodePoints(alternatives, low, member.high); } } @@ -399,7 +475,12 @@ namespace libprisma inline std::string widenUnicodeClasses(std::string_view pattern) { - if (pattern.find("\\u") == std::string_view::npos) + bool wide = pattern.find("\\u") != std::string_view::npos; + for (size_t at = pattern.find("\\x"); !wide && at != std::string_view::npos; at = pattern.find("\\x", at + 2)) + { + wide = byteEscapeAt(pattern, at) > 0x7F; + } + if (!wide) { return std::string{ pattern }; } diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index 7b5589e..a89877f 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -344,6 +344,24 @@ int main(int argc, char** argv) failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U0001F600", false); failed += !CheckPattern("[^\\u2019][\\s\\S]", "\U00000416a", true); + // \xHH is a code point, as JavaScript and java.util.regex read it, not a byte: inside a + // class, at either end of a range, and outside a class. + failed += !CheckPattern("[\\u00e0-\\xff]+", "\U000000E0\U000000F0\U000000FF", true); + failed += !CheckPattern("[\\u00e0-\\xff]", "f", false); + failed += !CheckPattern("[\\xa0-\\u00ff]+", "\U000000A0\U000000E9", true); + failed += !CheckPattern("[\\xa0-\\xff]", "\U000000E9", true); + failed += !CheckPattern("[^\\x85\\u2019]", "\U00000085", false); + failed += !CheckPattern("[^\\x85\\u2019]", "\U0000041F", true); + failed += !CheckPattern("a\\xe9+", "a\U000000E9\U000000E9", true); + failed += !CheckPattern("[a-\\u00ff]+", "az\U000000E9", true); + + // A range over every surrogate takes both halves of each pair in JavaScript, so in UTF-8 it + // takes the characters above U+FFFF as well: Prism's "any but ASCII" is \x80 to U+FFFF. + failed += !CheckPattern("[\\x80-\\uffff]+", "\U000000E9\U0001F600", true); + failed += !CheckPattern("[^\\ud800-\\udfff]", "\U0001F600", false); + failed += !CheckPattern("[^\\ud800-\\udfff]", "\U0000FFFF", true); + failed += !CheckPattern("[\\x80-\\ud7ff]", "\U0001F600", false); + const auto text = std::string("#iFclude "); const auto tokens = highlighter.tokenize(text, "brightscript"); auto result = Inspection(); @@ -613,6 +631,19 @@ int main(int argc, char** argv) }); } + // Prism spells "any character but ASCII" as \xA0 to U+FFFF, as in JavaScript's function names. + failed += !CheckTokens( + highlighter, + "javascript", + "функция(1);", + { + { "function", "функция" }, + { "punctuation", "(" }, + { "number", "1" }, + { "punctuation", ")" }, + { "punctuation", ";" }, + }); + // A file extension that is no language name reads as itself, as an unknown name does. for (const auto& [language, name] : std::vector>{ { "bsl", "BSL (1C:Enterprise)" }, From 7245a6f7b721a3af6e79a54a856019e6568d6926 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 20 Nov 2026 16:15:36 +0000 Subject: [PATCH 6/7] refactor: write appendCodePoints as two statements The four branches that skip the surrogates are the two halves of the range on either side of them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- libprisma/UnicodeEscapes.h | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index e26c99f..ca42ba3 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -178,22 +178,13 @@ namespace libprisma appendUtf8Range(alternatives, 0x10000, 0x10FFFF); } - if (low < 0xD800 && high > 0xDFFF) + if (low < 0xD800) { - appendUtf8Range(alternatives, low, 0xD7FF); - appendUtf8Range(alternatives, 0xE000, high); + appendUtf8Range(alternatives, low, high < 0xD7FF ? high : 0xD7FF); } - else if (high < 0xD800 || low > 0xDFFF) + if (high > 0xDFFF) { - appendUtf8Range(alternatives, low, high); - } - else if (low < 0xD800) - { - appendUtf8Range(alternatives, low, 0xD7FF); - } - else if (high > 0xDFFF) - { - appendUtf8Range(alternatives, 0xE000, high); + appendUtf8Range(alternatives, low > 0xE000 ? low : 0xE000, high); } } From 5c1a77135442d2ebcee5aaaa75958e79678e168f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 20 Nov 2026 16:20:32 +0000 Subject: [PATCH 7/7] fix: keep icase from folding the UTF-8 bytes of a pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With icase, Boost folds each byte by the locale it compiles a pattern under. On Windows that is the ANSI code page, and in CP1251 the bytes above 0x7F are letters with a case. The UTF-8 bytes this library writes were folded as if they were those letters: D0 B2 ("в") matched D0 B3 ("г"), and the byte range 80-84 in U+0400 to U+0484 became 90-84, which Boost rejects. All three case-insensitive BSL patterns failed to compile under CP1251, and tokenize stopped highlighting there. Write the UTF-8 bytes inside (?-i:) and keep the ASCII members of a class outside it, where icase still applies. The case of a letter above ASCII is already spelled out by foldCase in generate.js. check-unicode takes a locale to run these checks under, and the workflow builds ru_RU.CP1251 with localedef for it. This stands in for Windows, where Boost uses w32_regex_traits rather than cpp_regex_traits; it was not run on Windows itself. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013dHLrzWVjUm5n5yTmAfmWm --- .github/workflows/check-grammars.yml | 6 +++- libprisma/UnicodeEscapes.h | 37 +++++++++++++-------- tools/check-unicode.cpp | 49 ++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check-grammars.yml b/.github/workflows/check-grammars.yml index 677d970..eff43c7 100644 --- a/.github/workflows/check-grammars.yml +++ b/.github/workflows/check-grammars.yml @@ -47,12 +47,16 @@ jobs: g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt + # The CP1251 locale stands in for a Windows build, where Boost folds case by the ANSI + # code page: there, bytes above 0x7F are letters with a case. - name: Check Unicode highlighting and recursion run: | g++ -std=c++20 -O0 -g -Ilibprisma -o tools/check-unicode \ tools/check-unicode.cpp libprisma/Highlight.cpp libprisma/LanguageTree.cpp \ libprisma/SyntaxHighlighter.cpp libprisma/TokenList.cpp -lboost_regex - ./tools/check-unicode libprisma/grammars.dat + mkdir -p "$RUNNER_TEMP/locale" + localedef -i ru_RU -f CP1251 "$RUNNER_TEMP/locale/ru_RU.CP1251" + LOCPATH="$RUNNER_TEMP/locale" ./tools/check-unicode libprisma/grammars.dat ru_RU.CP1251 - name: Compare what the two engines matched run: | diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index ca42ba3..17f3150 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -366,7 +366,7 @@ namespace libprisma return 0; } - out += "(?:[\\xF0-\\xF4][\\x80-\\xBF]{3})"; + out += "(?-i:[\\xF0-\\xF4][\\x80-\\xBF]{3})"; return end - begin; } @@ -423,26 +423,36 @@ namespace libprisma } } + std::string any; if (!narrow.empty()) { - std::string members = "["; + any += '['; if (narrow.front() == '^') { - members += '\\'; + any += '\\'; } - members += narrow; - members += ']'; - alternatives.insert(alternatives.begin(), std::move(members)); + any += narrow; + any += ']'; } - std::string any; - for (size_t i = 0; i < alternatives.size(); ++i) + // The UTF-8 bytes go in (?-i:), so that icase does not fold them by the locale's code + // page; the narrow members stay outside it, where icase applies to them. + if (!alternatives.empty()) { - if (i > 0) + if (!any.empty()) { any += '|'; } - any += alternatives[i]; + any += "(?-i:"; + for (size_t i = 0; i < alternatives.size(); ++i) + { + if (i > 0) + { + any += '|'; + } + any += alternatives[i]; + } + any += ')'; } if (characters.negated) @@ -454,7 +464,7 @@ namespace libprisma { out += "(?!" + any + ")"; } - out += "(?:[\\x00-\\x7F]|[\\xC0-\\xDF][\\x80-\\xBF]|[\\xE0-\\xEF][\\x80-\\xBF]{2}|[\\xF0-\\xF7][\\x80-\\xBF]{3}))"; + out += "(?-i:[\\x00-\\x7F]|[\\xC0-\\xDF][\\x80-\\xBF]|[\\xE0-\\xEF][\\x80-\\xBF]{2}|[\\xF0-\\xF7][\\x80-\\xBF]{3}))"; } else { @@ -489,11 +499,12 @@ namespace libprisma if (codePoint >= 0) { // grouped, because a quantifier after it has to apply to the whole - // character and not to the last byte of its encoding + // character and not to the last byte of its encoding, and with icase off, + // so that the bytes are not folded by the locale's code page const bool multiByte = codePoint > 0x7F; if (multiByte) { - out += "(?:"; + out += "(?-i:"; } appendUtf8(out, codePoint); if (multiByte) diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp index a89877f..43fcd31 100644 --- a/tools/check-unicode.cpp +++ b/tools/check-unicode.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -182,13 +183,17 @@ bool CheckAliases( bool CheckPattern( const std::string& pattern, const std::string& text, - bool expected) + bool expected, + bool icase = false) { try { - const auto regex = boost::regex( - libprisma::widenUnicodeClasses(pattern), - boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m); + auto flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m; + if (icase) + { + flags |= boost::regex_constants::icase; + } + const auto regex = boost::regex(libprisma::widenUnicodeClasses(pattern), flags); if (boost::regex_match(text, regex) == expected) { return true; @@ -254,9 +259,9 @@ bool CheckThreads(const std::string& grammars, const SyntaxHighlighter& referenc int main(int argc, char** argv) { - if (argc != 2) + if (argc != 2 && argc != 3) { - std::cerr << "usage: check-unicode \n"; + std::cerr << "usage: check-unicode []\n"; return 2; } auto input = std::ifstream(argv[1], std::ios::binary); @@ -688,6 +693,38 @@ int main(int argc, char** argv) failed += !CheckThreads(grammars, highlighter); + // With icase, Boost folds each byte by the locale it compiles a pattern under: on Windows + // the ANSI code page, where bytes above 0x7F are letters with a case. The UTF-8 bytes this + // library writes must not be folded: in CP1251, D0 B2 ("в") would equal D0 B3 ("г"), and + // the range 80-84 would turn into 90-84, which Boost rejects. + if (argc == 3) + { + std::locale::global(std::locale(argv[2])); + + failed += !CheckPattern("[\\u0432\\u0412]", "\U00000433", false, true); + failed += !CheckPattern("[\\u0438\\u0418]", "\U00000428", false, true); + failed += !CheckPattern("[\\u0432\\u0412]", "\U00000432", true, true); + failed += !CheckPattern("[\\u0400-\\u0484]", "\U00000480", true, true); + failed += !CheckPattern("[^\\w\\u0400-\\u04ff]", "\U00000416", false, true); + + const auto code = std::string( + "Процедура Тест() Экспорт\n" + "\tЕСЛИ НЕ а Тогда КонецЕсли;\n" + "\tПеременнаяЕсли = 1;\n" + "КонецПроцедуры"); + auto expected = Inspection(); + Inspect(highlighter.tokenize(code, "bsl"), expected); + auto result = Inspection(); + Inspect(SyntaxHighlighter(grammars).tokenize(code, "bsl"), result); + if (result.tokens != expected.tokens) + { + std::cerr << "BSL tokens differ under " << argv[2] << '\n'; + ++failed; + } + + std::locale::global(std::locale::classic()); + } + std::cout << failed << " Unicode highlighting checks failed\n"; return failed ? 1 : 0; }