From 2b15649f93945469f563fee78967c7cbe3628e19 Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Tue, 11 Aug 2026 20:22:49 -0400 Subject: [PATCH] fix(generate): fold case-insensitive patterns at the AST leaves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A previous fix moved case folding for `/i` patterns out of `regex_syntax` and into `expand_regex`, so folding could drop the two non-ASCII code points Unicode simple folding maps onto ASCII letters: the long s `ſ` (U+017F) onto `s`, and the Kelvin sign `K` (U+212A) onto `k`. Left in, they leak into otherwise-ASCII tokens and stop those tokens from being extracted as keywords. By that point, though, the HIR has already turned a negated class into a complement, so folding it applies the fold on the wrong side of the negation. `(?i)[^a-z]` folds a set that contains `A-Z`, which re-admits `a-z` and leaves a class matching very nearly everything. `regex_syntax` folds each leaf of a class expression before applying that leaf's negation and the set algebra above it. Keep that order and change only the fold: walk the AST, replace each leaf with its fold, and translate with `case_insensitive(false)`. --- crates/generate/src/nfa.rs | 6 +- crates/generate/src/prepare_grammar.rs | 1 + .../src/prepare_grammar/expand_tokens.rs | 183 ++++++---- .../generate/src/prepare_grammar/pattern.rs | 335 ++++++++++++++++++ 4 files changed, 461 insertions(+), 64 deletions(-) create mode 100644 crates/generate/src/prepare_grammar/pattern.rs diff --git a/crates/generate/src/nfa.rs b/crates/generate/src/nfa.rs index e563d626b..7101639de 100644 --- a/crates/generate/src/nfa.rs +++ b/crates/generate/src/nfa.rs @@ -294,7 +294,11 @@ impl CharacterSet { /// Produces a `CharacterSet` containing every character in `self` that is not present in /// `other`. - #[allow(clippy::must_use_candidate, clippy::return_self_not_must_use)] + #[allow( + clippy::must_use_candidate, + clippy::return_self_not_must_use, + dead_code + )] pub fn difference(mut self, mut other: Self) -> Self { self.remove_intersection(&mut other); self diff --git a/crates/generate/src/prepare_grammar.rs b/crates/generate/src/prepare_grammar.rs index 61d22ada8..663f44df5 100644 --- a/crates/generate/src/prepare_grammar.rs +++ b/crates/generate/src/prepare_grammar.rs @@ -4,6 +4,7 @@ mod extract_default_aliases; mod extract_tokens; mod flatten_grammar; mod intern_symbols; +mod pattern; mod process_inlines; use std::{ diff --git a/crates/generate/src/prepare_grammar/expand_tokens.rs b/crates/generate/src/prepare_grammar/expand_tokens.rs index 35635701a..5e9e9ddec 100644 --- a/crates/generate/src/prepare_grammar/expand_tokens.rs +++ b/crates/generate/src/prepare_grammar/expand_tokens.rs @@ -1,21 +1,17 @@ -use regex_syntax::{ - ParserBuilder, - hir::{Class, ClassUnicode, ClassUnicodeRange, Hir, HirKind}, -}; +use regex_syntax::hir::{Class, Hir, HirKind}; use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ grammars::{LexicalGrammar, LexicalVariable}, nfa::{CharacterSet, Nfa, NfaState}, - prepare_grammar::extract_tokens::LexicalToken, + prepare_grammar::{extract_tokens::LexicalToken, pattern}, rules::{Precedence, Rule, RuleId, RulePool, Symbol}, }; struct NfaBuilder { nfa: Nfa, is_sep: bool, - case_insensitive: bool, precedence_stack: Vec, } @@ -82,7 +78,6 @@ pub fn expand_tokens( let mut builder = NfaBuilder { nfa: Nfa::new(), is_sep: true, - case_insensitive: false, precedence_stack: vec![0], }; let separator_root = build_separator(pool, separator_roots); @@ -183,38 +178,6 @@ pub enum ExpandRegexError { Assertion, } -/// Case-fold `base` for the `i` flag, keeping folding ASCII-safe. -/// -/// `regex_syntax`'s own case folding is Unicode simple folding, which maps two -/// non-ASCII code points onto ASCII letters: -/// - the long s `ſ` (U+017F) onto `s` -/// - the Kelvin sign `K` (U+212A) onto `k` -/// -/// This leaks non-ASCII code points into otherwise-ASCII tokens, which is virtually -/// never intended and also stops such tokens from being extracted as keywords. So -/// we parse patterns unfolded and fold here instead: fold via `regex_syntax`, then -/// drop those two code points *only when folding introduced them*. A set that already -/// contained `ſ`/`K` (an explicit literal, or a broad class like `[^"]` or `\p{L}`) -/// keeps them. -fn case_fold_ascii_safe(base: &CharacterSet) -> CharacterSet { - let mut class = ClassUnicode::new( - base.ranges() - .map(|r| ClassUnicodeRange::new(*r.start(), *r.end())), - ); - class.case_fold_simple(); - - let mut folded = CharacterSet::empty(); - for r in class.ranges() { - folded = folded.add_range(r.start(), r.end()); - } - for exotic in ['\u{017f}', '\u{212a}'] { - if folded.contains(exotic) && !base.contains(exotic) { - folded = folded.difference(CharacterSet::from_char(exotic)); - } - } - folded -} - impl NfaBuilder { fn expand_rule( &mut self, @@ -240,15 +203,8 @@ impl NfaBuilder { // Parse WITHOUT case folding and fold ourselves (see // `case_fold_ascii_safe`). Letting `regex_syntax` fold would pull the // long s `ſ` and Kelvin sign `K` into ASCII `s`/`k`. - let mut parser = ParserBuilder::new() - .case_insensitive(false) - .unicode(true) - .utf8(false) - .build(); - let hir = parser - .parse(&s) + let hir = pattern::parse(&s, pool.resolve(f).contains('i')) .map_err(|e| ExpandRuleError::Parse(e.to_string()))?; - self.case_insensitive = pool.resolve(f).contains('i'); self.expand_regex(&hir, next_state_id) .map_err(ExpandRuleError::ExpandRegex) } @@ -340,13 +296,7 @@ impl NfaBuilder { .chars() .rev() { - let char_set = CharacterSet::from_char(character); - let char_set = if self.case_insensitive { - case_fold_ascii_safe(&char_set) - } else { - char_set - }; - self.push_advance(char_set, next_state_id); + self.push_advance(CharacterSet::from_char(character), next_state_id); next_state_id = self.nfa.last_state_id(); } @@ -358,21 +308,13 @@ impl NfaBuilder { for c in class.ranges() { chars = chars.add_range(c.start(), c.end()); } - if self.case_insensitive { - chars = case_fold_ascii_safe(&chars); - } self.push_advance(chars, next_state_id); Ok(true) } Class::Bytes(bytes_class) => { // Byte classes only come from non-Unicode `(?-u:...)` groups, which // JS regex syntax can't express, so this is currently unreachable - // from grammar patterns. Byte folding is ASCII-only (bytes are <= 0xFF) - // so it can never introduce `ſ`/`K` and needs no special handling. - let mut bytes_class = bytes_class.clone(); - if self.case_insensitive { - bytes_class.case_fold_simple(); - } + // from grammar patterns. let mut chars = CharacterSet::default(); for c in bytes_class.ranges() { chars = chars.add_range(c.start().into(), c.end().into()); @@ -974,6 +916,121 @@ mod tests { ("\"", None), // the one excluded character ], ); + // `ſ`/`K` fold with nothing, so an explicit one pulls in no ASCII letter. + check( + |p| { + let (v, f) = (p.intern("[\u{017f}\u{212a}]+"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("\u{017f}\u{212a}.", Some((0, "\u{017f}\u{212a}"))), + ("s", None), + ("k", None), + ], + ); + // `\p{L}` already contains `ſ`/`K`, so folding adds nothing to strip. + check( + |p| { + let (v, f) = (p.intern(r"\p{L}+"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("\u{017f}", Some((0, "\u{017f}"))), + ("\u{212a}", Some((0, "\u{212a}"))), + ("aA", Some((0, "aA"))), + ("1", None), + ], + ); + // Folding happens at the leaves, before negation: `[^a-z]` under `/i` + // must exclude `A-Z`, not re-admit it. + check( + |p| { + let (v, f) = (p.intern("[^a-z]"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("!", Some((0, "!"))), + ("a", None), + ("A", None), + ("\u{017f}", Some((0, "\u{017f}"))), + ("\u{212a}", Some((0, "\u{212a}"))), + ], + ); + // The same ordering through a nested class and through class-set algebra. + check( + |p| { + let (v, f) = (p.intern("[^[a-c]]"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[("d", Some((0, "d"))), ("a", None), ("A", None), ("C", None)], + ); + check( + |p| { + let (v, f) = (p.intern("[[a-z]--[b-d]]"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("a", Some((0, "a"))), + ("A", Some((0, "A"))), + ("b", None), + ("B", None), + ], + ); + check( + |p| { + let (v, f) = (p.intern("[[a-z]&&[b-d]]"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("b", Some((0, "b"))), + ("B", Some((0, "B"))), + ("e", None), + ("E", None), + ], + ); + // Scoped flags apply only where they are in effect. + check( + |p| { + let (v, f) = (p.intern("(?i)a(?-i)b"), p.intern("")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("ab", Some((0, "ab"))), + ("Ab", Some((0, "Ab"))), + ("aB", None), + ], + ); + // Case-insensitivity does not leak from one token to the next. + check( + |p| { + let (v1, f1) = (p.intern("ab"), p.intern("i")); + let (v2, f2) = (p.intern("cd"), p.intern("")); + let s = p.intern("ef"); + ( + vec![p.pattern(v1, f1), p.pattern(v2, f2), p.string(s)], + vec![], + ) + }, + &[ + ("AB", Some((0, "AB"))), + ("cd", Some((1, "cd"))), + ("CD", None), + ("ef", Some((2, "ef"))), + ("EF", None), + ], + ); + check( + |p| { + let (v, f) = (p.intern(r"\$\{[a-z0-9_\.]*[^a-z0-9_\.\}]"), p.intern("i")); + (vec![p.pattern(v, f)], vec![]) + }, + &[ + ("${a}", None), + ("${A}", None), + ("${a!", Some((0, "${a!"))), + ("${!", Some((0, "${!"))), + ], + ); // Emojis check( |p| { diff --git a/crates/generate/src/prepare_grammar/pattern.rs b/crates/generate/src/prepare_grammar/pattern.rs new file mode 100644 index 000000000..f906ff18e --- /dev/null +++ b/crates/generate/src/prepare_grammar/pattern.rs @@ -0,0 +1,335 @@ +//! Parsing of token patterns. +//! +//! Unicode simple case folding maps two non-ASCII code points onto ASCII letters: +//! - the long s `ſ` onto `s` +//! - the Kelvin sign `K` onto `k` +//! +//! Taken as is, every case-insensitive ASCII token picks both up, which is virtually +//! never intended and stops the token from being extracted as a keyword. To +//! fix this, we expand the `i` flag ourselves. +//! +//! The [`regex_syntax`] crate parses a pattern into an [`ast::Ast`], then lowers +//! it to an [`hir::Hir`] via a [`hir::translate::Translator`]. The translator is +//! what applies `i`, and it folds each leaf of a class expression before applying +//! that leaf's negation and the set algebra above it. This order is what makes +//! `(?i)[^x]` exclude `X` rather than re-admit it, so we keep that order and only +//! change the fold: +//! - Walk the AST +//! - Fold each leaf as if `ſ` and `K` were their own equivalence classes +//! - Hand the result to the translator with its own folding turned off +//! +//! Folding at the leaves is sufficient for correctness, as case folding sorts every +//! character into a group of case variants (`{a, A}`, `{s, S, ſ}`, etc.), with +//! each character in exactly one group. Folding a set adds (for each character +//! in it) the rest of that character's group, so the result is always built out +//! of whole groups. Negation and set algebra preserve that, because they treat +//! every character in a group alike: a group ends up either wholly inside or wholly +//! outside the result. Folding a set that is already whole groups adds nothing, +//! so once leaves are folded, every fold the translator would still apply does +//! nothing and it compiles exactly what is would have with our fold in place of +//! its own. + +use regex_syntax::{ + ast::{ + self, Ast, ClassBracketed, ClassSet, ClassSetItem, ClassSetRange, ClassSetUnion, Flag, + FlagsItem, FlagsItemKind, GroupKind, Span, parse::ParserBuilder, + }, + hir::{ + self, Class, ClassUnicode, ClassUnicodeRange, Hir, HirKind, + translate::{Translator, TranslatorBuilder}, + }, +}; + +/// The flags that decide whether we fold a scope's leaves. +#[derive(Clone, Copy)] +struct FoldMode { + case_insensitive: bool, + unicode: bool, +} + +impl FoldMode { + /// Whether we fold this scope's leaves ourselves. + /// + /// Inside `(?-u:...)` the translator works on bytes and folds ASCII only, which + /// can never pull in `ſ` or `K`, so those scopes keep its folding. + const fn folds_here(self) -> bool { + self.case_insensitive && self.unicode + } +} + +/// The state of the walk: The flags in effect, and one translator to resolve +/// `\p{...}` and friends without rebuilding it per leaf. +struct Expander<'a> { + translator: Translator, + pattern: &'a str, + mode: FoldMode, +} + +/// Parse a token pattern into an [`Hir`], folding any `i` flag manually. +pub(super) fn parse( + pattern: &str, + case_insensitive: bool, +) -> Result> { + let mut ast = ParserBuilder::new() + .build() + .parse(pattern) + .map_err(|e| Box::new(e.into()))?; + + let mut expander = Expander { + translator: TranslatorBuilder::new() + .case_insensitive(false) + .unicode(true) + .utf8(false) + .build(), + pattern, + mode: FoldMode { + case_insensitive, + unicode: true, + }, + }; + expander.expand(&mut ast).map_err(|e| Box::new(e.into()))?; + expander + .translator + .translate(pattern, &ast) + .map_err(|e| Box::new(e.into())) +} + +impl Expander<'_> { + fn expand(&mut self, ast: &mut Ast) -> Result<(), hir::Error> { + match ast { + Ast::Flags(f) => self.set_flags(&mut f.flags), + Ast::Repetition(r) => self.expand(&mut r.ast)?, + Ast::Group(g) => { + let outer = self.mode; + if let GroupKind::NonCapturing(flags) = &mut g.kind { + self.set_flags(flags); + } + self.expand(&mut g.ast)?; + self.mode = outer; + } + // Flags set in one branch carry into the next, as they do in the translator, + // which scopes flags to groups but not to these. + Ast::Alternation(a) => { + for branch in &mut a.asts { + self.expand(branch)?; + } + } + Ast::Concat(c) => { + for element in &mut c.asts { + self.expand(element)?; + } + } + Ast::ClassBracketed(b) if self.mode.folds_here() => { + self.expand_class_set(&mut b.kind)?; + } + // The remaining class-valued nodes are the same leaves as a bracket holds, + // so they fold through the same path. + _ if self.mode.folds_here() => { + let mut item = match ast { + Ast::Literal(l) => ClassSetItem::Literal((**l).clone()), + Ast::ClassUnicode(u) => ClassSetItem::Unicode((**u).clone()), + Ast::ClassPerl(p) => ClassSetItem::Perl((**p).clone()), + // `.` is fold-closed, nothing else matches a character + _ => return Ok(()), + }; + self.expand_class_item(&mut item)?; + if let ClassSetItem::Bracketed(class) = item { + *ast = Ast::ClassBracketed(class); + } + } + // Left to the translator: either no `i` is in effect, or this is a + // `(?-u:...)` scope, where its own folding is ASCII-only and cannot + // pull in `ſ` or `K`. + #[rustfmt::skip] + Ast::Empty(_) | Ast::Dot(_) | Ast::Assertion(_) | Ast::Literal(_) + | Ast::ClassUnicode(_) | Ast::ClassPerl(_) | Ast::ClassBracketed(_) => {} + } + Ok(()) + } + + /// Apply a flag directive to the mode, then rewrite the directive to say what + /// the translator should still do about `i` in the scope it opens. + /// + /// `i` comes out everywhere, since folding a leaf we already folded would put + /// `ſ`/`K` back in, and goes back in only when we skipped the folding outselves + /// (inside `(?-u:...)`. + /// + /// `i` comes out everywhere, since folding a leaf we already folded would put + /// `ſ`/`K` straight back. It goes back in wherever [`FoldMode::folds_here`] + /// says we left the folding alone, which would otherwise lose case-insensitivity + /// in that scope entirely. + fn set_flags(&mut self, flags: &mut ast::Flags) { + let mut negated = false; + for item in &flags.items { + match item.kind { + FlagsItemKind::Negation => negated = true, + FlagsItemKind::Flag(Flag::CaseInsensitive) => self.mode.case_insensitive = !negated, + FlagsItemKind::Flag(Flag::Unicode) => self.mode.unicode = !negated, + FlagsItemKind::Flag(_) => {} + } + } + let item = |kind| FlagsItem { + span: flags.span, + kind, + }; + flags + .items + .retain(|it| !matches!(it.kind, FlagsItemKind::Flag(Flag::CaseInsensitive))); + if self.mode.folds_here() { + flags.items.push(item(FlagsItemKind::Negation)); + flags + .items + .push(item(FlagsItemKind::Flag(Flag::CaseInsensitive))); + } else if self.mode.case_insensitive { + flags + .items + .insert(0, item(FlagsItemKind::Flag(Flag::CaseInsensitive))); + } + } + + /// Fold the leaves of a class expression, leaving its structure alone. The + /// translator still performs the negations and the `&&`/`--`/`~~` itself, on + /// operands that are already folded. + fn expand_class_set(&mut self, set: &mut ClassSet) -> Result<(), hir::Error> { + match set { + ClassSet::Item(item) => self.expand_class_item(item), + ClassSet::BinaryOp(op) => { + self.expand_class_set(&mut op.lhs)?; + self.expand_class_set(&mut op.rhs) + } + } + } + + fn expand_class_item(&mut self, item: &mut ClassSetItem) -> Result<(), hir::Error> { + // Whether the leaf carries its own negation (`\P{L}, `[:^alpha:]`, `\D`, etc.). + let (span, negated) = match item { + ClassSetItem::Bracketed(b) => return self.expand_class_set(&mut b.kind), + ClassSetItem::Union(u) => { + return u + .items + .iter_mut() + .try_for_each(|i| self.expand_class_item(i)); + } + ClassSetItem::Empty(_) => return Ok(()), + ClassSetItem::Literal(l) => (l.span, false), + ClassSetItem::Range(r) => (r.span, false), + ClassSetItem::Ascii(a) => (a.span, a.negated), + ClassSetItem::Perl(p) => (p.span, p.negated), + ClassSetItem::Unicode(u) => (u.span, u.is_negated()), + }; + + // `leaf_set` applies the leaf's own negation, so undo it to recover the + // operand the translator would have folded, and hand the negation back + // to the replacement so it is reapplied after the fold. + let mut operand = self.leaf_set(item)?; + if negated { + operand.negate(); + } + + let mut folded = operand.clone(); + Self::fold_ascii_safe(&mut folded); + // A leaf that is already fold-closed stays as written, which keeps most + // unicode properties out of the expansion. + if folded.ranges() != operand.ranges() { + *item = ClassSetItem::Bracketed(Box::new(ClassBracketed { + span, + negated, + kind: ClassSet::Item(Self::class_to_item(span, &folded)), + })); + } + + Ok(()) + } + + /// The set a leaf denotes, with any negation of its own already applied. + /// + /// Literals and ranges are read directly from the AST. Everything else goes + /// back through the translator, which own the property and POSIX tables. + fn leaf_set(&mut self, item: &ClassSetItem) -> Result { + let range = match item { + ClassSetItem::Literal(l) => ClassUnicodeRange::new(l.c, l.c), + ClassSetItem::Range(r) => ClassUnicodeRange::new(r.start.c, r.end.c), + _ => { + let ast = Ast::ClassBracketed(Box::new(ClassBracketed { + span: *item.span(), + negated: false, + kind: ClassSet::Item(item.clone()), + })); + return Ok(Self::class_of( + self.translator.translate(self.pattern, &ast)?, + )); + } + }; + Ok(ClassUnicode::new([range])) + } + + /// The class a single-class [`Hir`] denotes. + /// + /// We only ever translate one [`Ast::ClassBracketed`], so [`Hir::class`] built + /// the result: A class, or one of the two shapes it collapses to. The structural + /// kinds (`Empty`, `Look`, `Repetition`, `Capture`, `Concat`, `Alternation`) + /// need AST nodes we never construct in [`Self::leaf_set`]. + fn class_of(hir: Hir) -> ClassUnicode { + match hir.into_kind() { + HirKind::Class(Class::Unicode(class)) => class, + HirKind::Literal(literal) => { + // A one character class, translated in unicode mode, so this must + // be that character's UTF-8. + let literal = str::from_utf8(&literal.0).unwrap(); + ClassUnicode::new(literal.chars().map(|c| ClassUnicodeRange::new(c, c))) + } + // An empty class collapses to `Hir::fail` (an empty byte class). + _ => ClassUnicode::empty(), + } + } + + /// Re-encode a class of the AST nodes the tranlator will read back, the inverse + /// of [`Self::leaf_set`]. + /// + /// Every node reuses the leaf's own span to preserve error spans. The literal + /// kind is irrelevant, in unicode mode the translator only reads `c`. + fn class_to_item(span: Span, class: &ClassUnicode) -> ClassSetItem { + let literal = |c| ast::Literal { + span, + kind: ast::LiteralKind::Verbatim, + c, + }; + ClassSetUnion { + span, + items: class + .ranges() + .iter() + .map(|r| { + ClassSetItem::Range(ClassSetRange { + span, + start: literal(r.start()), + end: literal(r.end()), + }) + }) + .collect(), + } + .into_item() + } + + /// Fold `class` in place, with `ſ` and `K` each in a group of their own rather + /// than the the `s`/`S` and `k`/`K` groups. + /// + /// Dropping them before the fold stops a `ſ` in the class from pulling in `s`/`S`. + /// Dropping them after removes the ones folding `s`/`S` introduced. The union + /// restores any the class genuinely held. + fn fold_ascii_safe(class: &mut ClassUnicode) { + // The code points that Unicode simple case folding maps onto ASCII letters: + // the long s `ſ` folds with `s`/`S`, and the Kelvin sign `K` with `k`/`K`. + const NON_ASCII_FOLDS: [char; 2] = ['\u{17f}', '\u{212a}']; + let exotic = ClassUnicode::new(NON_ASCII_FOLDS.map(|c| ClassUnicodeRange::new(c, c))); + + // The ones the pattern asked for itself, which folding must not drop + let mut asked_for = exotic.clone(); + asked_for.intersect(class); + + class.difference(&exotic); + class.case_fold_simple(); + class.difference(&exotic); + class.union(&asked_for); + } +}