diff --git a/crates/generate/src/prepare_grammar.rs b/crates/generate/src/prepare_grammar.rs index e940b5730..304f8edc6 100644 --- a/crates/generate/src/prepare_grammar.rs +++ b/crates/generate/src/prepare_grammar.rs @@ -14,9 +14,9 @@ use std::{ }; pub use expand_tokens::ExpandTokensError; +#[cfg(test)] +pub use expand_tokens::expand_tokens; pub use extract_tokens::ExtractTokensError; -#[cfg(test)] // TODO: Is this defined in the proper place? -pub use extract_tokens::LexicalToken; pub use flatten_grammar::FlattenGrammarError; use indexmap::IndexMap; pub use intern_symbols::InternSymbolsError; @@ -26,11 +26,10 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ - grammars::{InputGrammar, PrecedenceEntry, ProductionStore}, + grammars::{InputGrammar, PrecedenceEntry, ProductionStore, VariableType}, strpool::StrPool, }; -pub use self::expand_tokens::expand_tokens; use self::{ expand_repeats::{ExpandRepeatsError, expand_repeats}, extract_default_aliases::extract_default_aliases, @@ -43,7 +42,7 @@ use super::{ Diagnostic, grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar}, prepare_grammar::flatten_grammar::{FlattenState, assemble_syntax_grammar}, - rules::{AliasMap, Precedence, Rule}, + rules::{AliasMap, Precedence, Rule, RuleId}, strpool::StrId, }; @@ -109,6 +108,20 @@ pub struct PreparedGrammar { pub str_pool: StrPool, } +/// A token extracted from the input grammar but not yet expanded into the lexical NFA. +/// +/// Token extraction creates this while `root` still points to the original rule in +/// the pool. [`PendingTokenExtraction::expand_and_commit`] passes them to `expand_tokens` +/// before committing the deferred syntax rewrites. +#[derive(Clone, Debug)] +pub struct LexicalToken { + /// Generated for anon tokens, rule name for absorbed variables + pub name: StrId, + pub kind: VariableType, + /// Pool root defining this token + pub root: RuleId, +} + /// Transform an input grammar into separate components that are ready /// for parse table construction. pub fn prepare_grammar( @@ -119,21 +132,17 @@ pub fn prepare_grammar( validate_indirect_recursion(&g)?; let interned_meta = intern_symbols(&mut g, diagnostics)?; - let mut ext_meta = extract_tokens(&mut g, &interned_meta)?; + let pending_tokens = extract_tokens(&mut g, &interned_meta)?; + let (mut ext_meta, lexical_grammar) = pending_tokens.expand_and_commit()?; expand_repeats(&mut g, &mut ext_meta)?; let mut state = FlattenState::default(); let mut out = ProductionStore::default(); flatten_grammar(&g, &ext_meta, &mut state, &mut out)?; - let lexical_grammar = expand_tokens( - &mut g.pool, - &ext_meta.lexical_variables, - &ext_meta.separator_roots, - )?; - - let default_aliases = extract_default_aliases(&g, &ext_meta, &mut out); - let inlines = process_inlines(&g, &ext_meta, &mut out)?; + let default_aliases = + extract_default_aliases(&g, &ext_meta, &lexical_grammar.variables, &mut out); + let inlines = process_inlines(&g, &ext_meta, &lexical_grammar.variables, &mut out)?; let (syntax_grammar, str_pool) = assemble_syntax_grammar(g, ext_meta, out); Ok(PreparedGrammar { @@ -308,8 +317,8 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()> mod tests { use super::*; use crate::{ - grammars::Variable, - rules::{RuleId, RulePool}, + grammars::{ProductionStep, Variable, VariableType}, + rules::{RuleId, RulePool, Symbol}, }; #[test] @@ -462,6 +471,352 @@ mod tests { } } + #[test] + fn test_token_body_shared_with_syntax() { + let grammar = build_grammar(|p| { + let a = leaf(p, "a"); + let b = leaf(p, "b"); + let shared = p.seq(&[a, b]); + let token = p.token(shared); + let program = { + let t = named(p, "t"); + let x = named(p, "x"); + p.seq(&[t, x]) + }; + + vec![ + Variable { + name: p.intern("program"), + root: program, + }, + Variable { + name: p.intern("t"), + root: token, + }, + Variable { + name: p.intern("x"), + root: shared, + }, + ] + }); + + let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap(); + + let syntax_variables = prepared + .syntax_grammar + .variables + .iter() + .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind)) + .collect::>(); + assert_eq!( + syntax_variables, + [("program", VariableType::Named), ("x", VariableType::Named),] + ); + + let lexical_variables = prepared + .lexical_grammar + .variables + .iter() + .map(|variable| { + ( + prepared.str_pool.resolve(variable.name), + variable.kind, + variable.implicit_precedence, + variable.start_state, + ) + }) + .collect::>(); + assert_eq!( + lexical_variables, + [ + ("t", VariableType::Named, 0, 2), + ("a", VariableType::Anonymous, 2, 4), + ("b", VariableType::Anonymous, 2, 6), + ] + ); + } + + #[test] + fn test_token_body_shared_with_syntax_reversed() { + let grammar = build_grammar(|p| { + let a = leaf(p, "a"); + let b = leaf(p, "b"); + let shared = p.seq(&[a, b]); + let token = p.token(shared); + let program = { + let x = named(p, "x"); + let t = named(p, "t"); + p.seq(&[x, t]) + }; + + vec![ + Variable { + name: p.intern("program"), + root: program, + }, + Variable { + name: p.intern("x"), + root: shared, + }, + Variable { + name: p.intern("t"), + root: token, + }, + ] + }); + + let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap(); + + let syntax_variables = prepared + .syntax_grammar + .variables + .iter() + .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind)) + .collect::>(); + assert_eq!( + syntax_variables, + [("program", VariableType::Named), ("x", VariableType::Named),] + ); + + let lexical_variables = prepared + .lexical_grammar + .variables + .iter() + .map(|variable| { + ( + prepared.str_pool.resolve(variable.name), + variable.kind, + variable.implicit_precedence, + variable.start_state, + ) + }) + .collect::>(); + assert_eq!( + lexical_variables, + [ + ("a", VariableType::Anonymous, 2, 1), + ("b", VariableType::Anonymous, 2, 3), + ("t", VariableType::Named, 0, 6), + ] + ); + } + + #[test] + fn test_separator_body_shared_with_syntax() { + let mut pool = RulePool::default(); + let a = leaf(&mut pool, "a"); + let b = leaf(&mut pool, "b"); + let shared = pool.seq(&[a, b]); + let program = pool.intern("program"); + + let grammar = InputGrammar { + variables: vec![Variable { + name: program, + root: shared, + }], + extra_roots: vec![shared], + pool, + ..Default::default() + }; + + let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap(); + + let syntax_variables = prepared + .syntax_grammar + .variables + .iter() + .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind)) + .collect::>(); + assert_eq!(syntax_variables, [("program", VariableType::Named)]); + + let lexical_variables = prepared + .lexical_grammar + .variables + .iter() + .map(|variable| { + ( + prepared.str_pool.resolve(variable.name), + variable.kind, + variable.implicit_precedence, + variable.start_state, + ) + }) + .collect::>(); + assert_eq!( + lexical_variables, + [ + ("a", VariableType::Anonymous, 2, 5), + ("b", VariableType::Anonymous, 2, 11), + ] + ); + + assert!(prepared.syntax_grammar.extra_symbols.is_empty()); + } + + #[test] + fn test_shared_extra_symbol_is_renumbered_once() { + let mut pool = RulePool::default(); + + let target_ref = named(&mut pool, "target"); + let program = { + let kw = named(&mut pool, "kw"); + let keep = named(&mut pool, "keep"); + pool.seq(&[kw, keep, target_ref]) + }; + let kw = leaf(&mut pool, "keyword"); + let keep = { + let k = leaf(&mut pool, "k"); + let e = leaf(&mut pool, "e"); + pool.seq(&[k, e]) + }; + let target = { + let t = leaf(&mut pool, "t"); + let g = leaf(&mut pool, "g"); + pool.seq(&[t, g]) + }; + + let grammar = InputGrammar { + variables: vec![ + Variable { + name: pool.intern("program"), + root: program, + }, + Variable { + name: pool.intern("kw"), + root: kw, + }, + Variable { + name: pool.intern("keep"), + root: keep, + }, + Variable { + name: pool.intern("target"), + root: target, + }, + ], + extra_roots: vec![target_ref], + pool, + ..Default::default() + }; + + let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap(); + + let syntax_variables = prepared + .syntax_grammar + .variables + .iter() + .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind)) + .collect::>(); + assert_eq!( + syntax_variables, + [ + ("program", VariableType::Named), + ("keep", VariableType::Named), + ("target", VariableType::Named), + ] + ); + + let lexical_variables = prepared + .lexical_grammar + .variables + .iter() + .map(|variable| { + ( + prepared.str_pool.resolve(variable.name), + variable.kind, + variable.implicit_precedence, + variable.start_state, + ) + }) + .collect::>(); + assert_eq!( + lexical_variables, + [ + ("kw", VariableType::Named, 2, 7), + ("k", VariableType::Anonymous, 2, 9), + ("e", VariableType::Anonymous, 2, 11), + ("t", VariableType::Anonymous, 2, 13), + ("g", VariableType::Anonymous, 2, 15), + ] + ); + + assert_eq!( + prepared.syntax_grammar.extra_symbols, + [Symbol::non_terminal(2)] + ); + } + + #[test] + fn test_shared_syntax_subtree_is_renumbered_once() { + let grammar = build_grammar(|p| { + let pair = { + let thing = named(p, "thing"); + let dash = leaf(p, "-"); + p.seq(&[thing, dash]) + }; + let item_b = { + let semicolon = leaf(p, ";"); + p.seq(&[pair, semicolon]) + }; + let program = { + let kw = named(p, "kw"); + let item_a = named(p, "item_a"); + let item_b = named(p, "item_b"); + p.seq(&[kw, item_a, item_b]) + }; + let kw = leaf(p, "keyword"); + let thing = { + let t = leaf(p, "t"); + let u = leaf(p, "u"); + p.seq(&[t, u]) + }; + + vec![ + Variable { + name: p.intern("program"), + root: program, + }, + Variable { + name: p.intern("kw"), + root: kw, + }, + Variable { + name: p.intern("item_a"), + root: pair, + }, + Variable { + name: p.intern("item_b"), + root: item_b, + }, + Variable { + name: p.intern("thing"), + root: thing, + }, + ] + }); + + let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap(); + + let expected_steps = [ + ProductionStep::pack( + Symbol::non_terminal(3), + Precedence::None, + None, + None, + None, + 0, + ), + ProductionStep::pack(Symbol::terminal(1), Precedence::None, None, None, None, 0), + ]; + + let production_ids = prepared.syntax_grammar.variable_prod_ids(1); + assert_eq!(production_ids.len(), 1); + + let production = prepared.syntax_grammar.production(production_ids.start); + assert_eq!(production.steps, expected_steps); + assert_eq!(production.dynamic_precedence, 0); + } + fn named(pool: &mut RulePool, name: &str) -> RuleId { let id = pool.intern(name); pool.named_symbol(id) diff --git a/crates/generate/src/prepare_grammar/expand_tokens.rs b/crates/generate/src/prepare_grammar/expand_tokens.rs index 32444a3c0..a1eafc7e2 100644 --- a/crates/generate/src/prepare_grammar/expand_tokens.rs +++ b/crates/generate/src/prepare_grammar/expand_tokens.rs @@ -5,7 +5,7 @@ use thiserror::Error; use crate::{ grammars::{LexicalGrammar, LexicalVariable}, nfa::{CharacterSet, Nfa, NfaState}, - prepare_grammar::{extract_tokens::LexicalToken, pattern}, + prepare_grammar::{LexicalToken, pattern}, rules::{Precedence, Rule, RuleId, RulePool, Symbol}, }; diff --git a/crates/generate/src/prepare_grammar/extract_default_aliases.rs b/crates/generate/src/prepare_grammar/extract_default_aliases.rs index c76ee7003..f706476cd 100644 --- a/crates/generate/src/prepare_grammar/extract_default_aliases.rs +++ b/crates/generate/src/prepare_grammar/extract_default_aliases.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use crate::{ - grammars::{InputGrammar, ProductionStore}, + grammars::{InputGrammar, LexicalVariable, ProductionStore}, prepare_grammar::extract_tokens::ExtractedGrammarMeta, rules::{Alias, AliasMap, Symbol, SymbolType}, }; @@ -24,9 +24,10 @@ struct SymbolStatus { pub(super) fn extract_default_aliases( g: &InputGrammar, meta: &ExtractedGrammarMeta, + lexical_variables: &[LexicalVariable], out: &mut ProductionStore, ) -> AliasMap { - let mut terminal_status_list = vec![SymbolStatus::default(); meta.lexical_variables.len()]; + let mut terminal_status_list = vec![SymbolStatus::default(); lexical_variables.len()]; let mut non_terminal_status_list = vec![SymbolStatus::default(); g.variables.len()]; let mut external_status_list = vec![SymbolStatus::default(); meta.external_tokens.len()]; @@ -171,7 +172,6 @@ mod tests { use super::*; use crate::{ grammars::{Production, ProductionStep, Variable, VariableType}, - prepare_grammar::extract_tokens::LexicalToken, rules::{Precedence, RulePool}, }; @@ -181,20 +181,26 @@ mod tests { let dummy = pool.intern("_"); let root = pool.blank(); + let mut lexical_variables = Vec::new(); + let t0 = add_lexical_variable(&mut pool, &mut lexical_variables, "t0"); + let t1 = add_lexical_variable(&mut pool, &mut lexical_variables, "t1"); + let t2 = add_lexical_variable(&mut pool, &mut lexical_variables, "t2"); + let t3 = add_lexical_variable(&mut pool, &mut lexical_variables, "t3"); + // v1: every token aliased. let v1 = vec![ - aliased(&mut pool, Symbol::terminal(0), "a1"), - aliased(&mut pool, Symbol::terminal(1), "a2"), - aliased(&mut pool, Symbol::terminal(2), "a3"), - aliased(&mut pool, Symbol::terminal(3), "a4"), + aliased(&mut pool, t0, "a1"), + aliased(&mut pool, t1, "a2"), + aliased(&mut pool, t2, "a3"), + aliased(&mut pool, t3, "a4"), ]; // v2: t0 same alias, t1 unaliased, t2 aliased differently, t3 aliased twice as a6 let v2 = vec![ - aliased(&mut pool, Symbol::terminal(0), "a1"), - plain(Symbol::terminal(1)), - aliased(&mut pool, Symbol::terminal(2), "a5"), - aliased(&mut pool, Symbol::terminal(3), "a6"), - aliased(&mut pool, Symbol::terminal(3), "a6"), + aliased(&mut pool, t0, "a1"), + plain(t1), + aliased(&mut pool, t2, "a5"), + aliased(&mut pool, t3, "a6"), + aliased(&mut pool, t3, "a6"), ]; let mut out = ProductionStore::default(); @@ -216,18 +222,8 @@ mod tests { pool, ..Default::default() }; - let meta = ExtractedGrammarMeta { - lexical_variables: (0..4) - .map(|_| LexicalToken { - name: dummy, - kind: VariableType::Anonymous, - root, - }) - .collect(), - ..Default::default() - }; - - let pool_map = extract_default_aliases(&g, &meta, &mut out); + let meta = ExtractedGrammarMeta::default(); + let pool_map = extract_default_aliases(&g, &meta, &lexical_variables, &mut out); // t0 -> a1, t2 -> a3 (v1 wins the tie from appearing first), t3 -> a6 (used twice) // t1 -> none (appears unaliased in v2). @@ -259,6 +255,21 @@ mod tests { assert_eq!(step_alias(8), None); // v2 t3(a6) = default } + fn add_lexical_variable( + pool: &mut RulePool, + variables: &mut Vec, + name: &str, + ) -> Symbol { + let symbol = Symbol::terminal(variables.len()); + variables.push(LexicalVariable { + name: pool.intern(name), + kind: VariableType::Anonymous, + implicit_precedence: 0, + start_state: 0, + }); + symbol + } + fn aliased(pool: &mut RulePool, symbol: Symbol, name: &str) -> ProductionStep { let value = pool.intern(name); ProductionStep::pack( diff --git a/crates/generate/src/prepare_grammar/extract_tokens.rs b/crates/generate/src/prepare_grammar/extract_tokens.rs index 2c13af5e3..9375d1999 100644 --- a/crates/generate/src/prepare_grammar/extract_tokens.rs +++ b/crates/generate/src/prepare_grammar/extract_tokens.rs @@ -4,8 +4,12 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ - grammars::{ExternalToken, InputGrammar, VariableType}, - prepare_grammar::intern_symbols::InternedGrammarMeta, + grammars::{ExternalToken, InputGrammar, LexicalGrammar, VariableType}, + prepare_grammar::{ + LexicalToken, + expand_tokens::{ExpandTokensResult, expand_tokens}, + intern_symbols::InternedGrammarMeta, + }, rules::{MetadataParams, Rule, RuleId, RulePool, Symbol}, strpool::StrId, }; @@ -58,22 +62,13 @@ impl std::fmt::Display for NonTerminalWordTokenError { } } -/// A single extracted token. -#[derive(Clone, Debug)] -pub struct LexicalToken { - /// Generated for anon tokens, rule name for absorbed variables - pub name: StrId, - pub kind: VariableType, - /// Pool root defining this token - pub root: RuleId, -} - -/// The extra pass's outputs besides the in-place rewrites. +/// Syntax grammar metadata produced during token extraction. +/// +/// Its symbols are resolved to their final indices before deferred pool rewrites +/// are committed. #[derive(Clone, Debug, Default)] pub(super) struct ExtractedGrammarMeta { pub kinds: Vec, - pub lexical_variables: Vec, - pub separator_roots: Vec, pub extra_symbols: Vec, pub external_tokens: Vec, pub reserved_sets: Vec<(StrId, Vec)>, @@ -89,6 +84,8 @@ struct TokenExtractor { lexical: Vec, usage_counts: Vec, memo: FxHashMap>, + /// Terminal rewrites deferred until lexical expansion has consumed the original pool nodes. + rewrites: Vec<(RuleId, u32)>, } impl TokenExtractor { @@ -137,7 +134,8 @@ impl TokenExtractor { Ok(index) } - /// In-place token extraction over one root. + /// Token extraction over one root. Records each terminal rewrite so that all + /// roots are inspected before any shared pool node is mutated. /// - `String`/`Pattern`: always extracted /// - `token(...)`: metadata extracts the inner child when no other metadata /// params are set, otherwise the whole metadata node @@ -163,7 +161,7 @@ impl TokenExtractor { &mut aux_token_count, is_first, )?; - pool.set_node(id, Rule::from(Symbol::terminal(i as usize))); + self.rewrites.push((id, i)); } Rule::Pattern(..) => { let i = self.extract_token( @@ -174,7 +172,7 @@ impl TokenExtractor { &mut aux_token_count, is_first, )?; - pool.set_node(id, Rule::from(Symbol::terminal(i as usize))); + self.rewrites.push((id, i)); } Rule::Metadata { params, rule } => { let p = pool.params(params); @@ -200,7 +198,7 @@ impl TokenExtractor { &mut aux_token_count, is_first, )?; - pool.set_node(id, Rule::from(Symbol::terminal(i as usize))); + self.rewrites.push((id, i)); } else { stack.push(rule); } @@ -217,7 +215,7 @@ impl TokenExtractor { Ok(()) } - /// Structural lookup + /// Find the lexical token whose body is structurally equal to `root`. fn find(&self, pool: &RulePool, root: RuleId) -> Option { self.memo.get(&pool.subtree_hash(root)).and_then(|cands| { cands @@ -226,12 +224,122 @@ impl TokenExtractor { .find(|&i| pool.subtree_eq(self.lexical[i as usize].root, root)) }) } + + /// Sort pending rewrites for lookup and coalesce duplicate visits to shared nodes. + fn finalize_rewrites(&mut self) { + self.rewrites + .sort_unstable_by_key(|(id, token)| (id.index(), *token)); + self.rewrites.dedup(); + // Discovery only observes unmodified subtrees, so a node can map to just + // one terminal. The binary search in [`Self::symbol_after_rewrites`] and + // the commit loop both rely on the keys being unique. + debug_assert!( + self.rewrites + .windows(2) + .all(|w| w[0].0.index() != w[1].0.index()) + ); + } + + /// Return the symbol `id` will contain after committing pending rewrites, without + /// mutating the pool. + fn symbol_after_rewrites(&self, pool: &RulePool, id: RuleId) -> Option { + self.rewrites + .binary_search_by_key(&id.index(), |(id, _)| id.index()) + .ok() + .map(|i| Symbol::terminal(self.rewrites[i].1 as usize)) + .or_else(|| pool.node(id).symbol()) + } } -pub(super) fn extract_tokens( - g: &mut InputGrammar, +/// Token extraction's pending result. +/// +/// A mutable borrow is held onto the [`InputGrammar`] to prevent other passes from +/// changing the pool while token and separator roots still refer to their original +/// bodies. [`Self::expand_and_commit`] consumes this, expands the bodies, and commits the +/// deferred terminal rewrites and renumbers symbols. +pub(super) struct PendingTokenExtraction<'g> { + grammar: &'g mut InputGrammar, + meta: ExtractedGrammarMeta, + lexical_variables: Vec, + separator_roots: Vec, + rewrites: Vec<(RuleId, u32)>, + syntax_variable_replacements: FxHashMap, + syntax_variable_shift: Vec, + stack: Vec, +} + +impl PendingTokenExtraction<'_> { + /// Expand the token and separator roots, then commit the terminal rewrites and + /// renumber the remaining syntax symbols. + pub(super) fn expand_and_commit( + self, + ) -> ExpandTokensResult<(ExtractedGrammarMeta, LexicalGrammar)> { + let Self { + grammar, + meta, + lexical_variables, + separator_roots, + rewrites, + syntax_variable_replacements, + syntax_variable_shift, + mut stack, + } = self; + + let lexical_grammar = + expand_tokens(&mut grammar.pool, &lexical_variables, &separator_roots)?; + + for (id, token_index) in rewrites { + grammar + .pool + .set_node(id, Rule::from(Symbol::terminal(token_index as usize))); + } + + let replace_symbol = |symbol: Symbol| { + if !symbol.is_non_terminal() { + return symbol; + } + + syntax_variable_replacements.get(&symbol.index).map_or_else( + || { + Symbol::non_terminal( + symbol.index as usize + - syntax_variable_shift[symbol.index as usize] as usize, + ) + }, + |&token_index| Symbol::terminal(token_index as usize), + ) + }; + + if !syntax_variable_replacements.is_empty() { + let mut renumbered_nodes = vec![false; grammar.pool.node_count()]; + for variable in &grammar.variables { + renumber_root( + &mut grammar.pool, + variable.root, + &replace_symbol, + &mut stack, + &mut renumbered_nodes, + ); + } + for &root in &grammar.external_roots { + renumber_root( + &mut grammar.pool, + root, + &replace_symbol, + &mut stack, + &mut renumbered_nodes, + ); + } + } + + Ok((meta, lexical_grammar)) + } +} + +pub(super) fn extract_tokens<'g>( + g: &'g mut InputGrammar, interned: &InternedGrammarMeta, -) -> ExtractTokensResult { +) -> ExtractTokensResult> { let mut extractor = TokenExtractor::default(); let mut stack = Vec::new(); @@ -241,6 +349,7 @@ pub(super) fn extract_tokens( for (&root, &(name, _)) in g.external_roots.iter().zip(&interned.external_tokens) { extractor.extract_in_root(&mut g.pool, root, name, false, &mut stack)?; } + extractor.finalize_rewrites(); // If a variable's entire rule was extracted as a token and that token didn't // appear within any other rule, then remove that variable from the syntax @@ -257,7 +366,7 @@ pub(super) fn extract_tokens( retained.push(g.variables[0]); kinds.push(interned.kinds[0]); for (i, v) in g.variables.iter().enumerate().skip(1) { - if let Some(sym) = g.pool.node(v.root).symbol() + if let Some(sym) = extractor.symbol_after_rewrites(&g.pool, v.root) && sym.is_terminal() && extractor.usage_counts[sym.index as usize] == 1 { @@ -294,13 +403,6 @@ pub(super) fn extract_tokens( ) }; - for v in &g.variables { - renumber_root(&mut g.pool, v.root, &replace_symbol, &mut stack); - } - for &root in &g.external_roots { - renumber_root(&mut g.pool, root, &replace_symbol, &mut stack); - } - // Renumber each conflict through absorption, then canonicalize let conflicts = interned .conflicts @@ -333,11 +435,18 @@ pub(super) fn extract_tokens( let inline = interned.inline.iter().map(|&s| replace_symbol(s)).collect(); + // Resolve each metadata root to the symbol it will represent after token extraction. + // If that symbol is still a non terminal, adjust its index for any grammar + // variables removed above. The syntax rules are renumbered only after all metadata + // roots have been resolved. let mut separator_roots = Vec::new(); let mut extra_symbols = Vec::with_capacity(g.extra_roots.len()); for &root in &g.extra_roots { - if let Some(s) = g.pool.node(root).symbol() { - extra_symbols.push(replace_symbol(s)); + if let Some(s) = extractor + .symbol_after_rewrites(&g.pool, root) + .map(replace_symbol) + { + extra_symbols.push(s); } else if let Some(i) = extractor.find(&g.pool, root) { extra_symbols.push(Symbol::terminal(i as usize)); } else { @@ -347,7 +456,10 @@ pub(super) fn extract_tokens( let mut external_tokens = Vec::with_capacity(g.external_roots.len()); for (&root, &(name, kind)) in g.external_roots.iter().zip(&interned.external_tokens) { - let Some(s) = g.pool.node(root).symbol() else { + let Some(s) = extractor + .symbol_after_rewrites(&g.pool, root) + .map(replace_symbol) + else { Err(ExtractTokensError::NonSymbolExternalToken)? }; if s.is_non_terminal() { @@ -397,8 +509,11 @@ pub(super) fn extract_tokens( for set in &g.reserved_sets { let mut symbols = Vec::with_capacity(set.roots.len()); for &root in &set.roots { - if let Some(s) = g.pool.node(root).symbol() { - symbols.push(replace_symbol(s)); + if let Some(s) = extractor + .symbol_after_rewrites(&g.pool, root) + .map(replace_symbol) + { + symbols.push(s); } else if let Some(i) = extractor.find(&g.pool, root) { symbols.push(Symbol::terminal(i as usize)); } else { @@ -416,10 +531,8 @@ pub(super) fn extract_tokens( reserved_sets.push((set.name, symbols)); } - Ok(ExtractedGrammarMeta { + let meta = ExtractedGrammarMeta { kinds, - lexical_variables: extractor.lexical, - separator_roots, extra_symbols, external_tokens, reserved_sets, @@ -427,18 +540,34 @@ pub(super) fn extract_tokens( conflicts, inline, word, + }; + Ok(PendingTokenExtraction { + grammar: g, + meta, + lexical_variables: extractor.lexical, + separator_roots, + rewrites: extractor.rewrites, + syntax_variable_replacements: replacements, + syntax_variable_shift: shift, + stack, }) } +/// Renumber nodes reachable from `root`, skipping every node already reached via +/// another root in this renumbering pass. fn renumber_root( pool: &mut RulePool, root: RuleId, replace: &impl Fn(Symbol) -> Symbol, stack: &mut Vec, + visited: &mut [bool], ) { stack.clear(); stack.push(root); while let Some(id) = stack.pop() { + if std::mem::replace(&mut visited[id.index()], true) { + continue; + } match pool.node(id) { Rule::Sym { kind, index } => { let s = Symbol { kind, index }; @@ -498,7 +627,36 @@ mod test { Variable { name: pool.intern("rule_3"), root: r3 }, ]; let mut grammar = pool_grammar(pool, variables); - let ext = extract(&mut grammar).unwrap(); + let pending = extract_pending(&mut grammar).unwrap(); + + // Token extraction must preserve each original token body until lexical + // expansion runs. + let roots = pending + .lexical_variables + .iter() + .map(|v| v.root) + .collect::>(); + { + let pool = &mut pending.grammar.pool; + + let e0 = str(pool, "a"); + assert!(pool.subtree_eq(roots[0], e0)); + + let e1 = pat(pool, "b"); + assert!(pool.subtree_eq(roots[1], e1)); + + let e2 = { + let (c, d) = (str(pool, "c"), str(pool, "d")); + let cd = pool.choice(&[c, d]); + pool.repeat(cd) + }; + assert!(pool.subtree_eq(roots[2], e2)); + + let e3 = pat(pool, "e"); + assert!(pool.subtree_eq(roots[3], e3)); + } + + let (ext, lexical_grammar) = pending.expand_and_commit().unwrap(); // rule_1 was absorbed into the lexical grammar, rule_0, rule_2, and rule_3 remain let names = grammar @@ -555,8 +713,8 @@ mod test { // `/e/` is used in exactly one place (as rule_1's whole body), so rule_1 // was absorbed into the lexical grammar (and donated its name to the token). - let lex = ext - .lexical_variables + let lex = lexical_grammar + .variables .iter() .map(|v| (grammar.pool.resolve(v.name), v.kind)) .collect::>(); @@ -569,24 +727,6 @@ mod test { ("rule_1", VariableType::Named), ] ); - let roots = ext - .lexical_variables - .iter() - .map(|v| v.root) - .collect::>(); - let e0 = str(&mut grammar.pool, "a"); - assert!(grammar.pool.subtree_eq(roots[0], e0)); - let e1 = pat(&mut grammar.pool, "b"); - assert!(grammar.pool.subtree_eq(roots[1], e1)); - let e2 = { - let p = &mut grammar.pool; - let (c, d) = (str(p, "c"), str(p, "d")); - let cd = p.choice(&[c, d]); - p.repeat(cd) - }; - assert!(grammar.pool.subtree_eq(roots[2], e2)); - let e3 = pat(&mut grammar.pool, "e"); - assert!(grammar.pool.subtree_eq(roots[3], e3)); } #[test] @@ -600,7 +740,13 @@ mod test { root: r0, }]; let mut grammar = pool_grammar(pool, variables); - let ext = extract(&mut grammar).unwrap(); + let pending = extract_pending(&mut grammar).unwrap(); + + let root = pending.lexical_variables[0].root; + let expected = str(&mut pending.grammar.pool, "hello"); + assert!(pending.grammar.pool.subtree_eq(root, expected)); + + let (_, lexical_grammar) = pending.expand_and_commit().unwrap(); let names = grammar .variables @@ -616,14 +762,12 @@ mod test { } ); - let lex = ext - .lexical_variables + let lex = lexical_grammar + .variables .iter() .map(|v| (grammar.pool.resolve(v.name), v.kind)) .collect::>(); assert_eq!(lex, [("hello", VariableType::Anonymous)]); - let e = str(&mut grammar.pool, "hello"); - assert!(grammar.pool.subtree_eq(ext.lexical_variables[0].root, e)); } #[test] @@ -648,15 +792,18 @@ mod test { ]; let mut grammar = pool_grammar(pool, variables); grammar.extra_roots = vec![sep, extra_ref]; - let ext = extract(&mut grammar).unwrap(); + let pending = extract_pending(&mut grammar).unwrap(); - // comment's `//.*` was single use-> absorbed by terminal(1), so the extra ref - // resolves to it - assert_eq!(ext.extra_symbols, [Symbol::terminal(1)]); + // The single use `comment` rule `//.*` is absorbed as `terminal(1)`, so + // its extra reference becomes `extra_symbols[0]`. + assert_eq!(pending.separator_roots.len(), 1); + let separator_root = pending.separator_roots[0]; // the " " string routes to separators, not a token symbol - assert_eq!(ext.separator_roots.len(), 1); - let e = str(&mut grammar.pool, " "); - assert!(grammar.pool.subtree_eq(ext.separator_roots[0], e)); + let expected = str(&mut pending.grammar.pool, " "); + assert!(pending.grammar.pool.subtree_eq(separator_root, expected)); + + let (ext, _) = pending.expand_and_commit().unwrap(); + assert_eq!(ext.extra_symbols, [Symbol::terminal(1)]); } #[test] @@ -694,7 +841,7 @@ mod test { let rule_2 = pool.intern("rule_2"); let mut grammar = pool_grammar(pool, variables); grammar.external_roots = vec![e0, ea, er2]; - let ext = extract(&mut grammar).unwrap(); + let (ext, _) = extract(&mut grammar).unwrap(); assert_eq!( ext.external_tokens, @@ -778,7 +925,13 @@ mod test { }, ]; let mut grammar = pool_grammar(pool, variables); - let ext = extract(&mut grammar).unwrap(); + let pending = extract_pending(&mut grammar).unwrap(); + + let root = pending.lexical_variables[0].root; + let expected = str(&mut pending.grammar.pool, "a"); + assert!(pending.grammar.pool.subtree_eq(root, expected)); + + let (ext, lexical_grammar) = pending.expand_and_commit().unwrap(); let names = grammar .variables @@ -798,14 +951,12 @@ mod test { Rule::Sym { kind: SymbolType::Terminal, index: 0 } ); - let lex = ext - .lexical_variables + let lex = lexical_grammar + .variables .iter() .map(|v| (grammar.pool.resolve(v.name), v.kind)) .collect::>(); assert_eq!(lex, [("a", VariableType::Anonymous)]); - let e = str(&mut grammar.pool, "a"); - assert!(grammar.pool.subtree_eq(ext.lexical_variables[0].root, e)); } #[test] @@ -860,8 +1011,15 @@ mod test { } } - fn extract(g: &mut InputGrammar) -> ExtractTokensResult { + fn extract_pending(g: &mut InputGrammar) -> ExtractTokensResult> { let meta = intern_symbols(g, &mut Vec::new()).unwrap(); extract_tokens(g, &meta) } + + fn extract( + g: &mut InputGrammar, + ) -> ExtractTokensResult<(ExtractedGrammarMeta, LexicalGrammar)> { + let pending = extract_pending(g)?; + Ok(pending.expand_and_commit().unwrap()) + } } diff --git a/crates/generate/src/prepare_grammar/process_inlines.rs b/crates/generate/src/prepare_grammar/process_inlines.rs index 32e6da970..60f5f986a 100644 --- a/crates/generate/src/prepare_grammar/process_inlines.rs +++ b/crates/generate/src/prepare_grammar/process_inlines.rs @@ -5,7 +5,10 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ - grammars::{InlinedProductionMap, InputGrammar, Production, ProductionStep, ProductionStore}, + grammars::{ + InlinedProductionMap, InputGrammar, LexicalVariable, Production, ProductionStep, + ProductionStore, + }, prepare_grammar::extract_tokens::ExtractedGrammarMeta, rules::{Precedence, Symbol, SymbolType}, }; @@ -180,6 +183,7 @@ pub enum ProcessInlinesError { pub(super) fn process_inlines( g: &InputGrammar, meta: &ExtractedGrammarMeta, + lexical_variables: &[LexicalVariable], out: &mut ProductionStore, ) -> ProcessInlinesResult { if meta.inline.is_empty() { @@ -194,7 +198,7 @@ pub(super) fn process_inlines( ))?, SymbolType::Terminal => Err(ProcessInlinesError::Token( g.pool - .resolve(meta.lexical_variables[symbol.index as usize].name) + .resolve(lexical_variables[symbol.index as usize].name) .to_string(), ))?, SymbolType::NonTerminal if symbol.index == 0 => Err(ProcessInlinesError::FirstRule( @@ -235,7 +239,6 @@ mod tests { use super::*; use crate::{ grammars::VariableType, - prepare_grammar::extract_tokens::LexicalToken, rules::{Alias, Associativity, RulePool, Symbol}, }; @@ -266,12 +269,13 @@ mod tests { ], ); - let g = InputGrammar::default(); + let mut g = InputGrammar::default(); let meta = ExtractedGrammarMeta { inline: vec![Symbol::non_terminal(1)], ..Default::default() }; - let map = process_inlines(&g, &meta, &mut out).unwrap(); + let lexical_variables = make_lexical_variables_through(&mut g.pool, 14); + let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap(); let prod0 = out.var_prods[0].0; // Nothing to inline at step 0. @@ -336,7 +340,7 @@ mod tests { add_variable(&mut out, &[(vec![plain(Symbol::terminal(15))], 0)]); add_variable(&mut out, &[(vec![plain(Symbol::terminal(16))], 0)]); - let g = InputGrammar::default(); + let mut g = InputGrammar::default(); let meta = ExtractedGrammarMeta { inline: vec![ Symbol::non_terminal(1), @@ -345,7 +349,8 @@ mod tests { ], ..Default::default() }; - let map = process_inlines(&g, &meta, &mut out).unwrap(); + let lexical_variables = make_lexical_variables_through(&mut g.pool, 16); + let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap(); let prod0 = out.var_prods[0].0; let (ids, prods) = inlined(&out, &map, prod0, 1).unwrap(); @@ -441,12 +446,13 @@ mod tests { ); add_variable(&mut out, &[(vec![plain(Symbol::terminal(13))], 0)]); - let g = InputGrammar::default(); + let mut g = InputGrammar::default(); let meta = ExtractedGrammarMeta { inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)], ..Default::default() }; - let map = process_inlines(&g, &meta, &mut out).unwrap(); + let lexical_variables = make_lexical_variables_through(&mut g.pool, 13); + let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap(); let prod0 = out.var_prods[0].0; let (ids, prods) = inlined(&out, &map, prod0, 0).unwrap(); @@ -521,28 +527,42 @@ mod tests { fn test_error_when_inlining_tokens() { let mut pool = RulePool::default(); let name = pool.intern("something"); - let root = pool.blank(); let g = InputGrammar { pool, ..Default::default() }; let meta = ExtractedGrammarMeta { inline: vec![Symbol::terminal(0)], - lexical_variables: vec![LexicalToken { - name, - kind: VariableType::Named, - root, - }], ..Default::default() }; + let lexical_variables = [LexicalVariable { + name, + kind: VariableType::Named, + implicit_precedence: 0, + start_state: 0, + }]; let mut out = ProductionStore::default(); - let result = process_inlines(&g, &meta, &mut out); + let result = process_inlines(&g, &meta, &lexical_variables, &mut out); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err, ProcessInlinesError::Token("something".to_string())); } + fn make_lexical_variables_through( + pool: &mut RulePool, + last_index: usize, + ) -> Vec { + (0..=last_index) + .map(|i| LexicalVariable { + name: pool.intern(&format!("t{i}")), + kind: VariableType::Anonymous, + implicit_precedence: 0, + start_state: 0, + }) + .collect() + } + /// Append one variable's productions to `out` and record its production id range. fn add_variable(out: &mut ProductionStore, prods: &[(Vec, i32)]) { let start = out.productions.len() as u32; @@ -614,6 +634,7 @@ mod tests { crate::grammars::Variable { name, root } }) .to_vec(); + let lexical_variables = make_lexical_variables_through(&mut pool, 10); let g = InputGrammar { pool, variables, @@ -624,7 +645,7 @@ mod tests { ..Default::default() }; assert_eq!( - process_inlines(&g, &meta, &mut out).unwrap_err(), + process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap_err(), ProcessInlinesError::NoReachableProductions("rule1".to_string()) ); } diff --git a/crates/generate/src/rules.rs b/crates/generate/src/rules.rs index 9fe5d687b..6c35381b0 100644 --- a/crates/generate/src/rules.rs +++ b/crates/generate/src/rules.rs @@ -184,6 +184,11 @@ impl RulePool { self.nodes[id.index()] } + #[must_use] + pub const fn node_count(&self) -> usize { + self.nodes.len() + } + pub fn set_node(&mut self, id: RuleId, node: Rule) { self.nodes[id.index()] = node; }