libs-code/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:26:23 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix(generate): defer token rewrites until lexical expansion

Browse source 
In a future feature, `RulePool` nodes can be shared between lexical and
syntax roots. No grammars generated by grammar.js/grammra.json are affected
by this issue, as they lead to a tree pool rather than a DAG.

With a DAG `RulePool` representation, however, rewriting tokens during
extraction can destroy a token body before lexical expansion consumes
it and can renumber shared syntax nodes more than once.

Record terminal rewrites while inspecting the original pool, expand tokens
and separators first, then commit the rewrites and renumber each reachable
syntax node once. Resolve grammar metadata against the pending rewrites
before mutating the pool.
This commit is contained in:
Will Lillis 2026-08-21 17:46:01 -04:00
parent 1b64a459ff
commit aa1b924997
6 changed files with 687 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

387
crates/generate/src/prepare_grammar.rs
View file

@ -14,9 +14,9 @@ use std::{
};
pub use expand_tokens::ExpandTokensError;
#[cfg(test)]
pub use expand_tokens::expand_tokens;
pub use extract_tokens::ExtractTokensError;
#[cfg(test)] // TODO: Is this defined in the proper place?
pub use extract_tokens::LexicalToken;
pub use flatten_grammar::FlattenGrammarError;
use indexmap::IndexMap;
pub use intern_symbols::InternSymbolsError;
@ -26,11 +26,10 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
grammars::{InputGrammar, PrecedenceEntry, ProductionStore},
grammars::{InputGrammar, PrecedenceEntry, ProductionStore, VariableType},
strpool::StrPool,
};
pub use self::expand_tokens::expand_tokens;
use self::{
expand_repeats::{ExpandRepeatsError, expand_repeats},
extract_default_aliases::extract_default_aliases,
@ -43,7 +42,7 @@ use super::{
Diagnostic,
grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
prepare_grammar::flatten_grammar::{FlattenState, assemble_syntax_grammar},
rules::{AliasMap, Precedence, Rule},
rules::{AliasMap, Precedence, Rule, RuleId},
strpool::StrId,
};
@ -109,6 +108,20 @@ pub struct PreparedGrammar {
pub str_pool: StrPool,
}
/// A token extracted from the input grammar but not yet expanded into the lexical NFA.
///
/// Token extraction creates this while `root` still points to the original rule in
/// the pool. [`PendingTokenExtraction::expand_and_commit`] passes them to `expand_tokens`
/// before committing the deferred syntax rewrites.
#[derive(Clone, Debug)]
pub struct LexicalToken {
/// Generated for anon tokens, rule name for absorbed variables
pub name: StrId,
pub kind: VariableType,
/// Pool root defining this token
pub root: RuleId,
}
/// Transform an input grammar into separate components that are ready
/// for parse table construction.
pub fn prepare_grammar(
@ -119,21 +132,17 @@ pub fn prepare_grammar(
validate_indirect_recursion(&g)?;
let interned_meta = intern_symbols(&mut g, diagnostics)?;
let mut ext_meta = extract_tokens(&mut g, &interned_meta)?;
let pending_tokens = extract_tokens(&mut g, &interned_meta)?;
let (mut ext_meta, lexical_grammar) = pending_tokens.expand_and_commit()?;
expand_repeats(&mut g, &mut ext_meta)?;
let mut state = FlattenState::default();
let mut out = ProductionStore::default();
flatten_grammar(&g, &ext_meta, &mut state, &mut out)?;
let lexical_grammar = expand_tokens(
&mut g.pool,
&ext_meta.lexical_variables,
&ext_meta.separator_roots,
)?;
let default_aliases = extract_default_aliases(&g, &ext_meta, &mut out);
let inlines = process_inlines(&g, &ext_meta, &mut out)?;
let default_aliases =
extract_default_aliases(&g, &ext_meta, &lexical_grammar.variables, &mut out);
let inlines = process_inlines(&g, &ext_meta, &lexical_grammar.variables, &mut out)?;
let (syntax_grammar, str_pool) = assemble_syntax_grammar(g, ext_meta, out);
Ok(PreparedGrammar {
@ -308,8 +317,8 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()>
mod tests {
use super::*;
use crate::{
grammars::Variable,
rules::{RuleId, RulePool},
grammars::{ProductionStep, Variable, VariableType},
rules::{RuleId, RulePool, Symbol},
};
#[test]
@ -462,6 +471,352 @@ mod tests {
}
}
#[test]
fn test_token_body_shared_with_syntax() {
let grammar = build_grammar(|p| {
let a = leaf(p, "a");
let b = leaf(p, "b");
let shared = p.seq(&[a, b]);
let token = p.token(shared);
let program = {
let t = named(p, "t");
let x = named(p, "x");
p.seq(&[t, x])
};
vec![
Variable {
name: p.intern("program"),
root: program,
},
Variable {
name: p.intern("t"),
root: token,
},
Variable {
name: p.intern("x"),
root: shared,
},
]
});
let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
let syntax_variables = prepared
.syntax_grammar
.variables
.iter()
.map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
.collect::<Vec<_>>();
assert_eq!(
syntax_variables,
[("program", VariableType::Named), ("x", VariableType::Named),]
);
let lexical_variables = prepared
.lexical_grammar
.variables
.iter()
.map(|variable| {
(
prepared.str_pool.resolve(variable.name),
variable.kind,
variable.implicit_precedence,
variable.start_state,
)
})
.collect::<Vec<_>>();
assert_eq!(
lexical_variables,
[
("t", VariableType::Named, 0, 2),
("a", VariableType::Anonymous, 2, 4),
("b", VariableType::Anonymous, 2, 6),
]
);
}
#[test]
fn test_token_body_shared_with_syntax_reversed() {
let grammar = build_grammar(|p| {
let a = leaf(p, "a");
let b = leaf(p, "b");
let shared = p.seq(&[a, b]);
let token = p.token(shared);
let program = {
let x = named(p, "x");
let t = named(p, "t");
p.seq(&[x, t])
};
vec![
Variable {
name: p.intern("program"),
root: program,
},
Variable {
name: p.intern("x"),
root: shared,
},
Variable {
name: p.intern("t"),
root: token,
},
]
});
let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
let syntax_variables = prepared
.syntax_grammar
.variables
.iter()
.map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
.collect::<Vec<_>>();
assert_eq!(
syntax_variables,
[("program", VariableType::Named), ("x", VariableType::Named),]
);
let lexical_variables = prepared
.lexical_grammar
.variables
.iter()
.map(|variable| {
(
prepared.str_pool.resolve(variable.name),
variable.kind,
variable.implicit_precedence,
variable.start_state,
)
})
.collect::<Vec<_>>();
assert_eq!(
lexical_variables,
[
("a", VariableType::Anonymous, 2, 1),
("b", VariableType::Anonymous, 2, 3),
("t", VariableType::Named, 0, 6),
]
);
}
#[test]
fn test_separator_body_shared_with_syntax() {
let mut pool = RulePool::default();
let a = leaf(&mut pool, "a");
let b = leaf(&mut pool, "b");
let shared = pool.seq(&[a, b]);
let program = pool.intern("program");
let grammar = InputGrammar {
variables: vec![Variable {
name: program,
root: shared,
}],
extra_roots: vec![shared],
pool,
..Default::default()
};
let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
let syntax_variables = prepared
.syntax_grammar
.variables
.iter()
.map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
.collect::<Vec<_>>();
assert_eq!(syntax_variables, [("program", VariableType::Named)]);
let lexical_variables = prepared
.lexical_grammar
.variables
.iter()
.map(|variable| {
(
prepared.str_pool.resolve(variable.name),
variable.kind,
variable.implicit_precedence,
variable.start_state,
)
})
.collect::<Vec<_>>();
assert_eq!(
lexical_variables,
[
("a", VariableType::Anonymous, 2, 5),
("b", VariableType::Anonymous, 2, 11),
]
);
assert!(prepared.syntax_grammar.extra_symbols.is_empty());
}
#[test]
fn test_shared_extra_symbol_is_renumbered_once() {
let mut pool = RulePool::default();
let target_ref = named(&mut pool, "target");
let program = {
let kw = named(&mut pool, "kw");
let keep = named(&mut pool, "keep");
pool.seq(&[kw, keep, target_ref])
};
let kw = leaf(&mut pool, "keyword");
let keep = {
let k = leaf(&mut pool, "k");
let e = leaf(&mut pool, "e");
pool.seq(&[k, e])
};
let target = {
let t = leaf(&mut pool, "t");
let g = leaf(&mut pool, "g");
pool.seq(&[t, g])
};
let grammar = InputGrammar {
variables: vec![
Variable {
name: pool.intern("program"),
root: program,
},
Variable {
name: pool.intern("kw"),
root: kw,
},
Variable {
name: pool.intern("keep"),
root: keep,
},
Variable {
name: pool.intern("target"),
root: target,
},
],
extra_roots: vec![target_ref],
pool,
..Default::default()
};
let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
let syntax_variables = prepared
.syntax_grammar
.variables
.iter()
.map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
.collect::<Vec<_>>();
assert_eq!(
syntax_variables,
[
("program", VariableType::Named),
("keep", VariableType::Named),
("target", VariableType::Named),
]
);
let lexical_variables = prepared
.lexical_grammar
.variables
.iter()
.map(|variable| {
(
prepared.str_pool.resolve(variable.name),
variable.kind,
variable.implicit_precedence,
variable.start_state,
)
})
.collect::<Vec<_>>();
assert_eq!(
lexical_variables,
[
("kw", VariableType::Named, 2, 7),
("k", VariableType::Anonymous, 2, 9),
("e", VariableType::Anonymous, 2, 11),
("t", VariableType::Anonymous, 2, 13),
("g", VariableType::Anonymous, 2, 15),
]
);
assert_eq!(
prepared.syntax_grammar.extra_symbols,
[Symbol::non_terminal(2)]
);
}
#[test]
fn test_shared_syntax_subtree_is_renumbered_once() {
let grammar = build_grammar(|p| {
let pair = {
let thing = named(p, "thing");
let dash = leaf(p, "-");
p.seq(&[thing, dash])
};
let item_b = {
let semicolon = leaf(p, ";");
p.seq(&[pair, semicolon])
};
let program = {
let kw = named(p, "kw");
let item_a = named(p, "item_a");
let item_b = named(p, "item_b");
p.seq(&[kw, item_a, item_b])
};
let kw = leaf(p, "keyword");
let thing = {
let t = leaf(p, "t");
let u = leaf(p, "u");
p.seq(&[t, u])
};
vec![
Variable {
name: p.intern("program"),
root: program,
},
Variable {
name: p.intern("kw"),
root: kw,
},
Variable {
name: p.intern("item_a"),
root: pair,
},
Variable {
name: p.intern("item_b"),
root: item_b,
},
Variable {
name: p.intern("thing"),
root: thing,
},
]
});
let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
let expected_steps = [
ProductionStep::pack(
Symbol::non_terminal(3),
Precedence::None,
None,
None,
None,
0,
),
ProductionStep::pack(Symbol::terminal(1), Precedence::None, None, None, None, 0),
];
let production_ids = prepared.syntax_grammar.variable_prod_ids(1);
assert_eq!(production_ids.len(), 1);
let production = prepared.syntax_grammar.production(production_ids.start);
assert_eq!(production.steps, expected_steps);
assert_eq!(production.dynamic_precedence, 0);
}
fn named(pool: &mut RulePool, name: &str) -> RuleId {
let id = pool.intern(name);
pool.named_symbol(id)

2
crates/generate/src/prepare_grammar/expand_tokens.rs
View file

@ -5,7 +5,7 @@ use thiserror::Error;
use crate::{
grammars::{LexicalGrammar, LexicalVariable},
nfa::{CharacterSet, Nfa, NfaState},
prepare_grammar::{extract_tokens::LexicalToken, pattern},
prepare_grammar::{LexicalToken, pattern},
rules::{Precedence, Rule, RuleId, RulePool, Symbol},
};

59
crates/generate/src/prepare_grammar/extract_default_aliases.rs
View file

@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use crate::{
grammars::{InputGrammar, ProductionStore},
grammars::{InputGrammar, LexicalVariable, ProductionStore},
prepare_grammar::extract_tokens::ExtractedGrammarMeta,
rules::{Alias, AliasMap, Symbol, SymbolType},
};
@ -24,9 +24,10 @@ struct SymbolStatus {
pub(super) fn extract_default_aliases(
g: &InputGrammar,
meta: &ExtractedGrammarMeta,
lexical_variables: &[LexicalVariable],
out: &mut ProductionStore,
) -> AliasMap {
let mut terminal_status_list = vec![SymbolStatus::default(); meta.lexical_variables.len()];
let mut terminal_status_list = vec![SymbolStatus::default(); lexical_variables.len()];
let mut non_terminal_status_list = vec![SymbolStatus::default(); g.variables.len()];
let mut external_status_list = vec![SymbolStatus::default(); meta.external_tokens.len()];
@ -171,7 +172,6 @@ mod tests {
use super::*;
use crate::{
grammars::{Production, ProductionStep, Variable, VariableType},
prepare_grammar::extract_tokens::LexicalToken,
rules::{Precedence, RulePool},
};
@ -181,20 +181,26 @@ mod tests {
let dummy = pool.intern("_");
let root = pool.blank();
let mut lexical_variables = Vec::new();
let t0 = add_lexical_variable(&mut pool, &mut lexical_variables, "t0");
let t1 = add_lexical_variable(&mut pool, &mut lexical_variables, "t1");
let t2 = add_lexical_variable(&mut pool, &mut lexical_variables, "t2");
let t3 = add_lexical_variable(&mut pool, &mut lexical_variables, "t3");
// v1: every token aliased.
let v1 = vec![
aliased(&mut pool, Symbol::terminal(0), "a1"),
aliased(&mut pool, Symbol::terminal(1), "a2"),
aliased(&mut pool, Symbol::terminal(2), "a3"),
aliased(&mut pool, Symbol::terminal(3), "a4"),
aliased(&mut pool, t0, "a1"),
aliased(&mut pool, t1, "a2"),
aliased(&mut pool, t2, "a3"),
aliased(&mut pool, t3, "a4"),
];
// v2: t0 same alias, t1 unaliased, t2 aliased differently, t3 aliased twice as a6
let v2 = vec![
aliased(&mut pool, Symbol::terminal(0), "a1"),
plain(Symbol::terminal(1)),
aliased(&mut pool, Symbol::terminal(2), "a5"),
aliased(&mut pool, Symbol::terminal(3), "a6"),
aliased(&mut pool, Symbol::terminal(3), "a6"),
aliased(&mut pool, t0, "a1"),
plain(t1),
aliased(&mut pool, t2, "a5"),
aliased(&mut pool, t3, "a6"),
aliased(&mut pool, t3, "a6"),
];
let mut out = ProductionStore::default();
@ -216,18 +222,8 @@ mod tests {
pool,
..Default::default()
};
let meta = ExtractedGrammarMeta {
lexical_variables: (0..4)
.map(|_| LexicalToken {
name: dummy,
kind: VariableType::Anonymous,
root,
})
.collect(),
..Default::default()
};
let pool_map = extract_default_aliases(&g, &meta, &mut out);
let meta = ExtractedGrammarMeta::default();
let pool_map = extract_default_aliases(&g, &meta, &lexical_variables, &mut out);
// t0 -> a1, t2 -> a3 (v1 wins the tie from appearing first), t3 -> a6 (used twice)
// t1 -> none (appears unaliased in v2).
@ -259,6 +255,21 @@ mod tests {
assert_eq!(step_alias(8), None); // v2 t3(a6) = default
}
fn add_lexical_variable(
pool: &mut RulePool,
variables: &mut Vec<LexicalVariable>,
name: &str,
) -> Symbol {
let symbol = Symbol::terminal(variables.len());
variables.push(LexicalVariable {
name: pool.intern(name),
kind: VariableType::Anonymous,
implicit_precedence: 0,
start_state: 0,
});
symbol
}
fn aliased(pool: &mut RulePool, symbol: Symbol, name: &str) -> ProductionStep {
let value = pool.intern(name);
ProductionStep::pack(

316
crates/generate/src/prepare_grammar/extract_tokens.rs
View file

@ -4,8 +4,12 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
grammars::{ExternalToken, InputGrammar, VariableType},
prepare_grammar::intern_symbols::InternedGrammarMeta,
grammars::{ExternalToken, InputGrammar, LexicalGrammar, VariableType},
prepare_grammar::{
LexicalToken,
expand_tokens::{ExpandTokensResult, expand_tokens},
intern_symbols::InternedGrammarMeta,
},
rules::{MetadataParams, Rule, RuleId, RulePool, Symbol},
strpool::StrId,
};
@ -58,22 +62,13 @@ impl std::fmt::Display for NonTerminalWordTokenError {
}
}
/// A single extracted token.
#[derive(Clone, Debug)]
pub struct LexicalToken {
/// Generated for anon tokens, rule name for absorbed variables
pub name: StrId,
pub kind: VariableType,
/// Pool root defining this token
pub root: RuleId,
}
/// The extra pass's outputs besides the in-place rewrites.
/// Syntax grammar metadata produced during token extraction.
///
/// Its symbols are resolved to their final indices before deferred pool rewrites
/// are committed.
#[derive(Clone, Debug, Default)]
pub(super) struct ExtractedGrammarMeta {
pub kinds: Vec<VariableType>,
pub lexical_variables: Vec<LexicalToken>,
pub separator_roots: Vec<RuleId>,
pub extra_symbols: Vec<Symbol>,
pub external_tokens: Vec<ExternalToken>,
pub reserved_sets: Vec<(StrId, Vec<Symbol>)>,
@ -89,6 +84,8 @@ struct TokenExtractor {
lexical: Vec<LexicalToken>,
usage_counts: Vec<u32>,
memo: FxHashMap<u64, Vec<u32>>,
/// Terminal rewrites deferred until lexical expansion has consumed the original pool nodes.
rewrites: Vec<(RuleId, u32)>,
}
impl TokenExtractor {
@ -137,7 +134,8 @@ impl TokenExtractor {
Ok(index)
}
/// In-place token extraction over one root.
/// Token extraction over one root. Records each terminal rewrite so that all
/// roots are inspected before any shared pool node is mutated.
/// - `String`/`Pattern`: always extracted
/// - `token(...)`: metadata extracts the inner child when no other metadata
/// params are set, otherwise the whole metadata node
@ -163,7 +161,7 @@ impl TokenExtractor {
&mut aux_token_count,
is_first,
)?;
pool.set_node(id, Rule::from(Symbol::terminal(i as usize)));
self.rewrites.push((id, i));
}
Rule::Pattern(..) => {
let i = self.extract_token(
@ -174,7 +172,7 @@ impl TokenExtractor {
&mut aux_token_count,
is_first,
)?;
pool.set_node(id, Rule::from(Symbol::terminal(i as usize)));
self.rewrites.push((id, i));
}
Rule::Metadata { params, rule } => {
let p = pool.params(params);
@ -200,7 +198,7 @@ impl TokenExtractor {
&mut aux_token_count,
is_first,
)?;
pool.set_node(id, Rule::from(Symbol::terminal(i as usize)));
self.rewrites.push((id, i));
} else {
stack.push(rule);
}
@ -217,7 +215,7 @@ impl TokenExtractor {
Ok(())
}
/// Structural lookup
/// Find the lexical token whose body is structurally equal to `root`.
fn find(&self, pool: &RulePool, root: RuleId) -> Option<u32> {
self.memo.get(&pool.subtree_hash(root)).and_then(|cands| {
cands
@ -226,12 +224,122 @@ impl TokenExtractor {
.find(|&i| pool.subtree_eq(self.lexical[i as usize].root, root))
})
}
/// Sort pending rewrites for lookup and coalesce duplicate visits to shared nodes.
fn finalize_rewrites(&mut self) {
self.rewrites
.sort_unstable_by_key(|(id, token)| (id.index(), *token));
self.rewrites.dedup();
// Discovery only observes unmodified subtrees, so a node can map to just
// one terminal. The binary search in [`Self::symbol_after_rewrites`] and
// the commit loop both rely on the keys being unique.
debug_assert!(
self.rewrites
.windows(2)
.all(|w| w[0].0.index() != w[1].0.index())
);
}
/// Return the symbol `id` will contain after committing pending rewrites, without
/// mutating the pool.
fn symbol_after_rewrites(&self, pool: &RulePool, id: RuleId) -> Option<Symbol> {
self.rewrites
.binary_search_by_key(&id.index(), |(id, _)| id.index())
.ok()
.map(|i| Symbol::terminal(self.rewrites[i].1 as usize))
.or_else(|| pool.node(id).symbol())
}
}
pub(super) fn extract_tokens(
g: &mut InputGrammar,
/// Token extraction's pending result.
///
/// A mutable borrow is held onto the [`InputGrammar`] to prevent other passes from
/// changing the pool while token and separator roots still refer to their original
/// bodies. [`Self::expand_and_commit`] consumes this, expands the bodies, and commits the
/// deferred terminal rewrites and renumbers symbols.
pub(super) struct PendingTokenExtraction<'g> {
grammar: &'g mut InputGrammar,
meta: ExtractedGrammarMeta,
lexical_variables: Vec<LexicalToken>,
separator_roots: Vec<RuleId>,
rewrites: Vec<(RuleId, u32)>,
syntax_variable_replacements: FxHashMap<u32, u32>,
syntax_variable_shift: Vec<u32>,
stack: Vec<RuleId>,
}
impl PendingTokenExtraction<'_> {
/// Expand the token and separator roots, then commit the terminal rewrites and
/// renumber the remaining syntax symbols.
pub(super) fn expand_and_commit(
self,
) -> ExpandTokensResult<(ExtractedGrammarMeta, LexicalGrammar)> {
let Self {
grammar,
meta,
lexical_variables,
separator_roots,
rewrites,
syntax_variable_replacements,
syntax_variable_shift,
mut stack,
} = self;
let lexical_grammar =
expand_tokens(&mut grammar.pool, &lexical_variables, &separator_roots)?;
for (id, token_index) in rewrites {
grammar
.pool
.set_node(id, Rule::from(Symbol::terminal(token_index as usize)));
}
let replace_symbol = |symbol: Symbol| {
if !symbol.is_non_terminal() {
return symbol;
}
syntax_variable_replacements.get(&symbol.index).map_or_else(
|| {
Symbol::non_terminal(
symbol.index as usize
- syntax_variable_shift[symbol.index as usize] as usize,
)
},
|&token_index| Symbol::terminal(token_index as usize),
)
};
if !syntax_variable_replacements.is_empty() {
let mut renumbered_nodes = vec![false; grammar.pool.node_count()];
for variable in &grammar.variables {
renumber_root(
&mut grammar.pool,
variable.root,
&replace_symbol,
&mut stack,
&mut renumbered_nodes,
);
}
for &root in &grammar.external_roots {
renumber_root(
&mut grammar.pool,
root,
&replace_symbol,
&mut stack,
&mut renumbered_nodes,
);
}
}
Ok((meta, lexical_grammar))
}
}
pub(super) fn extract_tokens<'g>(
g: &'g mut InputGrammar,
interned: &InternedGrammarMeta,
) -> ExtractTokensResult<ExtractedGrammarMeta> {
) -> ExtractTokensResult<PendingTokenExtraction<'g>> {
let mut extractor = TokenExtractor::default();
let mut stack = Vec::new();
@ -241,6 +349,7 @@ pub(super) fn extract_tokens(
for (&root, &(name, _)) in g.external_roots.iter().zip(&interned.external_tokens) {
extractor.extract_in_root(&mut g.pool, root, name, false, &mut stack)?;
}
extractor.finalize_rewrites();
// If a variable's entire rule was extracted as a token and that token didn't
// appear within any other rule, then remove that variable from the syntax
@ -257,7 +366,7 @@ pub(super) fn extract_tokens(
retained.push(g.variables[0]);
kinds.push(interned.kinds[0]);
for (i, v) in g.variables.iter().enumerate().skip(1) {
if let Some(sym) = g.pool.node(v.root).symbol()
if let Some(sym) = extractor.symbol_after_rewrites(&g.pool, v.root)
&& sym.is_terminal()
&& extractor.usage_counts[sym.index as usize] == 1
{
@ -294,13 +403,6 @@ pub(super) fn extract_tokens(
)
};
for v in &g.variables {
renumber_root(&mut g.pool, v.root, &replace_symbol, &mut stack);
}
for &root in &g.external_roots {
renumber_root(&mut g.pool, root, &replace_symbol, &mut stack);
}
// Renumber each conflict through absorption, then canonicalize
let conflicts = interned
.conflicts
@ -333,11 +435,18 @@ pub(super) fn extract_tokens(
let inline = interned.inline.iter().map(|&s| replace_symbol(s)).collect();
// Resolve each metadata root to the symbol it will represent after token extraction.
// If that symbol is still a non terminal, adjust its index for any grammar
// variables removed above. The syntax rules are renumbered only after all metadata
// roots have been resolved.
let mut separator_roots = Vec::new();
let mut extra_symbols = Vec::with_capacity(g.extra_roots.len());
for &root in &g.extra_roots {
if let Some(s) = g.pool.node(root).symbol() {
extra_symbols.push(replace_symbol(s));
if let Some(s) = extractor
.symbol_after_rewrites(&g.pool, root)
.map(replace_symbol)
{
extra_symbols.push(s);
} else if let Some(i) = extractor.find(&g.pool, root) {
extra_symbols.push(Symbol::terminal(i as usize));
} else {
@ -347,7 +456,10 @@ pub(super) fn extract_tokens(
let mut external_tokens = Vec::with_capacity(g.external_roots.len());
for (&root, &(name, kind)) in g.external_roots.iter().zip(&interned.external_tokens) {
let Some(s) = g.pool.node(root).symbol() else {
let Some(s) = extractor
.symbol_after_rewrites(&g.pool, root)
.map(replace_symbol)
else {
Err(ExtractTokensError::NonSymbolExternalToken)?
};
if s.is_non_terminal() {
@ -397,8 +509,11 @@ pub(super) fn extract_tokens(
for set in &g.reserved_sets {
let mut symbols = Vec::with_capacity(set.roots.len());
for &root in &set.roots {
if let Some(s) = g.pool.node(root).symbol() {
symbols.push(replace_symbol(s));
if let Some(s) = extractor
.symbol_after_rewrites(&g.pool, root)
.map(replace_symbol)
{
symbols.push(s);
} else if let Some(i) = extractor.find(&g.pool, root) {
symbols.push(Symbol::terminal(i as usize));
} else {
@ -416,10 +531,8 @@ pub(super) fn extract_tokens(
reserved_sets.push((set.name, symbols));
}
Ok(ExtractedGrammarMeta {
let meta = ExtractedGrammarMeta {
kinds,
lexical_variables: extractor.lexical,
separator_roots,
extra_symbols,
external_tokens,
reserved_sets,
@ -427,18 +540,34 @@ pub(super) fn extract_tokens(
conflicts,
inline,
word,
};
Ok(PendingTokenExtraction {
grammar: g,
meta,
lexical_variables: extractor.lexical,
separator_roots,
rewrites: extractor.rewrites,
syntax_variable_replacements: replacements,
syntax_variable_shift: shift,
stack,
})
}
/// Renumber nodes reachable from `root`, skipping every node already reached via
/// another root in this renumbering pass.
fn renumber_root(
pool: &mut RulePool,
root: RuleId,
replace: &impl Fn(Symbol) -> Symbol,
stack: &mut Vec<RuleId>,
visited: &mut [bool],
) {
stack.clear();
stack.push(root);
while let Some(id) = stack.pop() {
if std::mem::replace(&mut visited[id.index()], true) {
continue;
}
match pool.node(id) {
Rule::Sym { kind, index } => {
let s = Symbol { kind, index };
@ -498,7 +627,36 @@ mod test {
Variable { name: pool.intern("rule_3"), root: r3 },
];
let mut grammar = pool_grammar(pool, variables);
let ext = extract(&mut grammar).unwrap();
let pending = extract_pending(&mut grammar).unwrap();
// Token extraction must preserve each original token body until lexical
// expansion runs.
let roots = pending
.lexical_variables
.iter()
.map(|v| v.root)
.collect::<Vec<_>>();
{
let pool = &mut pending.grammar.pool;
let e0 = str(pool, "a");
assert!(pool.subtree_eq(roots[0], e0));
let e1 = pat(pool, "b");
assert!(pool.subtree_eq(roots[1], e1));
let e2 = {
let (c, d) = (str(pool, "c"), str(pool, "d"));
let cd = pool.choice(&[c, d]);
pool.repeat(cd)
};
assert!(pool.subtree_eq(roots[2], e2));
let e3 = pat(pool, "e");
assert!(pool.subtree_eq(roots[3], e3));
}
let (ext, lexical_grammar) = pending.expand_and_commit().unwrap();
// rule_1 was absorbed into the lexical grammar, rule_0, rule_2, and rule_3 remain
let names = grammar
@ -555,8 +713,8 @@ mod test {
// `/e/` is used in exactly one place (as rule_1's whole body), so rule_1
// was absorbed into the lexical grammar (and donated its name to the token).
let lex = ext
.lexical_variables
let lex = lexical_grammar
.variables
.iter()
.map(|v| (grammar.pool.resolve(v.name), v.kind))
.collect::<Vec<_>>();
@ -569,24 +727,6 @@ mod test {
("rule_1", VariableType::Named),
]
);
let roots = ext
.lexical_variables
.iter()
.map(|v| v.root)
.collect::<Vec<_>>();
let e0 = str(&mut grammar.pool, "a");
assert!(grammar.pool.subtree_eq(roots[0], e0));
let e1 = pat(&mut grammar.pool, "b");
assert!(grammar.pool.subtree_eq(roots[1], e1));
let e2 = {
let p = &mut grammar.pool;
let (c, d) = (str(p, "c"), str(p, "d"));
let cd = p.choice(&[c, d]);
p.repeat(cd)
};
assert!(grammar.pool.subtree_eq(roots[2], e2));
let e3 = pat(&mut grammar.pool, "e");
assert!(grammar.pool.subtree_eq(roots[3], e3));
}
#[test]
@ -600,7 +740,13 @@ mod test {
root: r0,
}];
let mut grammar = pool_grammar(pool, variables);
let ext = extract(&mut grammar).unwrap();
let pending = extract_pending(&mut grammar).unwrap();
let root = pending.lexical_variables[0].root;
let expected = str(&mut pending.grammar.pool, "hello");
assert!(pending.grammar.pool.subtree_eq(root, expected));
let (_, lexical_grammar) = pending.expand_and_commit().unwrap();
let names = grammar
.variables
@ -616,14 +762,12 @@ mod test {
}
);
let lex = ext
.lexical_variables
let lex = lexical_grammar
.variables
.iter()
.map(|v| (grammar.pool.resolve(v.name), v.kind))
.collect::<Vec<_>>();
assert_eq!(lex, [("hello", VariableType::Anonymous)]);
let e = str(&mut grammar.pool, "hello");
assert!(grammar.pool.subtree_eq(ext.lexical_variables[0].root, e));
}
#[test]
@ -648,15 +792,18 @@ mod test {
];
let mut grammar = pool_grammar(pool, variables);
grammar.extra_roots = vec![sep, extra_ref];
let ext = extract(&mut grammar).unwrap();
let pending = extract_pending(&mut grammar).unwrap();
// comment's `//.*` was single use-> absorbed by terminal(1), so the extra ref
// resolves to it
assert_eq!(ext.extra_symbols, [Symbol::terminal(1)]);
// The single use `comment` rule `//.*` is absorbed as `terminal(1)`, so
// its extra reference becomes `extra_symbols[0]`.
assert_eq!(pending.separator_roots.len(), 1);
let separator_root = pending.separator_roots[0];
// the " " string routes to separators, not a token symbol
assert_eq!(ext.separator_roots.len(), 1);
let e = str(&mut grammar.pool, " ");
assert!(grammar.pool.subtree_eq(ext.separator_roots[0], e));
let expected = str(&mut pending.grammar.pool, " ");
assert!(pending.grammar.pool.subtree_eq(separator_root, expected));
let (ext, _) = pending.expand_and_commit().unwrap();
assert_eq!(ext.extra_symbols, [Symbol::terminal(1)]);
}
#[test]
@ -694,7 +841,7 @@ mod test {
let rule_2 = pool.intern("rule_2");
let mut grammar = pool_grammar(pool, variables);
grammar.external_roots = vec![e0, ea, er2];
let ext = extract(&mut grammar).unwrap();
let (ext, _) = extract(&mut grammar).unwrap();
assert_eq!(
ext.external_tokens,
@ -778,7 +925,13 @@ mod test {
},
];
let mut grammar = pool_grammar(pool, variables);
let ext = extract(&mut grammar).unwrap();
let pending = extract_pending(&mut grammar).unwrap();
let root = pending.lexical_variables[0].root;
let expected = str(&mut pending.grammar.pool, "a");
assert!(pending.grammar.pool.subtree_eq(root, expected));
let (ext, lexical_grammar) = pending.expand_and_commit().unwrap();
let names = grammar
.variables
@ -798,14 +951,12 @@ mod test {
Rule::Sym { kind: SymbolType::Terminal, index: 0 }
);
let lex = ext
.lexical_variables
let lex = lexical_grammar
.variables
.iter()
.map(|v| (grammar.pool.resolve(v.name), v.kind))
.collect::<Vec<_>>();
assert_eq!(lex, [("a", VariableType::Anonymous)]);
let e = str(&mut grammar.pool, "a");
assert!(grammar.pool.subtree_eq(ext.lexical_variables[0].root, e));
}
#[test]
@ -860,8 +1011,15 @@ mod test {
}
}
fn extract(g: &mut InputGrammar) -> ExtractTokensResult<ExtractedGrammarMeta> {
fn extract_pending(g: &mut InputGrammar) -> ExtractTokensResult<PendingTokenExtraction<'_>> {
let meta = intern_symbols(g, &mut Vec::new()).unwrap();
extract_tokens(g, &meta)
}
fn extract(
g: &mut InputGrammar,
) -> ExtractTokensResult<(ExtractedGrammarMeta, LexicalGrammar)> {
let pending = extract_pending(g)?;
Ok(pending.expand_and_commit().unwrap())
}
}

55
crates/generate/src/prepare_grammar/process_inlines.rs
View file

@ -5,7 +5,10 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
grammars::{InlinedProductionMap, InputGrammar, Production, ProductionStep, ProductionStore},
grammars::{
InlinedProductionMap, InputGrammar, LexicalVariable, Production, ProductionStep,
ProductionStore,
},
prepare_grammar::extract_tokens::ExtractedGrammarMeta,
rules::{Precedence, Symbol, SymbolType},
};
@ -180,6 +183,7 @@ pub enum ProcessInlinesError {
pub(super) fn process_inlines(
g: &InputGrammar,
meta: &ExtractedGrammarMeta,
lexical_variables: &[LexicalVariable],
out: &mut ProductionStore,
) -> ProcessInlinesResult<InlinedProductionMap> {
if meta.inline.is_empty() {
@ -194,7 +198,7 @@ pub(super) fn process_inlines(
))?,
SymbolType::Terminal => Err(ProcessInlinesError::Token(
g.pool
.resolve(meta.lexical_variables[symbol.index as usize].name)
.resolve(lexical_variables[symbol.index as usize].name)
.to_string(),
))?,
SymbolType::NonTerminal if symbol.index == 0 => Err(ProcessInlinesError::FirstRule(
@ -235,7 +239,6 @@ mod tests {
use super::*;
use crate::{
grammars::VariableType,
prepare_grammar::extract_tokens::LexicalToken,
rules::{Alias, Associativity, RulePool, Symbol},
};
@ -266,12 +269,13 @@ mod tests {
],
);
let g = InputGrammar::default();
let mut g = InputGrammar::default();
let meta = ExtractedGrammarMeta {
inline: vec![Symbol::non_terminal(1)],
..Default::default()
};
let map = process_inlines(&g, &meta, &mut out).unwrap();
let lexical_variables = make_lexical_variables_through(&mut g.pool, 14);
let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap();
let prod0 = out.var_prods[0].0;
// Nothing to inline at step 0.
@ -336,7 +340,7 @@ mod tests {
add_variable(&mut out, &[(vec![plain(Symbol::terminal(15))], 0)]);
add_variable(&mut out, &[(vec![plain(Symbol::terminal(16))], 0)]);
let g = InputGrammar::default();
let mut g = InputGrammar::default();
let meta = ExtractedGrammarMeta {
inline: vec![
Symbol::non_terminal(1),
@ -345,7 +349,8 @@ mod tests {
],
..Default::default()
};
let map = process_inlines(&g, &meta, &mut out).unwrap();
let lexical_variables = make_lexical_variables_through(&mut g.pool, 16);
let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap();
let prod0 = out.var_prods[0].0;
let (ids, prods) = inlined(&out, &map, prod0, 1).unwrap();
@ -441,12 +446,13 @@ mod tests {
);
add_variable(&mut out, &[(vec![plain(Symbol::terminal(13))], 0)]);
let g = InputGrammar::default();
let mut g = InputGrammar::default();
let meta = ExtractedGrammarMeta {
inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
..Default::default()
};
let map = process_inlines(&g, &meta, &mut out).unwrap();
let lexical_variables = make_lexical_variables_through(&mut g.pool, 13);
let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap();
let prod0 = out.var_prods[0].0;
let (ids, prods) = inlined(&out, &map, prod0, 0).unwrap();
@ -521,28 +527,42 @@ mod tests {
fn test_error_when_inlining_tokens() {
let mut pool = RulePool::default();
let name = pool.intern("something");
let root = pool.blank();
let g = InputGrammar {
pool,
..Default::default()
};
let meta = ExtractedGrammarMeta {
inline: vec![Symbol::terminal(0)],
lexical_variables: vec![LexicalToken {
name,
kind: VariableType::Named,
root,
}],
..Default::default()
};
let lexical_variables = [LexicalVariable {
name,
kind: VariableType::Named,
implicit_precedence: 0,
start_state: 0,
}];
let mut out = ProductionStore::default();
let result = process_inlines(&g, &meta, &mut out);
let result = process_inlines(&g, &meta, &lexical_variables, &mut out);
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err, ProcessInlinesError::Token("something".to_string()));
}
fn make_lexical_variables_through(
pool: &mut RulePool,
last_index: usize,
) -> Vec<LexicalVariable> {
(0..=last_index)
.map(|i| LexicalVariable {
name: pool.intern(&format!("t{i}")),
kind: VariableType::Anonymous,
implicit_precedence: 0,
start_state: 0,
})
.collect()
}
/// Append one variable's productions to `out` and record its production id range.
fn add_variable(out: &mut ProductionStore, prods: &[(Vec<ProductionStep>, i32)]) {
let start = out.productions.len() as u32;
@ -614,6 +634,7 @@ mod tests {
crate::grammars::Variable { name, root }
})
.to_vec();
let lexical_variables = make_lexical_variables_through(&mut pool, 10);
let g = InputGrammar {
pool,
variables,
@ -624,7 +645,7 @@ mod tests {
..Default::default()
};
assert_eq!(
process_inlines(&g, &meta, &mut out).unwrap_err(),
process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap_err(),
ProcessInlinesError::NoReachableProductions("rule1".to_string())
);
}

5
crates/generate/src/rules.rs
View file

@ -184,6 +184,11 @@ impl RulePool {
self.nodes[id.index()]
}
#[must_use]
pub const fn node_count(&self) -> usize {
self.nodes.len()
}
pub fn set_node(&mut self, id: RuleId, node: Rule) {
self.nodes[id.index()] = node;
}