libs-code/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:26:23 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

perf(generate): represent the grammar IR as a flat, pooled arena

Browse source 
Replace the owned `Rule`-tree grammar with a flat, pooled representation. `Rule`
nodes live in a `RulePool` arena and reference their children and params by
index (`RuleId`). Strings are interned in a `StrPool` and referred to by `StrId`.
Productions are stored as flat `ProductionStep`/`Production` slices indexed per
variable, rather than as nested vectors hanging off each `SyntaxVariable`. This
drops the per-rule heap allocation of the owned-tree form and is the groundwork
for the later table-building optimizations.

`parse_grammar` builds the pool, the prepare passes rewrite nodes in place, and
`prepare_grammar` returns a `PreparedGrammar` bundle that owns the run's `StrPool`
alongside the grammars. `build_tables`, `node_types`, and `render` borrow that
pool and resolve `StrId`s only at the output boundary.

Shows an average ~10% walltime reduction when combined with the previous
commit, with nearly all of the savings in `build_tables`. The `prepare`
stage is also ~70% faster and much more stable, but this contributes
much less to overall generate time. Improving early stages such as
`prepare` will be important for future interactive uses.
This commit is contained in:
Will Lillis 2026-08-09 11:51:45 -05:00
parent e72c8594fb
commit fb2052f02d
25 changed files with 6040 additions and 4270 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -52,6 +52,7 @@ similar_names = "allow"
string_lit_as_bytes = "allow"
struct_excessive_bools = "allow"
too_many_lines = "allow"
tuple_array_conversions = "allow"
[workspace.lints.rust]
mismatched_lifetime_syntaxes = "allow"

67
crates/generate/src/build_tables.rs
View file

@ -29,6 +29,7 @@ use crate::{
nfa::{CharacterSet, NfaCursor},
node_types::VariableInfo,
rules::{AliasMap, Symbol, SymbolType, TokenSet},
strpool::StrPool,
tables::{LexTable, ParseAction, ParseTable, ParseTableEntry},
};
@ -49,20 +50,21 @@ pub fn build_tables(
simple_aliases: &AliasMap,
variable_info: &[VariableInfo],
inlines: &InlinedProductionMap,
str_pool: &StrPool,
report_symbol_name: Option<&str>,
optimizations: OptLevel,
diagnostics: &mut Vec<Diagnostic>,
) -> BuildTableResult<Tables> {
let item_key_map = ItemKeyMap::new(syntax_grammar, inlines);
let item_key_map = ItemKeyMap::new(syntax_grammar, str_pool);
let item_set_builder =
ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines, &item_key_map);
let following_tokens =
get_following_tokens(syntax_grammar, lexical_grammar, inlines, &item_set_builder);
let following_tokens = get_following_tokens(syntax_grammar, lexical_grammar, &item_set_builder);
let (mut parse_table, parse_state_info) = build_parse_table(
syntax_grammar,
lexical_grammar,
item_set_builder,
variable_info,
str_pool,
diagnostics,
)?;
let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
@ -73,6 +75,7 @@ pub fn build_tables(
syntax_grammar.word_token,
&token_conflict_map,
&coincident_token_index,
str_pool,
);
populate_error_state(
&mut parse_table,
@ -81,6 +84,7 @@ pub fn build_tables(
&coincident_token_index,
&token_conflict_map,
&keywords,
str_pool,
);
populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
minimize_parse_table(
@ -90,6 +94,7 @@ pub fn build_tables(
simple_aliases,
&token_conflict_map,
&keywords,
str_pool,
optimizations,
);
let lex_tables = build_lex_table(
@ -109,6 +114,7 @@ pub fn build_tables(
lexical_grammar,
&parse_table,
&parse_state_info,
str_pool,
report_symbol_name,
);
}
@ -128,25 +134,20 @@ pub fn build_tables(
fn get_following_tokens(
syntax_grammar: &SyntaxGrammar,
lexical_grammar: &LexicalGrammar,
inlines: &InlinedProductionMap,
builder: &ParseItemSetBuilder,
) -> Vec<TokenSet> {
let n_terminals = lexical_grammar.variables.len();
let n_externals = syntax_grammar.external_tokens.len();
let mut result = vec![TokenSet::with_capacity(n_terminals, n_externals); n_terminals];
let productions = syntax_grammar
.variables
.iter()
.flat_map(|v| &v.productions)
.chain(&inlines.productions);
let all_tokens = (0..result.len())
.map(Symbol::terminal)
.collect::<TokenSet>();
for production in productions {
for i in 1..production.steps.len() {
let left_tokens = builder.last_set(&production.steps[i - 1].symbol);
let right_tokens = builder.first_set(&production.steps[i].symbol);
let right_reserved_tokens = builder.reserved_first_set(&production.steps[i].symbol);
for production in &syntax_grammar.productions {
let steps = &syntax_grammar.steps[production.step_range()];
for i in 1..steps.len() {
let left_tokens = builder.last_set(&steps[i - 1].symbol());
let right_tokens = builder.first_set(&steps[i].symbol());
let right_reserved_tokens = builder.reserved_first_set(&steps[i].symbol());
for left_token in left_tokens.iter() {
if left_token.is_terminal() {
result[left_token.index].insert_all_terminals(right_tokens);
@ -175,6 +176,7 @@ fn populate_error_state(
coincident_token_index: &CoincidentTokenIndex,
token_conflict_map: &TokenConflictMap,
keywords: &TokenSet,
str_pool: &StrPool,
) {
let state = &mut parse_table.states[0];
let n = lexical_grammar.variables.len();
@ -193,7 +195,7 @@ fn populate_error_state(
} else {
debug!(
"error recovery - token {} has no conflicts",
lexical_grammar.variables[i].name
str_pool.resolve(lexical_grammar.variables[i].name)
);
Some(Symbol::terminal(i))
}
@ -219,13 +221,14 @@ fn populate_error_state(
{
debug!(
"error recovery - exclude token {} because of conflict with {}",
lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
str_pool.resolve(lexical_grammar.variables[i].name),
str_pool.resolve(lexical_grammar.variables[t.index].name)
);
continue;
}
debug!(
"error recovery - include token {}",
lexical_grammar.variables[i].name
str_pool.resolve(lexical_grammar.variables[i].name)
);
state
.terminal_entries
@ -335,6 +338,7 @@ fn identify_keywords(
word_token: Option<Symbol>,
token_conflict_map: &TokenConflictMap,
coincident_token_index: &CoincidentTokenIndex,
str_pool: &StrPool,
) -> TokenSet {
if word_token.is_none() {
return TokenSet::new();
@ -357,7 +361,7 @@ fn identify_keywords(
{
debug!(
"Keywords - add candidate {}",
lexical_grammar.variables[i].name
str_pool.resolve(lexical_grammar.variables[i].name)
);
Some(Symbol::terminal(i))
} else {
@ -376,8 +380,8 @@ fn identify_keywords(
{
debug!(
"Keywords - exclude {} because it matches the same string as {}",
lexical_grammar.variables[token.index].name,
lexical_grammar.variables[other_token.index].name
str_pool.resolve(lexical_grammar.variables[token.index].name),
str_pool.resolve(lexical_grammar.variables[other_token.index].name)
);
return false;
}
@ -419,8 +423,8 @@ fn identify_keywords(
) {
debug!(
"Keywords - exclude {} because of conflict with {}",
lexical_grammar.variables[token.index].name,
lexical_grammar.variables[other_index].name
str_pool.resolve(lexical_grammar.variables[token.index].name),
str_pool.resolve(lexical_grammar.variables[other_index].name)
);
return false;
}
@ -428,7 +432,7 @@ fn identify_keywords(
debug!(
"Keywords - include {}",
lexical_grammar.variables[token.index].name,
str_pool.resolve(lexical_grammar.variables[token.index].name),
);
true
})
@ -462,6 +466,7 @@ fn report_state_info<'a>(
lexical_grammar: &LexicalGrammar,
parse_table: &ParseTable,
parse_state_info: &[ParseStateInfo<'a>],
str_pool: &StrPool,
report_symbol_name: &'a str,
) {
let mut all_state_indices = BTreeSet::new();
@ -486,13 +491,13 @@ fn report_state_info<'a>(
let max_symbol_name_length = syntax_grammar
.variables
.iter()
.map(|v| v.name.len())
.map(|v| str_pool.resolve(v.name).len())
.max()
.unwrap();
for (symbol, states) in &symbols_with_state_indices {
info!(
"{:width$}\t{}",
syntax_grammar.variables[symbol.index].name,
str_pool.resolve(syntax_grammar.variables[symbol.index].name),
states.len(),
width = max_symbol_name_length
);
@ -505,7 +510,9 @@ fn report_state_info<'a>(
symbols_with_state_indices
.iter()
.find_map(|(symbol, state_indices)| {
if syntax_grammar.variables[symbol.index].name == report_symbol_name {
if str_pool.resolve(syntax_grammar.variables[symbol.index].name)
== report_symbol_name
{
Some(state_indices)
} else {
None
@ -528,11 +535,11 @@ fn report_state_info<'a>(
.iter()
.map(|symbol| {
if symbol.is_terminal() {
lexical_grammar.variables[symbol.index].name.clone()
str_pool.resolve(lexical_grammar.variables[symbol.index].name)
} else if symbol.is_external() {
syntax_grammar.external_tokens[symbol.index].name.clone()
str_pool.resolve(syntax_grammar.external_tokens[symbol.index].name)
} else {
syntax_grammar.variables[symbol.index].name.clone()
str_pool.resolve(syntax_grammar.variables[symbol.index].name)
}
})
.collect::<Vec<_>>()
@ -540,7 +547,7 @@ fn report_state_info<'a>(
);
info!(
"\nitems:\n{}",
item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar),
item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar, str_pool),
);
}
}

136
crates/generate/src/build_tables/build_parse_table.rs
View file

@ -15,9 +15,11 @@ use super::{
};
use crate::{
Diagnostic,
build_tables::item::prec_display,
grammars::{LexicalGrammar, PrecedenceEntry, ReservedWordSetId, SyntaxGrammar, VariableType},
node_types::VariableInfo,
rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
strpool::StrPool,
tables::{
FieldLocation, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable,
ParseTableEntry, ProductionInfo, ProductionInfoId,
@ -63,6 +65,7 @@ struct ParseTableBuilder<'a> {
non_terminal_extra_states: Vec<(Symbol, usize)>,
actual_conflicts: FxHashSet<Vec<Symbol>>,
parse_table: ParseTable,
str_pool: &'a StrPool,
}
pub type BuildTableResult<T> = Result<T, ParseTableBuilderError>;
@ -244,6 +247,7 @@ impl<'a> ParseTableBuilder<'a> {
lexical_grammar: &'a LexicalGrammar,
item_set_builder: ParseItemSetBuilder<'a>,
variable_info: &'a [VariableInfo],
str_pool: &'a StrPool,
) -> Self {
Self {
syntax_grammar,
@ -263,6 +267,7 @@ impl<'a> ParseTableBuilder<'a> {
production_infos: Vec::new(),
max_aliased_production_length: 1,
},
str_pool,
}
}
@ -299,16 +304,19 @@ impl<'a> ParseTableBuilder<'a> {
.iter()
.filter(|s| s.is_non_terminal())
{
let variable = &self.syntax_grammar.variables[extra_non_terminal.index];
for production in &variable.productions {
for prod_id in self
.syntax_grammar
.variable_prod_ids(extra_non_terminal.index)
{
let production = self.syntax_grammar.production(prod_id);
non_terminal_extra_item_sets_by_first_terminal
.entry(production.first_symbol().unwrap())
.or_insert_with(ParseItemSet::default)
.insert(ParseItem {
variable_index: extra_non_terminal.index as u32,
production,
prod_id,
step_index: 1,
keys: self.item_set_builder.key_map.keys_for(production),
keys: self.item_set_builder.key_map.keys_for(prod_id),
has_preceding_inherited_fields: false,
})
.lookaheads
@ -426,7 +434,7 @@ impl<'a> ParseTableBuilder<'a> {
// If the item is unfinished, then this state has a transition for the item's
// next symbol. Advance the item to its next step and insert the resulting
// item into the successor item set.
if let Some(next_symbol) = item.symbol() {
if let Some(next_symbol) = item.symbol(self.syntax_grammar) {
let mut successor = item.successor();
let successor_set = if next_symbol.is_non_terminal() {
let variable = &self.syntax_grammar.variables[next_symbol.index];
@ -489,13 +497,13 @@ impl<'a> ParseTableBuilder<'a> {
ParseAction::Reduce {
symbol,
child_count: item.step_index as u16,
dynamic_precedence: item.production.dynamic_precedence,
dynamic_precedence: item.production(self.syntax_grammar).dynamic_precedence,
production_id: production_id as u16,
}
};
let precedence = item.precedence();
let associativity = item.associativity();
let precedence = item.precedence(self.syntax_grammar);
let associativity = item.associativity(self.syntax_grammar);
for lookahead in lookaheads.iter() {
let table_entry = self.parse_table.states[state_id]
.terminal_entries
@ -513,7 +521,7 @@ impl<'a> ParseTableBuilder<'a> {
self.syntax_grammar,
precedence,
&[symbol],
&reduction_info.precedence,
reduction_info.precedence,
&reduction_info.symbols,
) {
Ordering::Greater => {
@ -530,7 +538,7 @@ impl<'a> ParseTableBuilder<'a> {
}
}
reduction_info.precedence.clone_from(precedence);
reduction_info.precedence = precedence;
if let Err(i) = reduction_info.symbols.binary_search(&symbol) {
reduction_info.symbols.insert(i, symbol);
}
@ -627,9 +635,9 @@ impl<'a> ParseTableBuilder<'a> {
let parent_symbol_names = parent_symbols
.iter()
.map(|&variable_index| {
self.syntax_grammar.variables[variable_index as usize]
.name
.clone()
self.str_pool
.resolve(self.syntax_grammar.variables[variable_index as usize].name)
.to_string()
})
.collect::<Vec<_>>();
@ -678,9 +686,9 @@ impl<'a> ParseTableBuilder<'a> {
.entries
.iter()
.filter_map(|entry| {
if let Some(next_step) = entry.item.step() {
if next_step.symbol == keyword_capture_token {
Some(next_step.reserved_word_set_id)
if let Some(next_step) = entry.item.step(self.syntax_grammar) {
if next_step.symbol() == keyword_capture_token {
Some(ReservedWordSetId(usize::from(next_step.reserved)))
} else {
None
}
@ -721,17 +729,17 @@ impl<'a> ParseTableBuilder<'a> {
// REDUCE-REDUCE conflicts where all actions have the *same*
// precedence, and there can still be SHIFT/REDUCE conflicts.
let mut considered_associativity = false;
let mut shift_precedence = Vec::<(&Precedence, Symbol)>::new();
let mut shift_precedence = Vec::<(Precedence, Symbol)>::new();
let mut conflicting_items = BTreeSet::new();
for ParseItemSetEntry {
item, lookaheads, ..
} in &item_set.entries
{
if let Some(step) = item.step() {
if let Some(step) = item.step(self.syntax_grammar) {
if item.step_index > 0
&& self
.item_set_builder
.first_set(&step.symbol)
.first_set(&step.symbol())
.contains(&conflicting_lookahead)
{
if item.variable_index != u32::MAX {
@ -739,7 +747,7 @@ impl<'a> ParseTableBuilder<'a> {
}
let p = (
item.precedence(),
item.precedence(self.syntax_grammar),
Symbol::non_terminal(item.variable_index as usize),
);
if let Err(i) = shift_precedence.binary_search(&p) {
@ -778,7 +786,7 @@ impl<'a> ParseTableBuilder<'a> {
self.syntax_grammar,
p.0,
&[p.1],
&reduction_info.precedence,
reduction_info.precedence,
&reduction_info.symbols,
) {
Ordering::Greater => shift_is_more = true,
@ -901,23 +909,29 @@ impl<'a> ParseTableBuilder<'a> {
.map(|symbol| self.symbol_name(symbol))
.collect::<Vec<_>>();
let variable_name = self.syntax_grammar.variables[item.variable_index as usize]
.name
.clone();
let variable_name = self
.str_pool
.resolve(self.syntax_grammar.variables[item.variable_index as usize].name)
.to_string();
let production_step_symbols = item
.production
.production(self.syntax_grammar)
.steps
.iter()
.map(|step| self.symbol_name(&step.symbol))
.map(|step| self.symbol_name(&step.symbol()))
.collect::<Vec<_>>();
let precedence = match item.precedence() {
let precedence = match item.precedence(self.syntax_grammar) {
Precedence::None => None,
_ => Some(item.precedence().to_string()),
_ => Some(prec_display(
item.precedence(self.syntax_grammar),
self.str_pool,
)),
};
let associativity = item.associativity().map(|assoc| format!("{assoc:?}"));
let associativity = item
.associativity(self.syntax_grammar)
.map(|assoc| format!("{assoc:?}"));
Interpretation {
preceding_symbols,
@ -1000,17 +1014,17 @@ impl<'a> ParseTableBuilder<'a> {
fn compare_precedence(
grammar: &SyntaxGrammar,
left: &Precedence,
left: Precedence,
left_symbols: &[Symbol],
right: &Precedence,
right: Precedence,
right_symbols: &[Symbol],
) -> Ordering {
let precedence_entry_matches =
|entry: &PrecedenceEntry, precedence: &Precedence, symbols: &[Symbol]| -> bool {
|entry: &PrecedenceEntry, precedence: Precedence, symbols: &[Symbol]| -> bool {
match entry {
PrecedenceEntry::Name(n) => {
if let Precedence::Name(p) = precedence {
n == p
*n == p
} else {
false
}
@ -1024,9 +1038,9 @@ impl<'a> ParseTableBuilder<'a> {
match (left, right) {
// Integer precedences can be compared to other integer precedences,
// and to the default precedence, which is zero.
(Precedence::Integer(l), Precedence::Integer(r)) if *l != 0 || *r != 0 => l.cmp(r),
(Precedence::Integer(l), Precedence::None) if *l != 0 => l.cmp(&0),
(Precedence::None, Precedence::Integer(r)) if *r != 0 => 0.cmp(r),
(Precedence::Integer(l), Precedence::Integer(r)) if l != 0 || r != 0 => l.cmp(&r),
(Precedence::Integer(l), Precedence::None) if l != 0 => l.cmp(&0),
(Precedence::None, Precedence::Integer(r)) if r != 0 => 0.cmp(&r),
// Named precedences can be compared to other named precedences.
_ => grammar
@ -1066,7 +1080,7 @@ impl<'a> ParseTableBuilder<'a> {
.iter()
.filter_map(|ParseItemSetEntry { item, .. }| {
let variable_index = item.variable_index as usize;
if item.symbol() == Some(symbol)
if item.symbol(self.syntax_grammar) == Some(symbol)
&& !self.syntax_grammar.variables[variable_index].is_auxiliary()
{
Some(Symbol::non_terminal(variable_index))
@ -1087,12 +1101,17 @@ impl<'a> ParseTableBuilder<'a> {
field_map: BTreeMap::new(),
};
for (i, step) in item.production.steps.iter().enumerate() {
production_info.alias_sequence.push(step.alias.clone());
if let Some(field_name) = &step.field_name {
for (i, step) in item
.production(self.syntax_grammar)
.steps
.iter()
.enumerate()
{
production_info.alias_sequence.push(step.alias());
if let Some(field_name) = step.field() {
production_info
.field_map
.entry(field_name.clone())
.entry(field_name)
.or_default()
.push(FieldLocation {
index: i,
@ -1100,16 +1119,16 @@ impl<'a> ParseTableBuilder<'a> {
});
}
if step.symbol.kind == SymbolType::NonTerminal
&& !self.syntax_grammar.variables[step.symbol.index]
if step.symbol().kind == SymbolType::NonTerminal
&& !self.syntax_grammar.variables[step.symbol().index]
.kind
.is_visible()
{
let info = &self.variable_info[step.symbol.index];
for field_name in info.fields.keys() {
let info = &self.variable_info[step.symbol().index];
for &field_name in info.fields.keys() {
production_info
.field_map
.entry(field_name.clone())
.entry(field_name)
.or_default()
.push(FieldLocation {
index: i,
@ -1123,8 +1142,11 @@ impl<'a> ParseTableBuilder<'a> {
production_info.alias_sequence.pop();
}
if item.production.steps.len() > self.parse_table.max_aliased_production_length {
self.parse_table.max_aliased_production_length = item.production.steps.len();
if item.production(self.syntax_grammar).steps.len()
> self.parse_table.max_aliased_production_length
{
self.parse_table.max_aliased_production_length =
item.production(self.syntax_grammar).steps.len();
}
if let Some(index) = self
@ -1143,16 +1165,20 @@ impl<'a> ParseTableBuilder<'a> {
fn symbol_name(&self, symbol: &Symbol) -> String {
match symbol.kind {
SymbolType::End | SymbolType::EndOfNonTerminalExtra => "EOF".to_string(),
SymbolType::External => self.syntax_grammar.external_tokens[symbol.index]
.name
.clone(),
SymbolType::NonTerminal => self.syntax_grammar.variables[symbol.index].name.clone(),
SymbolType::External => self
.str_pool
.resolve(self.syntax_grammar.external_tokens[symbol.index].name)
.to_string(),
SymbolType::NonTerminal => self
.str_pool
.resolve(self.syntax_grammar.variables[symbol.index].name)
.to_string(),
SymbolType::Terminal => {
let variable = &self.lexical_grammar.variables[symbol.index];
if variable.kind == VariableType::Named {
variable.name.clone()
self.str_pool.resolve(variable.name).to_string()
} else {
format!("'{}'", variable.name)
format!("'{}'", self.str_pool.resolve(variable.name))
}
}
}
@ -1164,6 +1190,7 @@ pub fn build_parse_table<'a>(
lexical_grammar: &'a LexicalGrammar,
item_set_builder: ParseItemSetBuilder<'a>,
variable_info: &'a [VariableInfo],
str_pool: &'a StrPool,
diagnostics: &mut Vec<Diagnostic>,
) -> BuildTableResult<(ParseTable, Vec<ParseStateInfo<'a>>)> {
ParseTableBuilder::new(
@ -1171,6 +1198,7 @@ pub fn build_parse_table<'a>(
lexical_grammar,
item_set_builder,
variable_info,
str_pool,
)
.build(diagnostics)
}

31
crates/generate/src/build_tables/coincident_tokens.rs
View file

@ -1,12 +1,11 @@
use std::fmt;
use crate::{
grammars::LexicalGrammar,
rules::Symbol,
strpool::StrPool,
tables::{ParseStateId, ParseTable},
};
pub struct CoincidentTokenIndex<'a> {
pub struct CoincidentTokenIndex {
entries: Vec<Vec<ParseStateId>>,
/// Flat bitset for fast [`contains()`](Self::contains) checks. Indexed as `a * n + b`
/// (both `(a,b)` and `(b,a)` bits are set, so no min/max normalization needed).
@ -15,18 +14,16 @@ pub struct CoincidentTokenIndex<'a> {
/// Row `a` spans `[a * row_words .. (a+1) * row_words]`.
/// Bit `b` is set iff tokens `a` and `b` are coincident in some parse state.
pub(crate) row_bits: Vec<u64>,
grammar: &'a LexicalGrammar,
n: usize,
}
impl<'a> CoincidentTokenIndex<'a> {
impl<'a> CoincidentTokenIndex {
#[must_use]
pub fn new(table: &ParseTable, lexical_grammar: &'a LexicalGrammar) -> Self {
let n = lexical_grammar.variables.len();
let row_words = n.div_ceil(64);
let mut result = Self {
n,
grammar: lexical_grammar,
entries: vec![Vec::new(); n * n],
contains_bits: vec![0u64; (n * n).div_ceil(64)],
row_bits: vec![0u64; n * row_words],
@ -85,18 +82,26 @@ impl<'a> CoincidentTokenIndex<'a> {
}
}
impl fmt::Debug for CoincidentTokenIndex<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
#[expect(dead_code, reason = "Debugging aid")]
pub struct CoincidentTokenIndexDisplay<'a>(CoincidentTokenIndex, &'a LexicalGrammar, StrPool);
impl std::fmt::Debug for CoincidentTokenIndexDisplay<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writeln!(f, "CoincidentTokenIndex {{")?;
for i in 0..self.n {
for i in 0..self.0.n {
let mut coincident = Vec::new();
for j in 0..self.n {
if self.contains(Symbol::terminal(i), Symbol::terminal(j)) {
coincident.push(&self.grammar.variables[j].name);
for j in 0..self.0.n {
if self.0.contains(Symbol::terminal(i), Symbol::terminal(j)) {
coincident.push(self.2.resolve(self.1.variables[j].name));
}
}
if !coincident.is_empty() {
writeln!(f, " {}: {:?},", self.grammar.variables[i].name, coincident)?;
writeln!(
f,
" {}: {:?},",
self.2.resolve(self.1.variables[i].name),
coincident
)?;
}
}
write!(f, "}}")?;

318
crates/generate/src/build_tables/item.rs
View file

@ -2,33 +2,33 @@ use std::{
cmp::Ordering,
fmt,
hash::{Hash, Hasher},
sync::LazyLock,
};
use rustc_hash::FxHashMap;
use crate::{
grammars::{
InlinedProductionMap, LexicalGrammar, NO_RESERVED_WORDS, Production, ProductionStep,
ReservedWordSetId, SyntaxGrammar,
},
grammars::{LexicalGrammar, ProdRef, ProductionStep, ReservedWordSetId, SyntaxGrammar},
rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
strpool::StrPool,
};
static START_PRODUCTION: LazyLock<Production> = LazyLock::new(|| Production {
dynamic_precedence: 0,
steps: vec![ProductionStep {
symbol: Symbol {
index: 0,
kind: SymbolType::NonTerminal,
},
precedence: Precedence::None,
associativity: None,
alias: None,
field_name: None,
reserved_word_set_id: NO_RESERVED_WORDS,
}],
});
const START_STEPS: [ProductionStep; 1] = [ProductionStep {
sym_index: 0,
prec_val: 0,
alias: 0,
field: 0,
reserved: ProductionStep::NO_RESERVED_WORDS,
flags: SymbolType::NonTerminal as u8,
}];
pub const START_PRODUCTION_ID: u32 = u32::MAX;
const fn start_production() -> ProdRef<'static> {
ProdRef {
steps: &START_STEPS,
dynamic_precedence: 0,
}
}
/// Precomputed identity keys for one `(production, dot)` pair.
///
@ -47,40 +47,40 @@ pub struct DotKeys {
/// Identity keys for every `(production, dot)` in a grammar (all grammar productions,
/// every inlined production, and the augmented start production).
pub struct ItemKeyMap {
keys: FxHashMap<*const Production, Box<[DotKeys]>>,
keys: Vec<Box<[DotKeys]>>,
start: Box<[DotKeys]>,
}
impl ItemKeyMap {
pub fn new(grammar: &SyntaxGrammar, inlines: &InlinedProductionMap) -> Self {
let mut prods = Vec::<&Production>::with_capacity(
1 + grammar.variables.len() + inlines.productions.len(),
);
prods.push(&START_PRODUCTION);
for var in &grammar.variables {
prods.extend(var.productions.iter());
}
prods.extend(inlines.productions.iter());
pub fn new(grammar: &SyntaxGrammar, str_pool: &StrPool) -> Self {
let prod = |slot: usize| -> ProdRef {
if slot == 0 {
start_production()
} else {
grammar.production(slot as u32 - 1)
}
};
let slot_count = grammar.productions.len() + 1;
let mut contents: Vec<(u32, u32)> = Vec::with_capacity(prods.len());
for (pi, p) in prods.iter().enumerate() {
for dot in 0..=p.steps.len() {
let mut contents: Vec<(u32, u32)> = Vec::with_capacity(slot_count);
for pi in 0..slot_count {
for dot in 0..=prod(pi).steps.len() {
contents.push((pi as u32, dot as u32));
}
}
let content = |&(pi, dot): &(u32, u32)| ItemContent {
production: prods[pi as usize],
production: prod(pi as usize),
dot: dot as usize,
str_pool,
};
contents.sort_unstable_by(|a, b| content(a).cmp(&content(b)));
let mut slot_keys: Vec<Box<[DotKeys]>> = prods
.iter()
.map(|p| vec![DotKeys::default(); p.steps.len() + 1].into_boxed_slice())
.collect();
let mut slot_keys: Vec<Box<[DotKeys]>> = (0..slot_count)
.map(|pi| vec![DotKeys::default(); prod(pi).steps.len() + 1].into_boxed_slice())
.collect::<Vec<_>>();
// Dense ids in sorted order: equal content shares an id, distinct content gets the next up
let mut cmp_id = 0u32;
let mut prev: Option<(u32, u32)> = None;
let mut cmp_id = 0;
let mut prev = None;
for &(pi, dot) in &contents {
if let Some(p) = prev
&& content(&p) != content(&(pi, dot))
@ -91,33 +91,33 @@ impl ItemKeyMap {
prev = Some((pi, dot));
}
// Refine each cmp class by preceding symbols: read only under
// Refine each cmp class by preceding symbols: read only under
// `has_preceding_inherited_fields`.
let mut sym_classes: FxHashMap<(u32, Vec<Symbol>), u32> = FxHashMap::default();
let mut sym_classes = FxHashMap::default();
for &(pi, dot) in &contents {
let syms: Vec<Symbol> = prods[pi as usize].steps[..dot as usize]
let syms = prod(pi as usize).steps[..dot as usize]
.iter()
.map(|s| s.symbol)
.collect();
.map(|s| s.symbol())
.collect::<Vec<_>>();
let next = sym_classes.len() as u32;
let keys = &mut slot_keys[pi as usize][dot as usize];
keys.eq_with_syms = *sym_classes.entry((keys.cmp, syms)).or_insert(next);
}
let mut slots = slot_keys.into_iter();
let start = slots.next().unwrap();
let keys = prods[1..]
.iter()
.zip(slots)
.map(|(p, ks)| (core::ptr::from_ref::<Production>(p), ks))
.collect();
Self { keys, start }
let start = slot_keys.remove(0);
Self {
keys: slot_keys,
start,
}
}
/// The keys slice (indexed by `dot`) for a production of this grammar.
pub fn keys_for(&self, production: &Production) -> &[DotKeys] {
&self.keys[&core::ptr::from_ref::<Production>(production)]
pub fn keys_for(&self, id: u32) -> &[DotKeys] {
if id == START_PRODUCTION_ID {
&self.start
} else {
&self.keys[id as usize]
}
}
pub fn start_keys(&self) -> &[DotKeys] {
@ -129,28 +129,67 @@ impl ItemKeyMap {
/// `(production, dot)` directly. Ranking the set of all keys by this order makes
/// the dense `cmp` order-preserving. Item ordering only ever compares same-dot pairs,
/// and the dot comparison keeps the order total across dots so ranks are well defined.
#[derive(Eq)]
struct ItemContent<'a> {
production: &'a Production,
production: ProdRef<'a>,
dot: usize,
str_pool: &'a StrPool,
}
impl Eq for ItemContent<'_> {}
impl ItemContent<'_> {
fn prec(&self) -> &Precedence {
fn prec(&self) -> Precedence {
if self.dot > 0 {
&self.production.steps[self.dot - 1].precedence
self.production.steps[self.dot - 1].precedence()
} else {
&Precedence::None
Precedence::None
}
}
fn assoc(&self) -> Option<Associativity> {
if self.dot > 0 {
self.production.steps[self.dot - 1].associativity
self.production.steps[self.dot - 1].associativity()
} else {
None
}
}
fn prec_cmp(&self, a: Precedence, b: Precedence) -> Ordering {
match (a, b) {
(Precedence::None, Precedence::None) => Ordering::Equal,
(Precedence::Integer(a), Precedence::Integer(b)) => a.cmp(&b),
(Precedence::Name(a), Precedence::Name(b)) => {
self.str_pool.resolve(a).cmp(self.str_pool.resolve(b))
}
(Precedence::None, _) | (Precedence::Integer(_), Precedence::Name(_)) => Ordering::Less,
(_, Precedence::None) | (Precedence::Name(_), Precedence::Integer(_)) => {
Ordering::Greater
}
}
}
fn alias_cmp(&self, a: ProductionStep, b: ProductionStep) -> Ordering {
let key = |step: ProductionStep| {
step.alias()
.map(|alias| (self.str_pool.resolve(alias.value), alias.is_named))
};
key(a).cmp(&key(b))
}
fn field_cmp(&self, a: ProductionStep, b: ProductionStep) -> Ordering {
let key = |step: ProductionStep| step.field().map(|field| self.str_pool.resolve(field));
key(a).cmp(&key(b))
}
fn step_cmp(&self, a: ProductionStep, b: ProductionStep) -> Ordering {
a.symbol()
.cmp(&b.symbol())
.then_with(|| self.prec_cmp(a.precedence(), b.precedence()))
.then_with(|| a.associativity().cmp(&b.associativity()))
.then_with(|| self.alias_cmp(a, b))
.then_with(|| self.field_cmp(a, b))
.then_with(|| a.reserved.cmp(&b.reserved))
}
}
impl Ord for ItemContent<'_> {
@ -164,20 +203,17 @@ impl Ord for ItemContent<'_> {
.len()
.cmp(&other.production.steps.len())
})
.then_with(|| self.prec().cmp(other.prec()))
.then_with(|| self.prec_cmp(self.prec(), other.prec()))
.then_with(|| self.assoc().cmp(&other.assoc()))
.then_with(|| self.dot.cmp(&other.dot))
.then_with(|| {
let steps = self.production.steps.iter().zip(&other.production.steps);
for (i, (sa, sb)) in steps.enumerate() {
let steps = self.production.steps.iter().zip(other.production.steps);
for (i, (&sa, &sb)) in steps.enumerate() {
let o = if i < self.dot {
sa.alias
.cmp(&sb.alias)
.then_with(|| sa.field_name.cmp(&sb.field_name))
self.alias_cmp(sa, sb).then_with(|| self.field_cmp(sa, sb))
} else {
sa.cmp(sb)
self.step_cmp(sa, sb)
};
if o != Ordering::Equal {
return o;
}
@ -206,8 +242,8 @@ pub struct ParseItem<'a> {
pub variable_index: u32,
/// The number of symbols that have already been matched.
pub step_index: u32,
/// The production being matched.
pub production: &'a Production,
/// The id of the production being matched.
pub prod_id: u32,
/// The `production`'s identity keys, indexed by `step_index`.
pub keys: &'a [DotKeys],
/// A boolean indicating whether any of the already-matched children were
@ -254,18 +290,21 @@ pub struct ParseItemDisplay<'a>(
pub &'a ParseItem<'a>,
pub &'a SyntaxGrammar,
pub &'a LexicalGrammar,
pub &'a StrPool,
);
pub struct TokenSetDisplay<'a>(
pub &'a TokenSet,
pub &'a SyntaxGrammar,
pub &'a LexicalGrammar,
pub &'a StrPool,
);
pub struct ParseItemSetDisplay<'a>(
pub &'a ParseItemSet<'a>,
pub &'a SyntaxGrammar,
pub &'a LexicalGrammar,
pub &'a StrPool,
);
impl<'a> ParseItem<'a> {
@ -273,38 +312,52 @@ impl<'a> ParseItem<'a> {
pub fn start(key_map: &'a ItemKeyMap) -> Self {
ParseItem {
variable_index: u32::MAX,
production: &START_PRODUCTION,
prod_id: START_PRODUCTION_ID,
keys: key_map.start_keys(),
step_index: 0,
has_preceding_inherited_fields: false,
}
}
/// The production being matched.
#[must_use]
pub fn step(&self) -> Option<&'a ProductionStep> {
self.production.steps.get(self.step_index as usize)
pub fn production<'g>(&self, grammar: &'g SyntaxGrammar) -> ProdRef<'g> {
if self.prod_id == START_PRODUCTION_ID {
start_production()
} else {
grammar.production(self.prod_id)
}
}
#[must_use]
pub fn symbol(&self) -> Option<Symbol> {
self.step().map(|step| step.symbol)
pub fn step(&self, grammar: &SyntaxGrammar) -> Option<ProductionStep> {
self.production(grammar)
.steps
.get(self.step_index as usize)
.copied()
}
#[must_use]
pub fn associativity(&self) -> Option<Associativity> {
self.prev_step().and_then(|step| step.associativity)
pub fn symbol(&self, grammar: &SyntaxGrammar) -> Option<Symbol> {
self.step(grammar).map(ProductionStep::symbol)
}
#[must_use]
pub fn precedence(&self) -> &Precedence {
self.prev_step()
.map_or(&Precedence::None, |step| &step.precedence)
pub fn associativity(&self, grammar: &SyntaxGrammar) -> Option<Associativity> {
self.prev_step(grammar)
.and_then(ProductionStep::associativity)
}
#[must_use]
pub fn prev_step(&self) -> Option<&'a ProductionStep> {
pub fn precedence(&self, grammar: &SyntaxGrammar) -> Precedence {
self.prev_step(grammar)
.map_or(Precedence::None, ProductionStep::precedence)
}
#[must_use]
pub fn prev_step(&self, grammar: &SyntaxGrammar) -> Option<ProductionStep> {
if self.step_index > 0 {
Some(&self.production.steps[self.step_index as usize - 1])
Some(self.production(grammar).steps[self.step_index as usize - 1])
} else {
None
}
@ -312,7 +365,7 @@ impl<'a> ParseItem<'a> {
#[must_use]
pub const fn is_done(&self) -> bool {
self.step_index as usize == self.production.steps.len()
self.step_index as usize + 1 == self.keys.len()
}
#[must_use]
@ -325,7 +378,7 @@ impl<'a> ParseItem<'a> {
pub const fn successor(&self) -> Self {
ParseItem {
variable_index: self.variable_index,
production: self.production,
prod_id: self.prod_id,
keys: self.keys,
step_index: self.step_index + 1,
has_preceding_inherited_fields: self.has_preceding_inherited_fields,
@ -335,13 +388,9 @@ impl<'a> ParseItem<'a> {
/// Create an item identical to this one, but with a different production.
/// This is used when dynamically "inlining" certain symbols in a production.
#[must_use]
pub const fn substitute_production(
&self,
production: &'a Production,
keys: &'a [DotKeys],
) -> Self {
pub const fn substitute_production(&self, prod_id: u32, keys: &'a [DotKeys]) -> Self {
let mut result = *self;
result.production = production;
result.prod_id = prod_id;
result.keys = keys;
result
}
@ -380,6 +429,14 @@ impl<'a> ParseItemSet<'a> {
}
}
pub fn prec_display(prec: Precedence, str_pool: &StrPool) -> String {
match prec {
Precedence::None => "none".to_string(),
Precedence::Integer(i) => i.to_string(),
Precedence::Name(sid) => format!("'{}'", str_pool.resolve(sid)),
}
}
impl fmt::Display for ParseItemDisplay<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
if self.0.is_augmented() {
@ -388,60 +445,71 @@ impl fmt::Display for ParseItemDisplay<'_> {
write!(
f,
"{} →",
self.1.variables[self.0.variable_index as usize].name
self.3
.resolve(self.1.variables[self.0.variable_index as usize].name)
)?;
}
for (i, step) in self.0.production.steps.iter().enumerate() {
let production = self.0.production(self.1);
for (i, step) in production.steps.iter().enumerate() {
let symbol = step.symbol();
if i == self.0.step_index as usize {
write!(f, " •")?;
if !step.precedence.is_none()
|| step.associativity.is_some()
|| step.reserved_word_set_id != ReservedWordSetId::default()
if step.precedence() != Precedence::None
|| step.associativity().is_some()
|| step.reserved != 0
{
write!(f, " (")?;
if !step.precedence.is_none() {
write!(f, " {}", step.precedence)?;
if step.precedence() != Precedence::None {
write!(f, " {}", prec_display(step.precedence(), self.3))?;
}
if let Some(associativity) = step.associativity {
if let Some(associativity) = step.associativity() {
write!(f, " {associativity:?}")?;
}
if step.reserved_word_set_id != ReservedWordSetId::default() {
write!(f, "reserved: {}", step.reserved_word_set_id)?;
if step.reserved != 0 {
write!(f, "reserved: {}", step.reserved)?;
}
write!(f, " )")?;
}
}
write!(f, " ")?;
if step.symbol.is_terminal() {
if let Some(variable) = self.2.variables.get(step.symbol.index) {
write!(f, "{}", variable.name)?;
if symbol.is_terminal() {
if let Some(variable) = self.2.variables.get(symbol.index) {
write!(f, "{}", self.3.resolve(variable.name))?;
} else {
write!(f, "terminal-{}", step.symbol.index)?;
write!(f, "terminal-{}", symbol.index)?;
}
} else if step.symbol.is_external() {
write!(f, "{}", self.1.external_tokens[step.symbol.index].name)?;
} else if symbol.is_external() {
write!(
f,
"{}",
self.3.resolve(self.1.external_tokens[symbol.index].name)
)?;
} else {
write!(f, "{}", self.1.variables[step.symbol.index].name)?;
write!(f, "{}", self.3.resolve(self.1.variables[symbol.index].name))?;
}
if let Some(alias) = &step.alias {
write!(f, "@{}", alias.value)?;
if let Some(alias) = &step.alias() {
write!(f, "@{}", self.3.resolve(alias.value))?;
}
}
if self.0.is_done() {
write!(f, " •")?;
if let Some(step) = self.0.production.steps.last() {
if let Some(associativity) = step.associativity {
if step.precedence.is_none() {
if let Some(&step) = production.steps.last() {
if let Some(associativity) = step.associativity() {
if step.precedence() == Precedence::None {
write!(f, " ({associativity:?})")?;
} else {
write!(f, " ({} {associativity:?})", step.precedence)?;
write!(
f,
" ({} {associativity:?})",
prec_display(step.precedence(), self.3)
)?;
}
} else if !step.precedence.is_none() {
write!(f, " ({})", step.precedence)?;
} else if step.precedence() != Precedence::None {
write!(f, " ({})", prec_display(step.precedence(), self.3))?;
}
}
}
@ -486,14 +554,22 @@ impl fmt::Display for TokenSetDisplay<'_> {
if symbol.is_terminal() {
if let Some(variable) = self.2.variables.get(symbol.index) {
write!(f, "{}", display_variable_name(&variable.name))?;
write!(
f,
"{}",
display_variable_name(self.3.resolve(variable.name))
)?;
} else {
write!(f, "terminal-{}", symbol.index)?;
}
} else if symbol.is_external() {
write!(f, "{}", self.1.external_tokens[symbol.index].name)?;
write!(
f,
"{}",
self.3.resolve(self.1.external_tokens[symbol.index].name)
)?;
} else {
write!(f, "{}", self.1.variables[symbol.index].name)?;
write!(f, "{}", self.3.resolve(self.1.variables[symbol.index].name))?;
}
}
write!(f, "]")?;
@ -507,8 +583,8 @@ impl fmt::Display for ParseItemSetDisplay<'_> {
write!(
f,
"{}\t{}",
ParseItemDisplay(&entry.item, self.1, self.2),
TokenSetDisplay(&entry.lookaheads, self.1, self.2),
ParseItemDisplay(&entry.item, self.1, self.2, self.3),
TokenSetDisplay(&entry.lookaheads, self.1, self.2, self.3),
)?;
if entry.following_reserved_word_set != ReservedWordSetId::default() {
write!(

127
crates/generate/src/build_tables/item_set_builder.rs
View file

@ -8,6 +8,7 @@ use super::item::{
use crate::{
grammars::{InlinedProductionMap, LexicalGrammar, ReservedWordSetId, SyntaxGrammar},
rules::{Symbol, SymbolType, TokenSet},
strpool::StrPool,
};
#[derive(Clone, Debug, PartialEq, Eq)]
@ -25,7 +26,6 @@ struct FollowSetInfo {
pub struct ParseItemSetBuilder<'a> {
syntax_grammar: &'a SyntaxGrammar,
lexical_grammar: &'a LexicalGrammar,
first_sets: FxHashMap<Symbol, TokenSet>,
reserved_first_sets: FxHashMap<Symbol, ReservedWordSetId>,
last_sets: FxHashMap<Symbol, TokenSet>,
@ -50,7 +50,6 @@ impl<'a> ParseItemSetBuilder<'a> {
) -> Self {
let mut result = Self {
syntax_grammar,
lexical_grammar,
first_sets: FxHashMap::default(),
reserved_first_sets: FxHashMap::default(),
last_sets: FxHashMap::default(),
@ -107,14 +106,16 @@ impl<'a> ParseItemSetBuilder<'a> {
symbols_to_process.clear();
symbols_to_process.push(symbol);
while let Some(sym) = symbols_to_process.pop() {
for production in &syntax_grammar.variables[sym.index].productions {
if let Some(step) = production.steps.first() {
if step.symbol.is_terminal() || step.symbol.is_external() {
first_set.insert(step.symbol);
} else if processed_non_terminals.insert(step.symbol) {
symbols_to_process.push(step.symbol);
for prod_id in syntax_grammar.variable_prod_ids(sym.index) {
if let Some(step) = syntax_grammar.production(prod_id).steps.first() {
let symbol = step.symbol();
if symbol.is_terminal() || symbol.is_external() {
first_set.insert(symbol);
} else if processed_non_terminals.insert(symbol) {
symbols_to_process.push(symbol);
}
*reserved_first_set = (*reserved_first_set).max(step.reserved_word_set_id);
*reserved_first_set =
(*reserved_first_set).max(ReservedWordSetId(step.reserved as usize));
}
}
}
@ -125,12 +126,13 @@ impl<'a> ParseItemSetBuilder<'a> {
symbols_to_process.clear();
symbols_to_process.push(symbol);
while let Some(sym) = symbols_to_process.pop() {
for production in &syntax_grammar.variables[sym.index].productions {
if let Some(step) = production.steps.last() {
if step.symbol.is_terminal() || step.symbol.is_external() {
last_set.insert(step.symbol);
} else if processed_non_terminals.insert(step.symbol) {
symbols_to_process.push(step.symbol);
for prod_id in syntax_grammar.variable_prod_ids(sym.index) {
if let Some(step) = syntax_grammar.production(prod_id).steps.last() {
let symbol = step.symbol();
if symbol.is_terminal() || symbol.is_external() {
last_set.insert(symbol);
} else if processed_non_terminals.insert(symbol) {
symbols_to_process.push(symbol);
}
}
}
@ -191,15 +193,16 @@ impl<'a> ParseItemSetBuilder<'a> {
continue;
}
for production in &syntax_grammar.variables[sym_ix].productions {
for prod_id in syntax_grammar.variable_prod_ids(sym_ix) {
let production = syntax_grammar.production(prod_id);
if let Some(symbol) = production.first_symbol()
&& symbol.is_non_terminal()
{
if let Some(next_step) = production.steps.get(1) {
stack.push((
symbol.index,
&result.first_sets[&next_step.symbol],
result.reserved_first_sets[&next_step.symbol],
&result.first_sets[&next_step.symbol()],
result.reserved_first_sets[&next_step.symbol()],
false,
));
} else {
@ -218,32 +221,26 @@ impl<'a> ParseItemSetBuilder<'a> {
// lookahead info, as *additions* associated with non-terminal `i`.
let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
for (&variable_index, follow_set_info) in &follow_set_info_by_non_terminal {
let variable = &syntax_grammar.variables[variable_index];
let non_terminal = Symbol::non_terminal(variable_index);
let variable_index = variable_index as u32;
if syntax_grammar.variables_to_inline.contains(&non_terminal) {
continue;
}
for production in &variable.productions {
for prod_id in syntax_grammar.variable_prod_ids(variable_index as usize) {
let item = ParseItem {
variable_index,
production,
keys: key_map.keys_for(production),
prod_id,
keys: key_map.keys_for(prod_id),
step_index: 0,
has_preceding_inherited_fields: false,
};
if let Some(inlined_productions) =
inlines.inlined_productions(item.production, item.step_index)
{
for production in inlined_productions {
if let Some(ids) = inlines.inlined_prod_ids(item.prod_id, item.step_index) {
for &id in ids {
find_or_push(
additions_for_non_terminal,
TransitiveClosureAddition {
item: item.substitute_production(
production,
key_map.keys_for(production),
),
item: item.substitute_production(id, key_map.keys_for(id)),
info: follow_set_info.clone(),
},
);
@ -268,18 +265,17 @@ impl<'a> ParseItemSetBuilder<'a> {
pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
let mut result = ParseItemSet::default();
for entry in &item_set.entries {
if let Some(productions) = self
if let Some(ids) = self
.inlines
.inlined_productions(entry.item.production, entry.item.step_index)
.inlined_prod_ids(entry.item.prod_id, entry.item.step_index)
{
for production in productions {
for &id in ids {
self.add_item(
&mut result,
&ParseItemSetEntry {
item: entry.item.substitute_production(
production,
self.key_map.keys_for(production),
),
item: entry
.item
.substitute_production(id, self.key_map.keys_for(id)),
lookaheads: entry.lookaheads.clone(),
following_reserved_word_set: entry.following_reserved_word_set,
},
@ -309,23 +305,23 @@ impl<'a> ParseItemSetBuilder<'a> {
}
fn add_item(&self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
if let Some(step) = entry.item.step()
&& step.symbol.is_non_terminal()
if let Some(step) = entry.item.step(self.syntax_grammar)
&& step.symbol().is_non_terminal()
{
let next_step = entry.item.successor().step();
let next_step = entry.item.successor().step(self.syntax_grammar);
// Determine which tokens can follow this non-terminal.
let (following_tokens, following_reserved_tokens) = if let Some(next_step) = next_step {
(
self.first_sets.get(&next_step.symbol).unwrap(),
*self.reserved_first_sets.get(&next_step.symbol).unwrap(),
self.first_sets.get(&next_step.symbol()).unwrap(),
*self.reserved_first_sets.get(&next_step.symbol()).unwrap(),
)
} else {
(&entry.lookaheads, entry.following_reserved_word_set)
};
// Use the pre-computed *additions* to expand the non-terminal.
for addition in &self.transitive_closure_additions[step.symbol.index] {
for addition in &self.transitive_closure_additions[step.symbol().index] {
let entry = set.insert(addition.item);
entry.lookaheads.insert_all(&addition.info.lookaheads);
@ -359,50 +355,65 @@ impl<'a> ParseItemSetBuilder<'a> {
}
}
impl fmt::Debug for ParseItemSetBuilder<'_> {
#[expect(dead_code, reason = "Debugging aid")]
struct ParseItemSetBuilderDisplay<'a>(
pub &'a ParseItemSetBuilder<'a>,
pub &'a LexicalGrammar,
pub &'a StrPool,
);
impl fmt::Debug for ParseItemSetBuilderDisplay<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
writeln!(f, "ParseItemSetBuilder {{")?;
writeln!(f, " first_sets: {{")?;
for (symbol, first_set) in &self.first_sets {
for (symbol, first_set) in &self.0.first_sets {
let name = match symbol.kind {
SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
SymbolType::NonTerminal => self
.2
.resolve(self.0.syntax_grammar.variables[symbol.index].name),
SymbolType::External => self
.2
.resolve(self.0.syntax_grammar.external_tokens[symbol.index].name),
SymbolType::Terminal => self.2.resolve(self.1.variables[symbol.index].name),
SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
};
writeln!(
f,
" first({name:?}): {}",
TokenSetDisplay(first_set, self.syntax_grammar, self.lexical_grammar)
TokenSetDisplay(first_set, self.0.syntax_grammar, self.1, self.2)
)?;
}
writeln!(f, " }}")?;
writeln!(f, " last_sets: {{")?;
for (symbol, last_set) in &self.last_sets {
for (symbol, last_set) in &self.0.last_sets {
let name = match symbol.kind {
SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
SymbolType::NonTerminal => self
.2
.resolve(self.0.syntax_grammar.variables[symbol.index].name),
SymbolType::External => self
.2
.resolve(self.0.syntax_grammar.external_tokens[symbol.index].name),
SymbolType::Terminal => self.2.resolve(self.1.variables[symbol.index].name),
SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
};
writeln!(
f,
" last({name:?}): {}",
TokenSetDisplay(last_set, self.syntax_grammar, self.lexical_grammar)
TokenSetDisplay(last_set, self.0.syntax_grammar, self.1, self.2)
)?;
}
writeln!(f, " }}")?;
writeln!(f, " additions: {{")?;
for (i, variable) in self.syntax_grammar.variables.iter().enumerate() {
writeln!(f, " {}: {{", variable.name)?;
for addition in &self.transitive_closure_additions[i] {
for (i, variable) in self.0.syntax_grammar.variables.iter().enumerate() {
writeln!(f, " {}: {{", self.2.resolve(variable.name))?;
for addition in &self.0.transitive_closure_additions[i] {
writeln!(
f,
" {}",
ParseItemDisplay(&addition.item, self.syntax_grammar, self.lexical_grammar)
ParseItemDisplay(&addition.item, self.0.syntax_grammar, self.1, self.2)
)?;
}
writeln!(f, " }},")?;

36
crates/generate/src/build_tables/minimize_parse_table.rs
View file

@ -10,6 +10,7 @@ use crate::{
dedup::split_state_id_groups,
grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
rules::{AliasMap, Symbol, SymbolType, TokenSet},
strpool::StrPool,
tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
};
@ -67,6 +68,10 @@ impl SymbolKey {
}
}
#[expect(
clippy::too_many_arguments,
reason = "all parameters are required for parse table minimization"
)]
pub fn minimize_parse_table(
parse_table: &mut ParseTable,
syntax_grammar: &SyntaxGrammar,
@ -74,6 +79,7 @@ pub fn minimize_parse_table(
simple_aliases: &AliasMap,
token_conflict_map: &TokenConflictMap,
keywords: &TokenSet,
str_pool: &StrPool,
optimizations: OptLevel,
) {
let mut minimizer = Minimizer {
@ -83,6 +89,7 @@ pub fn minimize_parse_table(
token_conflict_map,
keywords,
simple_aliases,
str_pool,
};
if optimizations.contains(OptLevel::MergeStates) {
minimizer.merge_compatible_states();
@ -96,19 +103,20 @@ struct Minimizer<'a> {
parse_table: &'a mut ParseTable,
syntax_grammar: &'a SyntaxGrammar,
lexical_grammar: &'a LexicalGrammar,
token_conflict_map: &'a TokenConflictMap<'a>,
token_conflict_map: &'a TokenConflictMap,
keywords: &'a TokenSet,
simple_aliases: &'a AliasMap,
str_pool: &'a StrPool,
}
impl Minimizer<'_> {
fn remove_unit_reductions(&mut self) {
let mut aliased_symbols = FxHashSet::default();
for variable in &self.syntax_grammar.variables {
for production in &variable.productions {
for step in &production.steps {
if step.alias.is_some() {
aliased_symbols.insert(step.symbol);
for i in 0..self.syntax_grammar.variables.len() {
for prod_id in self.syntax_grammar.variable_prod_ids(i) {
for step in self.syntax_grammar.production(prod_id).steps {
if step.alias().is_some() {
aliased_symbols.insert(step.symbol());
}
}
}
@ -444,7 +452,10 @@ impl Minimizer<'_> {
if group1 != group2 {
debug!(
"split states {} {} - successors for {} are split: {s1} {s2}",
state1.id, state2.id, self.syntax_grammar.variables[idx1].name,
state1.id,
state2.id,
self.str_pool
.resolve(self.syntax_grammar.variables[idx1].name),
);
return true;
}
@ -596,13 +607,16 @@ impl Minimizer<'_> {
false
}
fn symbol_name(&self, symbol: &Symbol) -> &String {
fn symbol_name<'a>(&'a self, symbol: &Symbol) -> &'a str {
if symbol.is_non_terminal() {
&self.syntax_grammar.variables[symbol.index].name
self.str_pool
.resolve(self.syntax_grammar.variables[symbol.index].name)
} else if symbol.is_external() {
&self.syntax_grammar.external_tokens[symbol.index].name
self.str_pool
.resolve(self.syntax_grammar.external_tokens[symbol.index].name)
} else {
&self.lexical_grammar.variables[symbol.index].name
self.str_pool
.resolve(self.lexical_grammar.variables[symbol.index].name)
}
}

254
crates/generate/src/build_tables/token_conflicts.rs
View file

@ -1,4 +1,4 @@
use std::{cmp::Ordering, fmt};
use std::cmp::Ordering;
use rustc_hash::FxHashSet;
@ -9,6 +9,7 @@ use crate::{
grammars::{LexicalGrammar, SyntaxGrammar},
nfa::{CharacterSet, NfaCursor, NfaTransition},
rules::TokenSet,
strpool::StrPool,
};
bitflags! {
@ -24,13 +25,15 @@ bitflags! {
}
}
pub struct TokenConflictMap<'a> {
pub struct TokenConflictMap {
n: usize,
status_matrix: Vec<TokenConflictStatus>,
#[expect(dead_code, reason = "Debugging aid")]
following_tokens: Vec<TokenSet>,
#[allow(dead_code, reason = "Debugging/test aid")]
starting_chars_by_index: Vec<CharacterSet>,
#[expect(dead_code, reason = "Debugging aid")]
following_chars_by_index: Vec<CharacterSet>,
grammar: &'a LexicalGrammar,
/// Per-row bitsets for fast batch conflict checks.
/// Row `i` spans `[i * row_words .. (i+1) * row_words]`.
/// Bit `j` is set iff [`does_conflict(i, j)`](Self::does_conflict) || [`does_match_prefix(i, j)`](Self::does_match_prefix).
@ -40,7 +43,7 @@ pub struct TokenConflictMap<'a> {
pub(crate) row_words: usize,
}
impl<'a> TokenConflictMap<'a> {
impl TokenConflictMap {
/// Create a token conflict map based on a lexical grammar, which describes the structure
/// of each token, and a `following_token` map, which indicates which tokens may appear
/// immediately after each other token.
@ -48,7 +51,7 @@ impl<'a> TokenConflictMap<'a> {
/// This analyzes the possible kinds of overlap between each pair of tokens and stores
/// them in a matrix.
#[must_use]
pub fn new(grammar: &'a LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
pub fn new(grammar: &LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
let mut cursor = NfaCursor::new(&grammar.nfa, Vec::new());
let starting_chars = get_starting_chars(&mut cursor, grammar);
let following_chars = get_following_chars(&starting_chars, &following_tokens);
@ -103,13 +106,12 @@ impl<'a> TokenConflictMap<'a> {
}
}
TokenConflictMap {
Self {
n,
status_matrix,
following_tokens,
starting_chars_by_index: starting_chars,
following_chars_by_index: following_chars,
grammar,
conflict_or_prefix_bits,
overlap_either_bits,
row_words,
@ -225,52 +227,59 @@ impl<'a> TokenConflictMap<'a> {
}
}
impl fmt::Debug for TokenConflictMap<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
#[expect(dead_code, reason = "Debugging aid")]
struct TokenConflictMapDisplay<'a>(
pub &'a TokenConflictMap,
pub &'a LexicalGrammar,
pub &'a StrPool,
);
impl std::fmt::Debug for TokenConflictMapDisplay<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
writeln!(f, "TokenConflictMap {{")?;
let syntax_grammar = SyntaxGrammar::default();
writeln!(f, " following_tokens: {{")?;
for (i, following_tokens) in self.following_tokens.iter().enumerate() {
for (i, following_tokens) in self.0.following_tokens.iter().enumerate() {
writeln!(
f,
" follow({:?}): {},",
self.grammar.variables[i].name,
TokenSetDisplay(following_tokens, &syntax_grammar, self.grammar)
self.1.variables[i].name,
TokenSetDisplay(following_tokens, &syntax_grammar, self.1, self.2)
)?;
}
writeln!(f, " }},")?;
writeln!(f, " starting_characters: {{")?;
for i in 0..self.n {
for i in 0..self.0.n {
writeln!(
f,
" {:?}: {:?},",
self.grammar.variables[i].name, self.starting_chars_by_index[i]
self.1.variables[i].name, self.0.starting_chars_by_index[i]
)?;
}
writeln!(f, " }},")?;
writeln!(f, " following_characters: {{")?;
for i in 0..self.n {
for i in 0..self.0.n {
writeln!(
f,
" {:?}: {:?},",
self.grammar.variables[i].name, self.following_chars_by_index[i]
self.1.variables[i].name, self.0.following_chars_by_index[i]
)?;
}
writeln!(f, " }},")?;
writeln!(f, " status_matrix: {{")?;
for i in 0..self.n {
writeln!(f, " {:?}: {{", self.grammar.variables[i].name)?;
for j in 0..self.n {
for i in 0..self.0.n {
writeln!(f, " {:?}: {{", self.1.variables[i].name)?;
for j in 0..self.0.n {
writeln!(
f,
" {:?}: {:?},",
self.grammar.variables[j].name,
self.status_matrix[matrix_index(self.n, i, j)]
self.1.variables[j].name,
self.0.status_matrix[matrix_index(self.0.n, i, j)]
)?;
}
writeln!(f, " }},")?;
@ -490,29 +499,36 @@ fn compute_conflict_status(
mod tests {
use super::*;
use crate::{
grammars::{Variable, VariableType},
prepare_grammar::{ExtractedLexicalGrammar, expand_tokens},
rules::{Precedence, Rule, Symbol},
grammars::VariableType,
prepare_grammar::{LexicalToken, expand_tokens},
rules::{Precedence, RulePool, Symbol},
};
#[test]
fn test_starting_characters() {
let grammar = expand_tokens(ExtractedLexicalGrammar {
separators: Vec::new(),
variables: vec![
Variable {
name: "token_0".to_string(),
kind: VariableType::Named,
rule: Rule::pattern("[a-f]1|0x\\d", ""),
},
Variable {
name: "token_1".to_string(),
kind: VariableType::Named,
rule: Rule::pattern("d*ef", ""),
},
],
})
.unwrap();
let mut pool = RulePool::default();
let empty = pool.intern("");
let t0 = {
let v = pool.intern("[a-f]1|0x\\d");
pool.pattern(v, empty)
};
let t1 = {
let v = pool.intern("d*ef");
pool.pattern(v, empty)
};
let vars = vec![
LexicalToken {
name: pool.intern("token_0"),
kind: VariableType::Named,
root: t0,
},
LexicalToken {
name: pool.intern("token_1"),
kind: VariableType::Named,
root: t1,
},
];
let grammar = expand_tokens(&mut pool, &vars, &[]).unwrap();
let token_map = TokenConflictMap::new(&grammar, Vec::new());
@ -528,29 +544,40 @@ mod tests {
#[test]
fn test_token_conflicts() {
let grammar = expand_tokens(ExtractedLexicalGrammar {
separators: Vec::new(),
variables: vec![
Variable {
name: "in".to_string(),
kind: VariableType::Named,
rule: Rule::string("in"),
},
Variable {
name: "identifier".to_string(),
kind: VariableType::Named,
rule: Rule::pattern("\\w+", ""),
},
Variable {
name: "instanceof".to_string(),
kind: VariableType::Named,
rule: Rule::string("instanceof"),
},
],
})
.unwrap();
let mut pool = RulePool::default();
let empty = pool.intern("");
let in_tok = {
let s = pool.intern("in");
pool.string(s)
};
let ident = {
let v = pool.intern("\\w+");
pool.pattern(v, empty)
};
let instanceof = {
let s = pool.intern("instanceof");
pool.string(s)
};
let vars = vec![
LexicalToken {
name: pool.intern("in"),
kind: VariableType::Named,
root: in_tok,
},
LexicalToken {
name: pool.intern("identifier"),
kind: VariableType::Named,
root: ident,
},
LexicalToken {
name: pool.intern("instanceof"),
kind: VariableType::Named,
root: instanceof,
},
];
let grammar = expand_tokens(&mut pool, &vars, &[]).unwrap();
let var = |name| index_of_var(&grammar, name);
let var = |name| index_of_var(&pool, &grammar, name);
let token_map = TokenConflictMap::new(
&grammar,
@ -567,40 +594,51 @@ mod tests {
],
);
// Given the string "in", the `in` token is preferred over the `identifier` token
// Given the string "in", the `in` token is preferrred over the `identifier` token
assert!(token_map.does_match_same_string(var("in"), var("identifier")));
assert!(!token_map.does_match_same_string(var("identifier"), var("in")));
// Depending on what character follows, the string "in" may be treated as part of an
// `identifier` token.
// `identifier` token
assert!(token_map.does_conflict(var("identifier"), var("in")));
// Depending on what character follows, the string "instanceof" may be treated as part of
// an `identifier` token.
// Depending on what character follows, the string "instanceof" may be treated as part
// of an `identifier` token
assert!(token_map.does_conflict(var("identifier"), var("instanceof")));
assert!(token_map.does_conflict(var("instanceof"), var("in")));
}
#[test]
fn test_token_conflicts_with_separators() {
let grammar = expand_tokens(ExtractedLexicalGrammar {
separators: vec![Rule::pattern("\\s", "")],
variables: vec![
Variable {
name: "x".to_string(),
kind: VariableType::Named,
rule: Rule::string("x"),
},
Variable {
name: "newline".to_string(),
kind: VariableType::Named,
rule: Rule::string("\n"),
},
],
})
.unwrap();
let mut pool = RulePool::default();
let empty = pool.intern("");
let sep = {
let v = pool.intern("\\s");
pool.pattern(v, empty)
};
let x = {
let s = pool.intern("x");
pool.string(s)
};
let newline = {
let s = pool.intern("\n");
pool.string(s)
};
let vars = vec![
LexicalToken {
name: pool.intern("x"),
kind: VariableType::Named,
root: x,
},
LexicalToken {
name: pool.intern("newline"),
kind: VariableType::Named,
root: newline,
},
];
let grammar = expand_tokens(&mut pool, &vars, &[sep]).unwrap();
let var = |name| index_of_var(&grammar, name);
let var = |name| index_of_var(&pool, &grammar, name);
let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
@ -610,24 +648,36 @@ mod tests {
#[test]
fn test_token_conflicts_with_open_ended_tokens() {
let grammar = expand_tokens(ExtractedLexicalGrammar {
separators: vec![Rule::pattern("\\s", "")],
variables: vec![
Variable {
name: "x".to_string(),
kind: VariableType::Named,
rule: Rule::string("x"),
},
Variable {
name: "anything".to_string(),
kind: VariableType::Named,
rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*", "")),
},
],
})
.unwrap();
let mut pool = RulePool::default();
let empty = pool.intern("");
let sep = {
let v = pool.intern("\\s");
pool.pattern(v, empty)
};
let x = {
let s = pool.intern("x");
pool.string(s)
};
let anything = {
let v = pool.intern(".*");
let pat = pool.pattern(v, empty);
pool.prec(Precedence::Integer(-1), pat)
};
let vars = vec![
LexicalToken {
name: pool.intern("x"),
kind: VariableType::Named,
root: x,
},
LexicalToken {
name: pool.intern("anything"),
kind: VariableType::Named,
root: anything,
},
];
let grammar = expand_tokens(&mut pool, &vars, &[sep]).unwrap();
let var = |name| index_of_var(&grammar, name);
let var = |name| index_of_var(&pool, &grammar, name);
let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
@ -635,11 +685,11 @@ mod tests {
assert!(!token_map.does_match_shorter_or_longer(var("x"), var("anything")));
}
fn index_of_var(grammar: &LexicalGrammar, name: &str) -> usize {
fn index_of_var(pool: &RulePool, grammar: &LexicalGrammar, name: &str) -> usize {
grammar
.variables
.iter()
.position(|v| v.name == name)
.position(|v| pool.resolve(v.name) == name)
.unwrap()
}
}

52
crates/generate/src/generate.rs
View file

@ -10,7 +10,7 @@ use std::{
use bitflags::bitflags;
use node_types::VariableInfo;
use rules::{Alias, Symbol};
use rules::Symbol;
#[cfg(feature = "load")]
use semver::Version;
use serde::{Deserialize, Serialize};
@ -28,11 +28,12 @@ mod prepare_grammar;
mod quickjs;
mod render;
mod rules;
mod strpool;
mod tables;
pub use build_tables::ParseTableBuilderError;
use build_tables::build_tables;
use grammars::{InlinedProductionMap, InputGrammar, LexicalGrammar, SyntaxGrammar};
use grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar};
pub use node_types::{InvalidSupertypeError, SuperTypeCycleError, VariableInfoError};
pub use parse_grammar::ParseGrammarError;
use parse_grammar::parse_grammar;
@ -41,6 +42,10 @@ use prepare_grammar::prepare_grammar;
use render::render_c_code;
pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN, RenderError};
use crate::{
grammars::InputGrammar, prepare_grammar::PreparedGrammar, rules::Alias, strpool::StrPool,
};
struct JSONOutput {
#[cfg(feature = "load")]
node_types_json: String,
@ -49,6 +54,7 @@ struct JSONOutput {
inlines: InlinedProductionMap,
simple_aliases: BTreeMap<Symbol, Alias>,
variable_info: Vec<VariableInfo>,
str_pool: StrPool,
}
struct GeneratedParser {
@ -377,7 +383,7 @@ where
if !generate_parser {
let node_types_json =
generate_node_types_from_grammar(&input_grammar, diagnostics)?.node_types_json;
generate_node_types_from_grammar(input_grammar, diagnostics)?.node_types_json;
write_file(&src_path.join("node-types.json"), node_types_json)?;
return Ok(());
}
@ -389,7 +395,7 @@ where
c_code,
node_types_json,
} = generate_parser_for_grammar_with_opts(
&input_grammar,
input_grammar,
abi_version,
semantic_version.map(|v| (v.major as u8, v.minor as u8, v.patch as u8)),
report_symbol_name,
@ -415,32 +421,43 @@ pub fn generate_parser_for_grammar(
diagnostics: &mut Vec<Diagnostic>,
) -> GenerateResult<(String, String)> {
let input_grammar = parse_grammar(grammar_json, diagnostics)?;
let name = input_grammar.pool.resolve(input_grammar.name).to_string();
let parser = generate_parser_for_grammar_with_opts(
&input_grammar,
input_grammar,
LANGUAGE_VERSION,
semantic_version,
None,
optimizations,
diagnostics,
)?;
Ok((input_grammar.name, parser.c_code))
Ok((name, parser.c_code))
}
fn generate_node_types_from_grammar(
input_grammar: &InputGrammar,
input_grammar: InputGrammar,
diagnostics: &mut Vec<Diagnostic>,
) -> GenerateResult<JSONOutput> {
let (syntax_grammar, lexical_grammar, inlines, simple_aliases) =
prepare_grammar(input_grammar, diagnostics)?;
let variable_info =
node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?;
let PreparedGrammar {
syntax_grammar,
lexical_grammar,
inlines,
default_aliases,
str_pool,
} = prepare_grammar(input_grammar, diagnostics)?;
let variable_info = node_types::get_variable_info(
&syntax_grammar,
&lexical_grammar,
&default_aliases,
&str_pool,
)?;
#[cfg(feature = "load")]
let node_types_json = node_types::generate_node_types_json(
&syntax_grammar,
&lexical_grammar,
&simple_aliases,
&default_aliases,
&variable_info,
&str_pool,
)?;
Ok(JSONOutput {
#[cfg(feature = "load")]
@ -448,19 +465,21 @@ fn generate_node_types_from_grammar(
syntax_grammar,
lexical_grammar,
inlines,
simple_aliases,
simple_aliases: default_aliases,
variable_info,
str_pool,
})
}
fn generate_parser_for_grammar_with_opts(
input_grammar: &InputGrammar,
input_grammar: InputGrammar,
abi_version: usize,
semantic_version: Option<(u8, u8, u8)>,
report_symbol_name: Option<&str>,
optimizations: OptLevel,
diagnostics: &mut Vec<Diagnostic>,
) -> GenerateResult<GeneratedParser> {
let grammar_name = input_grammar.name;
let JSONOutput {
syntax_grammar,
lexical_grammar,
@ -469,6 +488,7 @@ fn generate_parser_for_grammar_with_opts(
variable_info,
#[cfg(feature = "load")]
node_types_json,
str_pool,
} = generate_node_types_from_grammar(input_grammar, diagnostics)?;
let supertype_symbol_map =
node_types::get_supertype_symbol_map(&syntax_grammar, &simple_aliases, &variable_info);
@ -478,16 +498,18 @@ fn generate_parser_for_grammar_with_opts(
&simple_aliases,
&variable_info,
&inlines,
&str_pool,
report_symbol_name,
optimizations,
diagnostics,
)?;
let c_code = render_c_code(
&input_grammar.name,
grammar_name,
tables,
syntax_grammar,
lexical_grammar,
simple_aliases,
str_pool,
abi_version,
semantic_version,
supertype_symbol_map,

370
crates/generate/src/grammars.rs
View file

@ -1,13 +1,14 @@
use std::{
collections::{BTreeMap, HashMap},
fmt,
};
use rustc_hash::FxHashMap;
use crate::node_types::ChildType;
use crate::{
node_types::ChildType,
rules::{Alias, AliasMap, Associativity, Precedence, RuleId, RulePool, SymbolType},
strpool::StrId,
};
use super::{
nfa::Nfa,
rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet},
rules::{Symbol, TokenSet},
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
@ -18,46 +19,45 @@ pub enum VariableType {
Named,
}
// Input grammar
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone, Copy)]
pub struct Variable {
pub name: String,
pub kind: VariableType,
pub rule: Rule,
pub name: StrId,
pub root: RuleId,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum PrecedenceEntry {
Name(String),
Symbol(String),
Name(StrId),
Symbol(StrId),
}
#[derive(Debug, Default, PartialEq, Eq)]
/// The parsed grammar.
#[derive(Default)]
pub struct InputGrammar {
pub name: String,
pub pool: RulePool,
pub name: StrId,
pub variables: Vec<Variable>,
pub extra_symbols: Vec<Rule>,
pub expected_conflicts: Vec<Vec<String>>,
pub external_roots: Vec<RuleId>,
pub extra_roots: Vec<RuleId>,
pub reserved_sets: Vec<ReservedWordContext>,
pub supertype_names: Vec<StrId>,
pub conflict_names: Vec<Vec<StrId>>,
pub inline_names: Vec<StrId>,
pub word_name: Option<StrId>,
pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
pub external_tokens: Vec<Rule>,
pub variables_to_inline: Vec<String>,
pub supertype_symbols: Vec<String>,
pub word_token: Option<String>,
pub reserved_words: Vec<ReservedWordContext<Rule>>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct ReservedWordContext<T> {
pub name: String,
pub reserved_words: Vec<T>,
#[derive(Clone)]
pub struct ReservedWordContext {
pub name: StrId,
pub roots: Vec<RuleId>,
}
// Extracted lexical grammar
#[derive(Debug, PartialEq, Eq)]
pub struct LexicalVariable {
pub name: String,
pub name: StrId,
pub kind: VariableType,
pub implicit_precedence: i32,
pub start_state: u32,
@ -69,63 +69,200 @@ pub struct LexicalGrammar {
pub variables: Vec<LexicalVariable>,
}
// Extracted syntax grammar
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)]
pub struct ProductionStep {
pub symbol: Symbol,
pub precedence: Precedence,
pub associativity: Option<Associativity>,
pub alias: Option<Alias>,
pub field_name: Option<String>,
pub reserved_word_set_id: ReservedWordSetId,
pub sym_index: u32,
pub prec_val: i32,
pub alias: u32,
pub field: u32,
pub reserved: u16,
pub flags: u8,
}
const _: () = assert!(std::mem::size_of::<ProductionStep>() == 20);
#[rustfmt::skip]
impl ProductionStep {
// `FStep::flags` bit layout
// - 0-2 kind (`SymbolType`)
// - 3-4 precedence tag (None 00, Integer 01, Name 10)
// - 5-6 associativity (None 00, Left 01, Right 10)
// - 7 alias `is_named`
const KIND_MASK: u8 = 0b0000_0111;
const PREC_INTEGER: u8 = 0b0000_1000;
const PREC_NAME: u8 = 0b0001_0000;
const PREC_MASK: u8 = 0b0001_1000;
const ASSOC_LEFT: u8 = 0b0010_0000;
const ASSOC_RIGHT: u8 = 0b0100_0000;
const ASSOC_MASK: u8 = 0b0110_0000;
const ALIAS_NAMED: u8 = 0b1000_0000;
/// `FStep::reserved` sentinel, meaning no reserved word set at all. Only the augmented
/// start production carries it, and it must never index the reserved-sets table.
pub const NO_RESERVED_WORDS: u16 = u16::MAX;
}
impl ProductionStep {
pub fn child_type(&self, default_aliases: &BTreeMap<Symbol, Alias>) -> ChildType {
if let Some(alias) = &self.alias {
ChildType::Aliased(alias.clone())
} else if let Some(alias) = default_aliases.get(&self.symbol) {
ChildType::Aliased(alias.clone())
/// Pack a production step's components into its flat `FStep` representation.
#[must_use]
pub fn pack(
symbol: Symbol,
prec: Precedence,
assoc: Option<Associativity>,
alias: Option<Alias>,
field: Option<StrId>,
reserved: u16,
) -> Self {
let mut step = Self {
sym_index: symbol.index as u32,
reserved,
flags: symbol.kind as u8,
..Default::default()
};
step.set_precedence(prec);
step.set_associativity(assoc);
step.set_alias(alias);
step.set_field(field);
step
}
#[must_use]
pub const fn symbol(self) -> Symbol {
Symbol {
kind: match self.flags & Self::KIND_MASK {
0 => SymbolType::External,
1 => SymbolType::End,
2 => SymbolType::EndOfNonTerminalExtra,
3 => SymbolType::Terminal,
_ => SymbolType::NonTerminal,
},
index: self.sym_index as usize,
}
}
#[must_use]
pub const fn precedence(self) -> Precedence {
match self.flags & Self::PREC_MASK {
Self::PREC_INTEGER => Precedence::Integer(self.prec_val),
Self::PREC_NAME => Precedence::Name(StrId::from_raw(self.prec_val as u32)),
_ => Precedence::None,
}
}
pub const fn set_precedence(&mut self, prec: Precedence) {
let (bits, val) = match prec {
Precedence::None => (0, 0),
Precedence::Integer(n) => (Self::PREC_INTEGER, n),
Precedence::Name(sid) => (Self::PREC_NAME, sid.raw() as i32),
};
self.prec_val = val;
self.flags = (self.flags & !Self::PREC_MASK) | bits;
}
#[must_use]
pub const fn associativity(self) -> Option<Associativity> {
match self.flags & Self::ASSOC_MASK {
Self::ASSOC_LEFT => Some(Associativity::Left),
Self::ASSOC_RIGHT => Some(Associativity::Right),
_ => None,
}
}
pub const fn set_associativity(&mut self, assoc: Option<Associativity>) {
let bits = match assoc {
None => 0,
Some(Associativity::Left) => Self::ASSOC_LEFT,
Some(Associativity::Right) => Self::ASSOC_RIGHT,
};
self.flags = (self.flags & !Self::ASSOC_MASK) | bits;
}
#[must_use]
pub fn alias(self) -> Option<Alias> {
(self.alias != 0).then(|| Alias {
value: StrId::from_raw(self.alias),
is_named: self.flags & Self::ALIAS_NAMED != 0,
})
}
pub fn set_alias(&mut self, alias: Option<Alias>) {
self.alias = alias.map_or(0, |a| a.value.raw());
self.set_alias_named(alias.is_some_and(|a| a.is_named));
}
pub const fn set_alias_named(&mut self, named: bool) {
self.flags = (self.flags & !Self::ALIAS_NAMED) | if named { Self::ALIAS_NAMED } else { 0 };
}
#[must_use]
pub fn field(self) -> Option<StrId> {
(self.field != 0).then(|| StrId::from_raw(self.field))
}
pub fn set_field(&mut self, field: Option<StrId>) {
self.field = field.map_or(0, StrId::raw);
}
pub fn child_type(&self, default_aliases: &AliasMap) -> ChildType {
if let Some(alias) = self.alias() {
ChildType::Aliased(alias)
} else if let Some(alias) = default_aliases.get(&self.symbol()) {
ChildType::Aliased(*alias)
} else {
ChildType::Normal(self.symbol)
ChildType::Normal(self.symbol())
}
}
}
// Extracted syntax grammar
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ReservedWordSetId(pub usize);
impl fmt::Display for ReservedWordSetId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
impl std::fmt::Display for ReservedWordSetId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
pub const NO_RESERVED_WORDS: ReservedWordSetId = ReservedWordSetId(usize::MAX);
#[derive(Clone, Debug, Default, PartialEq, Eq)]
/// A flattened production consisting of a step range and its dynamic precedence
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct Production {
pub steps: Vec<ProductionStep>,
pub steps_start: u32,
pub steps_len: u32,
pub dynamic_precedence: i32,
}
#[derive(Default)]
pub struct InlinedProductionMap {
impl Production {
#[must_use]
pub const fn step_range(self) -> std::ops::Range<usize> {
self.steps_start as usize..(self.steps_start + self.steps_len) as usize
}
}
/// Flattened output: one backing store of steps, productions as `[start, len)`
/// ranges into it, and per-variable production ranges.
#[derive(Clone, Default)]
pub struct ProductionStore {
pub steps: Vec<ProductionStep>,
pub productions: Vec<Production>,
pub production_map: HashMap<(*const Production, u32), Vec<usize>>,
pub var_prods: Vec<(u32, u32)>,
}
#[derive(Debug, Default)]
pub struct InlinedProductionMap {
pub map: FxHashMap<(u32, u32), Vec<u32>>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SyntaxVariable {
pub name: String,
pub name: StrId,
pub kind: VariableType,
pub productions: Vec<Production>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExternalToken {
pub name: String,
pub name: StrId,
pub kind: VariableType,
pub corresponding_internal_token: Option<Symbol>,
}
@ -141,92 +278,41 @@ pub struct SyntaxGrammar {
pub word_token: Option<Symbol>,
pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
pub reserved_word_sets: Vec<TokenSet>,
pub steps: Vec<ProductionStep>,
pub productions: Vec<Production>,
pub var_prods: Vec<(u32, u32)>,
}
#[cfg(test)]
impl ProductionStep {
impl SyntaxGrammar {
#[must_use]
pub fn new(symbol: Symbol) -> Self {
Self {
symbol,
precedence: Precedence::None,
associativity: None,
alias: None,
field_name: None,
reserved_word_set_id: ReservedWordSetId::default(),
pub fn production(&self, id: u32) -> ProdRef<'_> {
let p = self.productions[id as usize];
ProdRef {
steps: &self.steps[p.step_range()],
dynamic_precedence: p.dynamic_precedence,
}
}
/// The pooled production ids belonging to a variable
#[must_use]
pub fn with_prec(
mut self,
precedence: Precedence,
associativity: Option<Associativity>,
) -> Self {
self.precedence = precedence;
self.associativity = associativity;
self
}
#[must_use]
pub fn with_alias(mut self, value: &str, is_named: bool) -> Self {
self.alias = Some(Alias {
value: value.to_string(),
is_named,
});
self
}
#[must_use]
pub fn with_field_name(mut self, name: &str) -> Self {
self.field_name = Some(name.to_string());
self
pub fn variable_prod_ids(&self, variable_index: usize) -> std::ops::Range<u32> {
let (start, end) = self.var_prods[variable_index];
start..end
}
}
impl Production {
/// A production in the pooled [`SyntaxGrammar`] storage
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ProdRef<'a> {
pub steps: &'a [ProductionStep],
pub dynamic_precedence: i32,
}
impl ProdRef<'_> {
#[must_use]
pub fn first_symbol(&self) -> Option<Symbol> {
self.steps.first().map(|s| s.symbol)
}
}
#[cfg(test)]
impl Variable {
#[must_use]
pub fn named(name: &str, rule: Rule) -> Self {
Self {
name: name.to_string(),
kind: VariableType::Named,
rule,
}
}
#[must_use]
pub fn auxiliary(name: &str, rule: Rule) -> Self {
Self {
name: name.to_string(),
kind: VariableType::Auxiliary,
rule,
}
}
#[must_use]
pub fn hidden(name: &str, rule: Rule) -> Self {
Self {
name: name.to_string(),
kind: VariableType::Hidden,
rule,
}
}
#[must_use]
pub fn anonymous(name: &str, rule: Rule) -> Self {
Self {
name: name.to_string(),
kind: VariableType::Anonymous,
rule,
}
self.steps.first().map(|s| s.symbol())
}
}
@ -278,27 +364,7 @@ impl SyntaxVariable {
impl InlinedProductionMap {
#[must_use]
pub fn inlined_productions<'a>(
&'a self,
production: &Production,
step_index: u32,
) -> Option<impl Iterator<Item = &'a Production> + 'a> {
self.production_map
.get(&(std::ptr::from_ref::<Production>(production), step_index))
.map(|production_indices| {
production_indices
.iter()
.copied()
.map(move |index| &self.productions[index])
})
}
}
impl fmt::Display for PrecedenceEntry {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::Name(n) => write!(f, "'{n}'"),
Self::Symbol(s) => write!(f, "$.{s}"),
}
pub fn inlined_prod_ids(&self, prod_id: u32, step_index: u32) -> Option<&[u32]> {
self.map.get(&(prod_id, step_index)).map(Vec::as_slice)
}
}

1137
crates/generate/src/node_types.rs
View file

File diff suppressed because it is too large Load diff

474
crates/generate/src/parse_grammar.rs
View file

@ -7,8 +7,9 @@ use thiserror::Error;
use crate::{
Diagnostic,
grammars::{InputGrammar, PrecedenceEntry, ReservedWordContext, Variable, VariableType},
rules::{Precedence, Rule},
grammars::{InputGrammar, PrecedenceEntry, ReservedWordContext, Variable},
rules::{Precedence, Rule, RuleId, RulePool},
strpool::StrId,
};
#[derive(Deserialize)]
@ -133,28 +134,6 @@ impl From<serde_json::Error> for ParseGrammarError {
}
}
/// Check if a rule is referenced by another rule.
///
/// This function is used to determine if a variable is used in a given rule,
/// and `is_other` indicates if the rule is an external, and if it is,
/// to not assume that a named symbol that is equal to itself means it's being referenced.
///
/// For example, if we have an external rule **and** a normal rule both called `foo`,
/// `foo` should not be thought of as directly used unless it's used within another rule.
fn rule_is_referenced(rule: &Rule, target: &str, is_external: bool) -> bool {
match rule {
Rule::NamedSymbol(name) => name == target && !is_external,
Rule::Choice(rules) | Rule::Seq(rules) => {
rules.iter().any(|r| rule_is_referenced(r, target, false))
}
Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
rule_is_referenced(rule, target, is_external)
}
Rule::Repeat(inner) => rule_is_referenced(inner, target, false),
Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false,
}
}
impl InputGrammar {
/// Strip unused rules from the grammar and clean up references to them
/// in the surrounding config. (conflicts, supertypes, inline, extras,
@ -170,97 +149,95 @@ impl InputGrammar {
// Extras count their top-level `NamedSymbol` as a use (so naming
// a rule directly in `extras` keeps it), but externals do not (the
// external entry is the rule itself, not a reference to one).
let used: FxHashSet<String> = {
let by_name: FxHashMap<&str, &Rule> = self
.variables
.iter()
.map(|v| (v.name.as_str(), &v.rule))
.collect();
let mut visited: FxHashSet<&str> = FxHashSet::default();
let mut stack: Vec<&str> = Vec::new();
let used: FxHashSet<StrId> = {
let by_name: FxHashMap<StrId, RuleId> =
self.variables.iter().map(|v| (v.name, v.root)).collect();
let mut visited: FxHashSet<StrId> = FxHashSet::default();
let mut stack: Vec<StrId> = Vec::new();
if let Some(first) = self.variables.first() {
stack.push(first.name.as_str());
stack.push(first.name);
}
if let Some(word) = self.word_token.as_deref() {
if let Some(word) = self.word_name {
stack.push(word);
}
for rule in &self.extra_symbols {
collect_referenced_names(rule, false, &mut stack);
for &root in &self.extra_roots {
self.pool.collect_referenced_ids(root, false, &mut stack);
}
for rule in &self.external_tokens {
collect_referenced_names(rule, true, &mut stack);
for &root in &self.external_roots {
self.pool.collect_referenced_ids(root, true, &mut stack);
}
// Reserved-word entries are uses of the named rule (the entry
// names a token to reserve in some context). Top-level
// `NamedSymbol` counts, same as for extras.
for ctx in &self.reserved_words {
for rule in &ctx.reserved_words {
collect_referenced_names(rule, false, &mut stack);
for set in &self.reserved_sets {
for &root in &set.roots {
self.pool.collect_referenced_ids(root, false, &mut stack);
}
}
while let Some(name) = stack.pop() {
if !visited.insert(name) {
continue;
}
if let Some(rule) = by_name.get(name) {
collect_referenced_names(rule, false, &mut stack);
if let Some(&root) = by_name.get(&name) {
self.pool.collect_referenced_ids(root, false, &mut stack);
}
}
visited.into_iter().map(String::from).collect()
visited
};
for v in &self.variables {
if !used.contains(v.name.as_str()) {
if !used.contains(&v.name) {
continue;
}
if !self
.extra_symbols
.extra_roots
.iter()
.any(|r| rule_is_referenced(r, &v.name, false))
.any(|&r| self.pool.rule_is_referenced(r, v.name, false))
{
continue;
}
let inner_rule = match &v.rule {
Rule::Metadata { rule, .. } => rule.as_ref(),
other => other,
let inner_root = match self.pool.node(v.root) {
Rule::Metadata { rule, .. } => rule,
_ => v.root,
};
let matches_empty = match inner_rule {
Rule::String(s) => s.is_empty(),
Rule::Pattern(value, _) => Regex::new(value).is_ok_and(|reg| reg.is_match("")),
let matches_empty = match self.pool.node(inner_root) {
Rule::String(s) => self.pool.resolve(s).is_empty(),
Rule::Pattern(value, _) => {
Regex::new(self.pool.resolve(value)).is_ok_and(|reg| reg.is_match(""))
}
_ => false,
};
if matches_empty {
diagnostics.push(Diagnostic::EmptyStringMatch(v.name.clone()));
diagnostics.push(Diagnostic::EmptyStringMatch(
self.pool.resolve(v.name).to_string(),
));
}
}
// Drop unused variables and clean up any references to them in the
// surrounding grammar config.
let dropped: Vec<String> = self
// Drop unused variables and clean up references to them in the config
let dropped: Vec<StrId> = self
.variables
.iter()
.filter(|v| !used.contains(v.name.as_str()))
.map(|v| v.name.clone())
.filter(|v| !used.contains(&v.name))
.map(|v| v.name)
.collect();
self.variables.retain(|v| used.contains(v.name.as_str()));
for name in &dropped {
self.expected_conflicts.retain(|r| !r.contains(name));
self.supertype_symbols.retain(|r| r != name);
self.variables_to_inline.retain(|r| r != name);
self.extra_symbols
.retain(|r| !rule_is_referenced(r, name, true));
self.external_tokens
.retain(|r| !rule_is_referenced(r, name, true));
self.precedence_orderings.retain(|r| {
!r.iter()
.any(|e| matches!(e, PrecedenceEntry::Symbol(s) if s == name))
self.variables.retain(|v| used.contains(&v.name));
for &name in &dropped {
self.conflict_names.retain(|c| !c.contains(&name));
self.supertype_names.retain(|&s| s != name);
self.inline_names.retain(|&s| s != name);
let pool = &self.pool;
self.extra_roots
.retain(|&r| !pool.rule_is_referenced(r, name, true));
self.external_roots
.retain(|&r| !pool.rule_is_referenced(r, name, true));
self.precedence_orderings.retain(|o| {
!o.iter()
.any(|e| matches!(e, PrecedenceEntry::Symbol(s) if *s == name))
});
// Prune entries but keep the context: an intentionally-empty
// reserved-word context is a meaningful marker that rule bodies
// may reference by name.
for ctx in &mut self.reserved_words {
ctx.reserved_words
.retain(|r| !rule_is_referenced(r, name, false));
for set in &mut self.reserved_sets {
set.roots
.retain(|&r| !pool.rule_is_referenced(r, name, false));
}
}
@ -268,54 +245,32 @@ impl InputGrammar {
}
}
/// Append every `NamedSymbol` name reachable in `rule` to `out`. If
/// `skip_top_level` is true, a `NamedSymbol` at the root of `rule` is
/// ignored (used for externals entries, which name themselves).
fn collect_referenced_names<'a>(rule: &'a Rule, skip_top_level: bool, out: &mut Vec<&'a str>) {
match rule {
Rule::NamedSymbol(name) => {
if !skip_top_level {
out.push(name.as_str());
}
}
Rule::Choice(rules) | Rule::Seq(rules) => {
for r in rules {
collect_referenced_names(r, false, out);
}
}
Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
collect_referenced_names(rule, skip_top_level, out);
}
Rule::Repeat(inner) => collect_referenced_names(inner, false, out),
Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => {}
}
}
pub(crate) fn parse_grammar(
input: &str,
diagnostics: &mut Vec<Diagnostic>,
) -> ParseGrammarResult<InputGrammar> {
let grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
let mut pool = RulePool::default();
let extra_symbols =
let extra_roots =
grammar_json
.extras
.into_iter()
.try_fold(Vec::<Rule>::new(), |mut acc, item| {
let rule = parse_rule(item, false, diagnostics)?;
if let Rule::String(ref value) = rule
&& value.is_empty()
.try_fold(Vec::<RuleId>::new(), |mut acc, item| {
let root = pool.parse_rule(item, false, diagnostics)?;
if let Rule::String(s) = pool.node(root)
&& pool.resolve(s).is_empty()
{
Err(ParseGrammarError::InvalidExtra)?;
}
acc.push(rule);
acc.push(root);
ParseGrammarResult::Ok(acc)
})?;
let external_tokens = grammar_json
let external_roots = grammar_json
.externals
.into_iter()
.map(|e| parse_rule(e, false, diagnostics))
.map(|e| pool.parse_rule(e, false, diagnostics))
.collect::<ParseGrammarResult<Vec<_>>>()?;
let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len());
@ -323,8 +278,8 @@ pub(crate) fn parse_grammar(
let mut ordering = Vec::with_capacity(list.len());
for entry in list {
ordering.push(match entry {
RuleJSON::STRING { value } => PrecedenceEntry::Name(value),
RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name),
RuleJSON::STRING { value } => PrecedenceEntry::Name(pool.intern(&value)),
RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(pool.intern(&name)),
_ => Err(ParseGrammarError::Unexpected)?,
});
}
@ -335,15 +290,15 @@ pub(crate) fn parse_grammar(
.rules
.into_iter()
.map(|(name, r)| {
let name = pool.intern(&name);
Ok(Variable {
name,
kind: VariableType::Named,
rule: parse_rule(serde_json::from_value(r)?, false, diagnostics)?,
root: pool.parse_rule(serde_json::from_value(r)?, false, diagnostics)?,
})
})
.collect::<ParseGrammarResult<Vec<_>>>()?;
let reserved_words = grammar_json
let reserved_sets = grammar_json
.reserved
.into_iter()
.map(|(name, rule_values)| {
@ -351,126 +306,171 @@ pub(crate) fn parse_grammar(
Err(ParseGrammarError::InvalidReservedWordSet)?
};
let mut reserved_words = Vec::with_capacity(rule_values.len());
let name = pool.intern(&name);
let mut roots = Vec::with_capacity(rule_values.len());
for value in rule_values {
reserved_words.push(parse_rule(
serde_json::from_value(value)?,
false,
diagnostics,
)?);
roots.push(pool.parse_rule(serde_json::from_value(value)?, false, diagnostics)?);
}
Ok(ReservedWordContext {
name,
reserved_words,
})
Ok(ReservedWordContext { name, roots })
})
.collect::<ParseGrammarResult<Vec<_>>>()?;
let supertype_names = grammar_json
.supertypes
.iter()
.map(|s| pool.intern(s))
.collect();
let conflict_names = grammar_json
.conflicts
.iter()
.map(|c| c.iter().map(|n| pool.intern(n)).collect())
.collect();
let inline_names = grammar_json.inline.iter().map(|n| pool.intern(n)).collect();
let word_name = grammar_json.word.as_deref().map(|w| pool.intern(w));
let name = pool.intern(&grammar_json.name);
let grammar = InputGrammar {
name: grammar_json.name,
word_token: grammar_json.word,
expected_conflicts: grammar_json.conflicts,
supertype_symbols: grammar_json.supertypes,
variables_to_inline: grammar_json.inline,
precedence_orderings,
pool,
name,
variables,
extra_symbols,
external_tokens,
reserved_words,
external_roots,
extra_roots,
reserved_sets,
supertype_names,
conflict_names,
inline_names,
word_name,
precedence_orderings,
}
.normalize(diagnostics);
Ok(grammar)
}
fn parse_rule(
json: RuleJSON,
is_token: bool,
diagnostics: &mut Vec<Diagnostic>,
) -> ParseGrammarResult<Rule> {
match json {
RuleJSON::ALIAS {
content,
value,
named,
} => parse_rule(*content, is_token, diagnostics).map(|r| Rule::alias(r, value, named)),
RuleJSON::BLANK => Ok(Rule::Blank),
RuleJSON::STRING { value } => Ok(Rule::String(value)),
RuleJSON::PATTERN { value, flags } => {
let processed_flags = flags.map_or(String::new(), |f| {
f.matches(|c| {
if c == 'i' {
true
} else {
// silently ignore unicode flags
if c != 'u' && c != 'v' {
diagnostics.push(Diagnostic::UnsupportedRegexFlag {
flag: c,
pattern: value.clone(),
});
}
false
}
})
.collect()
});
Ok(Rule::Pattern(value, processed_flags))
}
RuleJSON::SYMBOL { name } => {
if is_token {
Err(ParseGrammarError::UnexpectedRule(name))?
} else {
Ok(Rule::NamedSymbol(name))
impl RulePool {
fn parse_rule(
&mut self,
json: RuleJSON,
is_token: bool,
diagnostics: &mut Vec<Diagnostic>,
) -> ParseGrammarResult<RuleId> {
match json {
RuleJSON::ALIAS {
content,
named,
value,
} => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let value = self.intern(&value);
Ok(self.alias(content, value, named))
}
RuleJSON::BLANK => Ok(self.blank()),
RuleJSON::PATTERN { value, flags } => {
let processed_flags = flags.map_or(String::new(), |f| {
f.matches(|c| {
if c == 'i' {
true
} else {
// silently ignore unicode flags
if c != 'u' && c != 'v' {
diagnostics.push(Diagnostic::UnsupportedRegexFlag {
flag: c,
pattern: value.clone(),
});
}
false
}
})
.collect()
});
let value = self.intern(&value);
let flags = self.intern(&processed_flags);
Ok(self.pattern(value, flags))
}
RuleJSON::SYMBOL { name } => {
if is_token {
Err(ParseGrammarError::UnexpectedRule(name))?
} else {
let sid = self.intern(&name);
Ok(self.named_symbol(sid))
}
}
RuleJSON::CHOICE { members } => {
let members = members
.into_iter()
.map(|m| self.parse_rule(m, is_token, diagnostics))
.collect::<ParseGrammarResult<Vec<_>>>()?;
Ok(self.choice(&members))
}
RuleJSON::SEQ { members } => {
let members = members
.into_iter()
.map(|m| self.parse_rule(m, is_token, diagnostics))
.collect::<ParseGrammarResult<Vec<_>>>()?;
Ok(self.seq(&members))
}
RuleJSON::FIELD { name, content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let name = self.intern(&name);
Ok(self.field(name, content))
}
RuleJSON::REPEAT { content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let repeat = self.repeat(content);
let blank = self.blank();
Ok(self.choice(&[repeat, blank]))
}
RuleJSON::REPEAT1 { content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
Ok(self.repeat(content))
}
RuleJSON::PREC { value, content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let value = value.intern(self);
Ok(self.prec(value, content))
}
RuleJSON::PREC_LEFT { value, content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let value = value.intern(self);
Ok(self.prec_left(value, content))
}
RuleJSON::PREC_RIGHT { value, content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let value = value.intern(self);
Ok(self.prec_right(value, content))
}
RuleJSON::PREC_DYNAMIC { value, content } => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
Ok(self.prec_dynamic(value, content))
}
RuleJSON::RESERVED {
context_name,
content,
} => {
let content = self.parse_rule(*content, is_token, diagnostics)?;
let ctx = self.intern(&context_name);
Ok(self.reserved(content, ctx))
}
RuleJSON::TOKEN { content } => {
let content = self.parse_rule(*content, true, diagnostics)?;
Ok(self.token(content))
}
RuleJSON::IMMEDIATE_TOKEN { content } => {
let content = self.parse_rule(*content, true, diagnostics)?;
Ok(self.immediate_token(content))
}
RuleJSON::STRING { value } => {
let sid = self.intern(&value);
Ok(self.string(sid))
}
}
RuleJSON::CHOICE { members } => members
.into_iter()
.map(|m| parse_rule(m, is_token, diagnostics))
.collect::<ParseGrammarResult<Vec<_>>>()
.map(Rule::choice),
RuleJSON::FIELD { content, name } => {
parse_rule(*content, is_token, diagnostics).map(|r| Rule::field(name, r))
}
RuleJSON::SEQ { members } => members
.into_iter()
.map(|m| parse_rule(m, is_token, diagnostics))
.collect::<ParseGrammarResult<Vec<_>>>()
.map(Rule::seq),
RuleJSON::REPEAT1 { content } => {
parse_rule(*content, is_token, diagnostics).map(Rule::repeat)
}
RuleJSON::REPEAT { content } => parse_rule(*content, is_token, diagnostics)
.map(|m| Rule::choice(vec![Rule::repeat(m), Rule::Blank])),
RuleJSON::PREC { value, content } => {
parse_rule(*content, is_token, diagnostics).map(|r| Rule::prec(value.into(), r))
}
RuleJSON::PREC_LEFT { value, content } => {
parse_rule(*content, is_token, diagnostics).map(|r| Rule::prec_left(value.into(), r))
}
RuleJSON::PREC_RIGHT { value, content } => {
parse_rule(*content, is_token, diagnostics).map(|r| Rule::prec_right(value.into(), r))
}
RuleJSON::PREC_DYNAMIC { value, content } => {
parse_rule(*content, is_token, diagnostics).map(|r| Rule::prec_dynamic(value, r))
}
RuleJSON::RESERVED {
content,
context_name,
} => parse_rule(*content, is_token, diagnostics).map(|r| Rule::Reserved {
rule: Box::new(r),
context_name,
}),
RuleJSON::TOKEN { content } => parse_rule(*content, true, diagnostics).map(Rule::token),
RuleJSON::IMMEDIATE_TOKEN { content } => {
parse_rule(*content, true, diagnostics).map(Rule::immediate_token)
}
}
}
impl From<PrecedenceValueJSON> for Precedence {
fn from(val: PrecedenceValueJSON) -> Self {
match val {
PrecedenceValueJSON::Integer(i) => Self::Integer(i),
PrecedenceValueJSON::Name(i) => Self::Name(i),
impl PrecedenceValueJSON {
fn intern(self, pool: &mut RulePool) -> Precedence {
match self {
Self::Integer(i) => Precedence::Integer(i),
Self::Name(n) => Precedence::Name(pool.intern(&n)),
}
}
}
@ -502,21 +502,31 @@ mod tests {
)
.unwrap();
assert_eq!(grammar.name, "my_lang");
assert_eq!(
grammar.variables,
vec![
Variable {
name: "file".to_string(),
kind: VariableType::Named,
rule: Rule::repeat(Rule::NamedSymbol("statement".to_string()))
},
Variable {
name: "statement".to_string(),
kind: VariableType::Named,
rule: Rule::String("foo".to_string())
},
]
);
assert_eq!(grammar.pool.resolve(grammar.name), "my_lang");
let names = grammar
.variables
.iter()
.map(|v| grammar.pool.resolve(v.name))
.collect::<Vec<_>>();
assert_eq!(names, ["file", "statement"]);
// file = repeat(named_symbol "statement")
let file_child = grammar.pool.node(grammar.variables[0].root);
let Rule::Repeat(inner) = file_child else {
panic!("Expected repeat, got {file_child:#?}");
};
let inner_node = grammar.pool.node(inner);
let Rule::NamedSymbol(name) = inner_node else {
panic!("Expected named symbol, got {inner_node:#?}");
};
assert_eq!(grammar.pool.resolve(name), "statement");
// statement = string "foo"
let statement_child = grammar.pool.node(grammar.variables[1].root);
let Rule::String(value) = statement_child else {
panic!("Expected string, got {statement_child:#?}");
};
assert_eq!(grammar.pool.resolve(value), "foo");
}
}

503
crates/generate/src/prepare_grammar.rs
View file

@ -14,6 +14,8 @@ use std::{
pub use expand_tokens::ExpandTokensError;
pub use extract_tokens::ExtractTokensError;
#[cfg(test)] // TODO: Is this defined in the proper place?
pub use extract_tokens::LexicalToken;
pub use flatten_grammar::FlattenGrammarError;
use indexmap::IndexMap;
pub use intern_symbols::InternSymbolsError;
@ -22,6 +24,11 @@ use rustc_hash::{FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
grammars::{InputGrammar, PrecedenceEntry, ProductionStore},
strpool::StrPool,
};
pub use self::expand_tokens::expand_tokens;
use self::{
expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
@ -29,51 +36,12 @@ use self::{
intern_symbols::intern_symbols, process_inlines::process_inlines,
};
use super::{
grammars::{
ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
SyntaxGrammar, Variable,
},
rules::{AliasMap, Precedence, Rule, Symbol},
Diagnostic,
grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
prepare_grammar::flatten_grammar::{FlattenState, assemble_syntax_grammar},
rules::{AliasMap, Precedence, Rule},
strpool::StrId,
};
use crate::{Diagnostic, grammars::ReservedWordContext};
pub struct IntermediateGrammar<T, U> {
variables: Vec<Variable>,
extra_symbols: Vec<T>,
expected_conflicts: Vec<Vec<Symbol>>,
precedence_orderings: Vec<Vec<PrecedenceEntry>>,
external_tokens: Vec<U>,
variables_to_inline: Vec<Symbol>,
supertype_symbols: Vec<Symbol>,
word_token: Option<Symbol>,
reserved_word_sets: Vec<ReservedWordContext<T>>,
}
pub type InternedGrammar = IntermediateGrammar<Rule, Variable>;
pub type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;
#[derive(Debug, PartialEq, Eq)]
pub struct ExtractedLexicalGrammar {
pub variables: Vec<Variable>,
pub separators: Vec<Rule>,
}
impl<T, U> Default for IntermediateGrammar<T, U> {
fn default() -> Self {
Self {
variables: Vec::default(),
extra_symbols: Vec::default(),
expected_conflicts: Vec::default(),
precedence_orderings: Vec::default(),
external_tokens: Vec::default(),
variables_to_inline: Vec::default(),
supertype_symbols: Vec::default(),
word_token: Option::default(),
reserved_word_sets: Vec::default(),
}
}
}
pub type PrepareGrammarResult<T> = Result<T, PrepareGrammarError>;
@ -91,14 +59,14 @@ pub enum PrepareGrammarError {
pub type ValidatePrecedenceResult<T> = Result<T, ValidatePrecedenceError>;
#[derive(Debug, Error, Serialize, Deserialize)]
#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
#[error(transparent)]
pub enum ValidatePrecedenceError {
Undeclared(#[from] UndeclaredPrecedenceError),
Ordering(#[from] ConflictingPrecedenceOrderingError),
}
#[derive(Debug, Error, Serialize, Deserialize)]
#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
pub struct IndirectRecursionError(pub Vec<String>);
impl std::fmt::Display for IndirectRecursionError {
@ -114,82 +82,89 @@ impl std::fmt::Display for IndirectRecursionError {
}
}
#[derive(Debug, Error, Serialize, Deserialize)]
#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
#[error("Undeclared precedence '{}' in rule '{}'", self.precedence, self.rule)]
pub struct UndeclaredPrecedenceError {
pub precedence: String,
pub rule: String,
}
impl std::fmt::Display for UndeclaredPrecedenceError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Undeclared precedence '{}' in rule '{}'",
self.precedence, self.rule
)?;
Ok(())
}
}
#[derive(Debug, Error, Serialize, Deserialize)]
#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
#[error("Conflicting orderings for precedences {} and {}", self.precedence_1, self.precedence_2)]
pub struct ConflictingPrecedenceOrderingError {
pub precedence_1: String,
pub precedence_2: String,
}
impl std::fmt::Display for ConflictingPrecedenceOrderingError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Conflicting orderings for precedences {} and {}",
self.precedence_1, self.precedence_2
)?;
Ok(())
}
pub struct PreparedGrammar {
pub syntax_grammar: SyntaxGrammar,
pub lexical_grammar: LexicalGrammar,
pub inlines: InlinedProductionMap,
pub default_aliases: AliasMap,
pub str_pool: StrPool,
}
/// Transform an input grammar into separate components that are ready
/// for parse table construction.
pub fn prepare_grammar(
input_grammar: &InputGrammar,
mut g: InputGrammar,
diagnostics: &mut Vec<Diagnostic>,
) -> PrepareGrammarResult<(
SyntaxGrammar,
LexicalGrammar,
InlinedProductionMap,
AliasMap,
)> {
validate_precedences(input_grammar)?;
validate_indirect_recursion(input_grammar)?;
) -> PrepareGrammarResult<PreparedGrammar> {
validate_precedences(&g)?;
validate_indirect_recursion(&g)?;
let interned_grammar = intern_symbols(input_grammar, diagnostics)?;
let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
let syntax_grammar = expand_repeats(syntax_grammar);
let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
let lexical_grammar = expand_tokens(lexical_grammar)?;
let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
let inlines = process_inlines(&syntax_grammar, &lexical_grammar)?;
Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
let interned_meta = intern_symbols(&mut g, diagnostics)?;
let mut ext_meta = extract_tokens(&mut g, &interned_meta)?;
expand_repeats(&mut g, &mut ext_meta);
let mut state = FlattenState::default();
let mut out = ProductionStore::default();
flatten_grammar(&g, &ext_meta, &mut state, &mut out)?;
let lexical_grammar = expand_tokens(
&mut g.pool,
&ext_meta.lexical_variables,
&ext_meta.separator_roots,
)?;
let default_aliases = extract_default_aliases(&g, &ext_meta, &mut out);
let inlines = process_inlines(&g, &ext_meta, &mut out)?;
let (syntax_grammar, str_pool) = assemble_syntax_grammar(g, ext_meta, out);
Ok(PreparedGrammar {
syntax_grammar,
lexical_grammar,
inlines,
default_aliases,
str_pool,
})
}
/// Check for indirect recursion cycles in the grammar that can cause infinite loops while
/// parsing. An indirect recursion cycle occurs when a non-terminal can derive itself through
/// a chain of single-symbol productions (e.g., A -> B, B -> A).
fn validate_indirect_recursion(grammar: &InputGrammar) -> Result<(), IndirectRecursionError> {
let mut epsilon_transitions: IndexMap<&str, BTreeSet<String>> = IndexMap::new();
let mut epsilon_transitions = IndexMap::new();
let mut stack = Vec::new();
for variable in &grammar.variables {
let productions = get_single_symbol_productions(&variable.rule);
// Filter out rules that *directly* reference themselves, as this doesn't
// cause a parsing loop.
let filtered: BTreeSet<String> = productions
.into_iter()
.filter(|s| s != &variable.name)
.collect();
epsilon_transitions.insert(variable.name.as_str(), filtered);
let mut productions = BTreeSet::new();
stack.clear();
stack.push(variable.root);
while let Some(id) = stack.pop() {
match grammar.pool.node(id) {
Rule::NamedSymbol(sid) if sid != variable.name => {
// Rules that *directly* reference themselves don't cause a parsing loop.
productions.insert(sid);
}
Rule::Choice(range) => stack.extend_from_slice(grammar.pool.child_slice(range)),
Rule::Metadata { rule, .. } => stack.push(rule),
_ => {}
}
}
epsilon_transitions.insert(variable.name, productions);
}
for start_symbol in epsilon_transitions.keys() {
for &start_symbol in epsilon_transitions.keys() {
let mut visited = BTreeSet::new();
let mut path = Vec::new();
if let Some((start_idx, end_idx)) =
@ -197,7 +172,7 @@ fn validate_indirect_recursion(grammar: &InputGrammar) -> Result<(), IndirectRec
{
let cycle_symbols = path[start_idx..=end_idx]
.iter()
.map(|s| (*s).to_string())
.map(|&s| grammar.pool.resolve(s).to_string())
.collect();
return Err(IndirectRecursionError(cycle_symbols));
}
@ -206,40 +181,28 @@ fn validate_indirect_recursion(grammar: &InputGrammar) -> Result<(), IndirectRec
Ok(())
}
fn get_single_symbol_productions(rule: &Rule) -> BTreeSet<String> {
match rule {
Rule::NamedSymbol(name) => BTreeSet::from([name.clone()]),
Rule::Choice(choices) => choices
.iter()
.flat_map(get_single_symbol_productions)
.collect(),
Rule::Metadata { rule, .. } => get_single_symbol_productions(rule),
_ => BTreeSet::new(),
}
}
/// Perform a depth-first search to detect cycles in single state transitions.
fn get_cycle<'a>(
current: &'a str,
transitions: &'a IndexMap<&'a str, BTreeSet<String>>,
visited: &mut BTreeSet<&'a str>,
path: &mut Vec<&'a str>,
fn get_cycle(
current: StrId,
transitions: &IndexMap<StrId, BTreeSet<StrId>>,
visited: &mut BTreeSet<StrId>,
path: &mut Vec<StrId>,
) -> Option<(usize, usize)> {
if let Some(first_idx) = path.iter().position(|s| *s == current) {
path.push(current);
return Some((first_idx, path.len() - 1));
}
if visited.contains(current) {
if visited.contains(&current) {
return None;
}
path.push(current);
visited.insert(current);
if let Some(next_symbols) = transitions.get(current) {
if let Some(next_symbols) = transitions.get(&current) {
for next in next_symbols {
if let Some(cycle) = get_cycle(next, transitions, visited, path) {
if let Some(cycle) = get_cycle(*next, transitions, visited, path) {
return Some(cycle);
}
}
@ -253,45 +216,30 @@ fn get_cycle<'a>(
/// within the `precedences` lists, and also that there are no conflicting
/// precedence orderings declared in those lists.
fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()> {
// Check that no rule contains a named precedence that is not present in
// any of the `precedences` lists.
fn validate(
rule_name: &str,
rule: &Rule,
names: &FxHashSet<&String>,
) -> ValidatePrecedenceResult<()> {
match rule {
Rule::Repeat(rule) => validate(rule_name, rule, names),
Rule::Seq(elements) | Rule::Choice(elements) => elements
.iter()
.try_for_each(|e| validate(rule_name, e, names)),
Rule::Metadata { rule, params } => {
if let Precedence::Name(n) = &params.precedence
&& !names.contains(n)
{
Err(UndeclaredPrecedenceError {
precedence: n.clone(),
rule: rule_name.to_string(),
})?;
}
validate(rule_name, rule, names)?;
Ok(())
}
_ => Ok(()),
let display = |e: &PrecedenceEntry| match *e {
PrecedenceEntry::Name(sid) => format!("'{}'", grammar.pool.resolve(sid)),
PrecedenceEntry::Symbol(sid) => format!("$.{}", grammar.pool.resolve(sid)),
};
let cmp = |a: PrecedenceEntry, b: PrecedenceEntry| match (a, b) {
(PrecedenceEntry::Name(a), PrecedenceEntry::Name(b))
| (PrecedenceEntry::Symbol(a), PrecedenceEntry::Symbol(b)) => {
grammar.pool.resolve(a).cmp(grammar.pool.resolve(b))
}
}
(PrecedenceEntry::Name(_), PrecedenceEntry::Symbol(_)) => Ordering::Less,
(PrecedenceEntry::Symbol(_), PrecedenceEntry::Name(_)) => Ordering::Greater,
};
// For any two precedence names `a` and `b`, if `a` comes before `b`
// in some list, then it cannot come *after* `b` in any list.
let mut pairs = FxHashMap::default();
for list in &grammar.precedence_orderings {
for (i, mut entry1) in list.iter().enumerate() {
for mut entry2 in list.iter().skip(i + 1) {
if entry2 == entry1 {
for (i, mut entry1) in list.iter().copied().enumerate() {
for mut entry2 in list.iter().copied().skip(i + 1) {
if entry1 == entry2 {
continue;
}
let mut ordering = Ordering::Greater;
if entry1 > entry2 {
if cmp(entry1, entry2).is_gt() {
ordering = Ordering::Less;
mem::swap(&mut entry1, &mut entry2);
}
@ -302,8 +250,8 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()>
hash_map::Entry::Occupied(e) => {
if e.get() != &ordering {
Err(ConflictingPrecedenceOrderingError {
precedence_1: entry1.to_string(),
precedence_2: entry2.to_string(),
precedence_1: display(&entry1),
precedence_2: display(&entry2),
})?;
}
}
@ -315,17 +263,37 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()>
let precedence_names = grammar
.precedence_orderings
.iter()
.flat_map(|l| l.iter())
.filter_map(|p| {
if let PrecedenceEntry::Name(n) = p {
Some(n)
} else {
None
}
.flatten()
.filter_map(|p| match *p {
PrecedenceEntry::Name(sid) => Some(sid),
PrecedenceEntry::Symbol(_) => None,
})
.collect::<FxHashSet<&String>>();
.collect::<FxHashSet<_>>();
let mut stack = Vec::new();
for variable in &grammar.variables {
validate(&variable.name, &variable.rule, &precedence_names)?;
stack.clear();
stack.push(variable.root);
while let Some(id) = stack.pop() {
match grammar.pool.node(id) {
Rule::Repeat(inner) => stack.push(inner),
Rule::Seq(range) | Rule::Choice(range) => {
stack.extend_from_slice(grammar.pool.child_slice(range));
}
Rule::Metadata { params, rule } => {
if let Precedence::Name(sid) = grammar.pool.params(params).precedence
&& !precedence_names.contains(&sid)
{
Err(UndeclaredPrecedenceError {
precedence: grammar.pool.resolve(sid).to_string(),
rule: grammar.pool.resolve(variable.name).to_string(),
})?;
}
stack.push(rule);
}
_ => {}
}
}
}
Ok(())
@ -334,89 +302,188 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()>
#[cfg(test)]
mod tests {
use super::*;
use crate::grammars::VariableType;
use crate::{
grammars::Variable,
rules::{RuleId, RulePool},
};
#[test]
fn test_validate_precedences_with_undeclared_precedence() {
let grammar = InputGrammar {
precedence_orderings: vec![
vec![
PrecedenceEntry::Name("a".to_string()),
PrecedenceEntry::Name("b".to_string()),
],
vec![
PrecedenceEntry::Name("b".to_string()),
PrecedenceEntry::Name("c".to_string()),
PrecedenceEntry::Name("d".to_string()),
],
fn test_validate_precedences_with_undeclared_precedences() {
let mut pool = RulePool::default();
// v1: seq(prec_left('b', "w"), prec('c', "x"))
let v1 = {
let w = leaf(&mut pool, "w");
let left = prec_left(&mut pool, "b", w);
let x = leaf(&mut pool, "x");
let right = prec(&mut pool, "c", x);
pool.seq(&[left, right])
};
// v2: repeat(choice(prec_left('omg', "y"), prec('c', "z")))
let v2 = {
let y = leaf(&mut pool, "y");
let left = prec_left(&mut pool, "omg", y);
let z = leaf(&mut pool, "z");
let right = prec(&mut pool, "c", z);
let choice = pool.choice(&[left, right]);
pool.repeat(choice)
};
let v1_name = pool.intern("v1");
let v2_name = pool.intern("v2");
let precedence_orderings = vec![
vec![name_entry(&mut pool, "a"), name_entry(&mut pool, "b")],
vec![
name_entry(&mut pool, "b"),
name_entry(&mut pool, "c"),
name_entry(&mut pool, "d"),
],
];
let grammar = InputGrammar {
variables: vec![
Variable {
name: "v1".to_string(),
kind: VariableType::Named,
rule: Rule::Seq(vec![
Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
]),
name: v1_name,
root: v1,
},
Variable {
name: "v2".to_string(),
kind: VariableType::Named,
rule: Rule::repeat(Rule::Choice(vec![
Rule::prec_left(Precedence::Name("omg".to_string()), Rule::string("y")),
Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
])),
name: v2_name,
root: v2,
},
],
precedence_orderings,
pool,
..Default::default()
};
let result = validate_precedences(&grammar);
assert_eq!(
result.unwrap_err().to_string(),
"Undeclared precedence 'omg' in rule 'v2'",
validate_precedences(&grammar).unwrap_err(),
ValidatePrecedenceError::Undeclared(UndeclaredPrecedenceError {
precedence: "omg".to_string(),
rule: "v2".to_string()
})
);
}
#[test]
fn test_validate_precedences_with_conflicting_order() {
let mut pool = RulePool::default();
let precedence_orderings = vec![
vec![name_entry(&mut pool, "a"), name_entry(&mut pool, "b")],
vec![
name_entry(&mut pool, "b"),
name_entry(&mut pool, "c"),
name_entry(&mut pool, "a"),
],
];
let grammar = InputGrammar {
precedence_orderings: vec![
vec![
PrecedenceEntry::Name("a".to_string()),
PrecedenceEntry::Name("b".to_string()),
],
vec![
PrecedenceEntry::Name("b".to_string()),
PrecedenceEntry::Name("c".to_string()),
PrecedenceEntry::Name("a".to_string()),
],
],
variables: vec![
Variable {
name: "v1".to_string(),
kind: VariableType::Named,
rule: Rule::Seq(vec![
Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
]),
},
Variable {
name: "v2".to_string(),
kind: VariableType::Named,
rule: Rule::repeat(Rule::Choice(vec![
Rule::prec_left(Precedence::Name("a".to_string()), Rule::string("y")),
Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
])),
},
],
pool,
precedence_orderings,
..Default::default()
};
let result = validate_precedences(&grammar);
assert_eq!(
result.unwrap_err().to_string(),
"Conflicting orderings for precedences 'a' and 'b'",
validate_precedences(&grammar).unwrap_err(),
ValidatePrecedenceError::Ordering(ConflictingPrecedenceOrderingError {
precedence_1: "'a'".to_string(),
precedence_2: "'b'".to_string()
})
);
}
#[test]
fn test_validate_indirect_recursion() {
// a -> b -> a
let case1 = build_grammar(|p| {
let b_ref = named(p, "b");
let x = leaf(p, "x");
let a = p.choice(&[b_ref, x]);
let a_ref = named(p, "a");
let b = p.prec(Precedence::Integer(1), a_ref);
vec![
Variable {
name: p.intern("a"),
root: a,
},
Variable {
name: p.intern("b"),
root: b,
},
]
});
// A direct self-reference is allowed.
let case2 = build_grammar(|p| {
let a = named(p, "a");
vec![Variable {
name: p.intern("a"),
root: a,
}]
});
// b -> c -> d -> b, entered from a non-cycle start rule.
let case3 = build_grammar(|p| {
let x = leaf(p, "x");
let c_ref = named(p, "c");
let d_ref = named(p, "d");
let b_ref = named(p, "b");
vec![
Variable {
name: p.intern("a"),
root: x,
},
Variable {
name: p.intern("b"),
root: c_ref,
},
Variable {
name: p.intern("c"),
root: d_ref,
},
Variable {
name: p.intern("d"),
root: b_ref,
},
]
});
let err1 = IndirectRecursionError(vec!["a".to_string(), "b".to_string(), "a".to_string()]);
let err3 = IndirectRecursionError(vec![
"b".to_string(),
"c".to_string(),
"d".to_string(),
"b".to_string(),
]);
for (g, expected) in &[(case1, Err(err1)), (case2, Ok(())), (case3, Err(err3))] {
assert_eq!(*expected, validate_indirect_recursion(g));
}
}
fn named(pool: &mut RulePool, name: &str) -> RuleId {
let id = pool.intern(name);
pool.named_symbol(id)
}
fn leaf(pool: &mut RulePool, s: &str) -> RuleId {
let id = pool.intern(s);
pool.string(id)
}
fn prec(pool: &mut RulePool, name: &str, content: RuleId) -> RuleId {
let p = Precedence::Name(pool.intern(name));
pool.prec(p, content)
}
fn prec_left(pool: &mut RulePool, name: &str, content: RuleId) -> RuleId {
let p = Precedence::Name(pool.intern(name));
pool.prec_left(p, content)
}
fn name_entry(pool: &mut RulePool, name: &str) -> PrecedenceEntry {
PrecedenceEntry::Name(pool.intern(name))
}
fn build_grammar(build: impl FnOnce(&mut RulePool) -> Vec<Variable>) -> InputGrammar {
let mut pool = RulePool::default();
let variables = build(&mut pool);
InputGrammar {
pool,
variables,
..Default::default()
}
}
}

634
crates/generate/src/prepare_grammar/expand_repeats.rs
View file

@ -1,291 +1,459 @@
use std::mem;
use rustc_hash::FxHashMap;
use super::ExtractedSyntaxGrammar;
use crate::{
grammars::{Variable, VariableType},
rules::{Rule, Symbol},
grammars::{InputGrammar, Variable, VariableType},
prepare_grammar::extract_tokens::ExtractedGrammarMeta,
rules::{Rule, RuleId, RulePool, Symbol},
strpool::StrId,
};
#[derive(Default)]
struct Expander {
variable_name: String,
repeat_count_in_variable: usize,
preceding_symbol_count: usize,
auxiliary_variables: Vec<Variable>,
existing_repeats: FxHashMap<Rule, Symbol>,
preceding: usize,
aux: Vec<Variable>,
memo: FxHashMap<u64, Vec<(RuleId, Symbol)>>,
stack: Vec<Task>,
}
#[derive(Copy, Clone)]
enum Task {
Visit(RuleId),
Expand { id: RuleId, content: RuleId },
}
impl Expander {
fn expand_variable(&mut self, index: usize, variable: &mut Variable) -> bool {
self.variable_name.clear();
self.variable_name.push_str(&variable.name);
self.repeat_count_in_variable = 0;
let mut rule = Rule::Blank;
mem::swap(&mut rule, &mut variable.rule);
// In the special case of a hidden variable with a repetition at its top level,
// convert that rule itself into a binary tree structure instead of introducing
// another auxiliary rule.
if let (VariableType::Hidden, Rule::Repeat(repeated_content)) = (variable.kind, &rule) {
let inner_rule = self.expand_rule(repeated_content);
variable.rule = Self::wrap_rule_in_binary_tree(Symbol::non_terminal(index), inner_rule);
variable.kind = VariableType::Auxiliary;
return true;
}
variable.rule = self.expand_rule(&rule);
false
}
fn expand_rule(&mut self, rule: &Rule) -> Rule {
match rule {
// For choices, sequences, and metadata, descend into the child rules,
// replacing any nested repetitions.
Rule::Choice(elements) => Rule::Choice(
elements
.iter()
.map(|element| self.expand_rule(element))
.collect(),
),
Rule::Seq(elements) => Rule::Seq(
elements
.iter()
.map(|element| self.expand_rule(element))
.collect(),
),
Rule::Metadata { rule, params } => Rule::Metadata {
rule: Box::new(self.expand_rule(rule)),
params: params.clone(),
},
// For repetitions, introduce an auxiliary rule that contains the
// repeated content, but can also contain a recursive binary tree structure.
Rule::Repeat(content) => {
let inner_rule = self.expand_rule(content);
if let Some(existing_symbol) = self.existing_repeats.get(&inner_rule) {
return Rule::Symbol(*existing_symbol);
/// Post-order repeat expansion over one root. Children expand first, and `Reserved`
/// nodes are not descended.
fn expand_root(
&mut self,
pool: &mut RulePool,
root: RuleId,
var_name: StrId,
aux_repeat_counter: &mut u32,
) {
self.stack.clear();
self.stack.push(Task::Visit(root));
'walk: while let Some(task) = self.stack.pop() {
match task {
Task::Visit(id) => match pool.node(id) {
Rule::Repeat(content) => {
self.stack.push(Task::Expand { id, content });
self.stack.push(Task::Visit(content));
}
// For choices, sequences, and metadata, descend into the child rules,
// replacing any nested repetitions.
Rule::Seq(range) | Rule::Choice(range) => {
let base = self.stack.len();
for &c in pool.child_slice(range) {
self.stack.push(Task::Visit(c));
}
self.stack[base..].reverse();
}
Rule::Metadata { rule, .. } => self.stack.push(Task::Visit(rule)),
_ => {} // For primitive rules, don't change anything.
},
Task::Expand { id, content } => {
// For repetitions, introduce an auxiliary rule that contains the
// repeated content, but can also contain a recursive binary tree structure.
let hash = pool.subtree_hash(content);
if let Some(candidates) = self.memo.get(&hash) {
for &(node, symbol) in candidates {
if pool.subtree_eq(node, content) {
pool.set_node(id, Rule::from(symbol));
continue 'walk;
}
}
}
*aux_repeat_counter += 1;
let name = format!("{}_repeat{aux_repeat_counter}", pool.resolve(var_name));
let name = pool.intern(&name);
// Aux rules are appended after the original variables, so they occupy
// non-terminal indices `preceding..`.
let symbol = Symbol::non_terminal(self.preceding + self.aux.len());
self.memo.entry(hash).or_default().push((content, symbol));
let root = wrap_in_binary_tree(pool, symbol, content);
self.aux.push(Variable { name, root });
pool.set_node(id, Rule::from(symbol));
}
self.repeat_count_in_variable += 1;
let rule_name = format!(
"{}_repeat{}",
self.variable_name, self.repeat_count_in_variable
);
let repeat_symbol = Symbol::non_terminal(
self.preceding_symbol_count + self.auxiliary_variables.len(),
);
self.existing_repeats
.insert(inner_rule.clone(), repeat_symbol);
self.auxiliary_variables.push(Variable {
name: rule_name,
kind: VariableType::Auxiliary,
rule: Self::wrap_rule_in_binary_tree(repeat_symbol, inner_rule),
});
Rule::Symbol(repeat_symbol)
}
// For primitive rules, don't change anything.
_ => rule.clone(),
}
}
fn wrap_rule_in_binary_tree(symbol: Symbol, rule: Rule) -> Rule {
Rule::choice(vec![
Rule::Seq(vec![Rule::Symbol(symbol), Rule::Symbol(symbol)]),
rule,
])
}
}
pub(super) fn expand_repeats(mut grammar: ExtractedSyntaxGrammar) -> ExtractedSyntaxGrammar {
let mut expander = Expander {
variable_name: String::new(),
repeat_count_in_variable: 0,
preceding_symbol_count: grammar.variables.len(),
auxiliary_variables: Vec::new(),
existing_repeats: FxHashMap::default(),
};
for (i, variable) in grammar.variables.iter_mut().enumerate() {
let expanded_top_level_repetition = expander.expand_variable(i, variable);
// If a hidden variable had a top-level repetition and it was converted to
// a recursive rule, then it can't be inlined.
if expanded_top_level_repetition {
grammar
.variables_to_inline
.retain(|symbol| *symbol != Symbol::non_terminal(i));
/// Build a repeat aux body of `choice(seq(sym, sym), inner)`
fn wrap_in_binary_tree(pool: &mut RulePool, symbol: Symbol, inner: RuleId) -> RuleId {
let s1 = pool.push_node(Rule::from(symbol));
let s2 = pool.push_node(Rule::from(symbol));
let range = pool.push_children(&[s1, s2]);
let seq = pool.push_node(Rule::Seq(range));
let mut elements = vec![seq];
let mut stack = vec![inner];
while let Some(id) = stack.pop() {
if let Rule::Choice(range) = pool.node(id) {
let base = stack.len();
stack.extend_from_slice(pool.child_slice(range));
stack[base..].reverse();
} else if !elements.iter().any(|&e| pool.subtree_eq(e, id)) {
elements.push(id);
}
}
if elements.len() == 1 {
elements[0]
} else {
let range = pool.push_children(&elements);
pool.push_node(Rule::Choice(range))
}
}
grammar.variables.extend(expander.auxiliary_variables);
grammar
pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGrammarMeta) {
let mut expander = Expander {
preceding: grammar.variables.len(),
..Default::default()
};
for i in 0..grammar.variables.len() {
let Variable { name, root } = grammar.variables[i];
let mut aux_repeat_count = 0;
// A hidden variable with a top level repetition becomes its own recursive binary
// tree instead of gaining an auxiliary rule, and can no longer be inlined.
if meta.kinds[i] == VariableType::Hidden
&& let Rule::Repeat(content) = grammar.pool.node(root)
{
expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count);
grammar.variables[i].root =
wrap_in_binary_tree(&mut grammar.pool, Symbol::non_terminal(i), content);
meta.kinds[i] = VariableType::Auxiliary;
meta.inline.retain(|s| *s != Symbol::non_terminal(i));
continue;
}
expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count);
}
for var in expander.aux {
grammar.variables.push(var);
meta.kinds.push(VariableType::Auxiliary);
}
}
#[cfg(test)]
mod tests {
use crate::rules::SymbolType;
use super::*;
#[test]
fn test_basic_repeat_expansion() {
// Repeats nested inside of sequences and choices are expanded.
let grammar = expand_repeats(build_grammar(vec![Variable::named(
"rule0",
Rule::seq(vec![
Rule::terminal(10),
Rule::choice(vec![
Rule::repeat(Rule::terminal(11)),
Rule::repeat(Rule::terminal(12)),
]),
Rule::terminal(13),
]),
)]));
let mut pool = RulePool::default();
let r0 = {
let a = term(&mut pool, 10);
let ch = {
let r1 = {
let t = term(&mut pool, 11);
pool.repeat(t)
};
let r2 = {
let t = term(&mut pool, 12);
pool.repeat(t)
};
pool.choice(&[r1, r2])
};
let b = term(&mut pool, 13);
pool.seq(&[a, ch, b])
};
let name = pool.intern("rule0");
let (mut grammar, meta) = expand(
pool,
vec![Variable { name, root: r0 }],
vec![VariableType::Named],
);
let names = grammar
.variables
.iter()
.map(|v| grammar.pool.resolve(v.name))
.collect::<Vec<_>>();
assert_eq!(names, ["rule0", "rule0_repeat1", "rule0_repeat2"]);
assert_eq!(
grammar.variables,
vec![
Variable::named(
"rule0",
Rule::seq(vec![
Rule::terminal(10),
Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),]),
Rule::terminal(13),
])
),
Variable::auxiliary(
"rule0_repeat1",
Rule::choice(vec![
Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
Rule::terminal(11),
])
),
Variable::auxiliary(
"rule0_repeat2",
Rule::choice(vec![
Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
Rule::terminal(12),
])
),
meta.kinds,
[
VariableType::Named,
VariableType::Auxiliary,
VariableType::Auxiliary
]
);
// rule0: seq(terminal(10), choice(non_terminal(1), non_terminal(2)), terminal(13))
let e0 = {
let pool = &mut grammar.pool;
let (a, b) = (term(pool, 10), term(pool, 13));
let ch = {
let (x, y) = (non_term(pool, 1), non_term(pool, 2));
pool.choice(&[x, y])
};
pool.seq(&[a, ch, b])
};
assert!(grammar.pool.subtree_eq(grammar.variables[0].root, e0));
// rule0_repeat1: choice(seq(nt1, nt1), terminal(11))
let e1 = {
let p = &mut grammar.pool;
let sq = {
let (x, y) = (non_term(p, 1), non_term(p, 1));
p.seq(&[x, y])
};
let t = term(p, 11);
p.choice(&[sq, t])
};
assert!(grammar.pool.subtree_eq(grammar.variables[1].root, e1));
// rule0_repeat2: choice(seq(nt2, nt2), terminal(12))
let e2 = {
let p = &mut grammar.pool;
let sq = {
let (x, y) = (non_term(p, 2), non_term(p, 2));
p.seq(&[x, y])
};
let t = term(p, 12);
p.choice(&[sq, t])
};
assert!(grammar.pool.subtree_eq(grammar.variables[2].root, e2));
}
#[test]
fn test_repeat_deduplication() {
// Terminal 4 appears inside of a repeat in three different places.
let grammar = expand_repeats(build_grammar(vec![
Variable::named(
"rule0",
Rule::choice(vec![
Rule::seq(vec![Rule::terminal(1), Rule::repeat(Rule::terminal(4))]),
Rule::seq(vec![Rule::terminal(2), Rule::repeat(Rule::terminal(4))]),
]),
),
Variable::named(
"rule1",
Rule::seq(vec![Rule::terminal(3), Rule::repeat(Rule::terminal(4))]),
),
]));
// repeat(terminal(4)) appears in 3 places. Only one aux rule is made
let mut pool = RulePool::default();
let r0 = {
let s1 = {
let t = term(&mut pool, 1);
let r = {
let x = term(&mut pool, 4);
pool.repeat(x)
};
pool.seq(&[t, r])
};
let s2 = {
let t = term(&mut pool, 2);
let r = {
let x = term(&mut pool, 4);
pool.repeat(x)
};
pool.seq(&[t, r])
};
pool.choice(&[s1, s2])
};
let r1 = {
let t = term(&mut pool, 3);
let r = {
let x = term(&mut pool, 4);
pool.repeat(x)
};
pool.seq(&[t, r])
};
let (n0, n1) = (pool.intern("rule0"), pool.intern("rule1"));
let variables = vec![
Variable { name: n0, root: r0 },
Variable { name: n1, root: r1 },
];
let (mut g, meta) = expand(pool, variables, vec![VariableType::Named; 2]);
// Only one auxiliary rule is created for repeating terminal 4.
let names = g
.variables
.iter()
.map(|v| g.pool.resolve(v.name))
.collect::<Vec<_>>();
assert_eq!(names, ["rule0", "rule1", "rule0_repeat1"]);
assert_eq!(
grammar.variables,
vec![
Variable::named(
"rule0",
Rule::choice(vec![
Rule::seq(vec![Rule::terminal(1), Rule::non_terminal(2)]),
Rule::seq(vec![Rule::terminal(2), Rule::non_terminal(2)]),
])
),
Variable::named(
"rule1",
Rule::seq(vec![Rule::terminal(3), Rule::non_terminal(2),])
),
Variable::auxiliary(
"rule0_repeat1",
Rule::choice(vec![
Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
Rule::terminal(4),
])
)
meta.kinds,
[
VariableType::Named,
VariableType::Named,
VariableType::Auxiliary
]
);
// rule0: choice(seq(t1, nt2), sseq(t2, nt2))
let e0 = {
let p = &mut g.pool;
let s1 = {
let (t, n) = (term(p, 1), non_term(p, 2));
p.seq(&[t, n])
};
let s2 = {
let (t, n) = (term(p, 2), non_term(p, 2));
p.seq(&[t, n])
};
p.choice(&[s1, s2])
};
assert!(g.pool.subtree_eq(g.variables[0].root, e0));
// rule1: seq(t3, nt2)
let e1 = {
let p = &mut g.pool;
let (t, n) = (term(p, 3), non_term(p, 2));
p.seq(&[t, n])
};
assert!(g.pool.subtree_eq(g.variables[1].root, e1));
// rule0_repeat1: choice(seq(nt2, nt2), terminal(4))
let e2 = {
let p = &mut g.pool;
let sq = {
let (x, y) = (non_term(p, 2), non_term(p, 2));
p.seq(&[x, y])
};
let t = term(p, 4);
p.choice(&[sq, t])
};
assert!(g.pool.subtree_eq(g.variables[2].root, e2));
}
#[test]
fn test_expansion_of_nested_repeats() {
let grammar = expand_repeats(build_grammar(vec![Variable::named(
"rule0",
Rule::seq(vec![
Rule::terminal(10),
Rule::repeat(Rule::seq(vec![
Rule::terminal(11),
Rule::repeat(Rule::terminal(12)),
])),
]),
)]));
// Nested repeats expand inside out. The inner one becomes `rule0_repeat1` (nt1),
// and then outer one (now referencing it) becomes rule0_repeat2
let mut pool = RulePool::default();
let r0 = {
let t10 = term(&mut pool, 10);
let outer = {
let t11 = term(&mut pool, 11);
let inner = {
let t = term(&mut pool, 12);
pool.repeat(t)
};
let content = pool.seq(&[t11, inner]);
pool.repeat(content)
};
pool.seq(&[t10, outer])
};
let name = pool.intern("rule0");
let (mut g, meta) = expand(
pool,
vec![Variable { name, root: r0 }],
vec![VariableType::Named],
);
let names = g
.variables
.iter()
.map(|v| g.pool.resolve(v.name))
.collect::<Vec<_>>();
assert_eq!(names, ["rule0", "rule0_repeat1", "rule0_repeat2"]);
assert_eq!(
grammar.variables,
vec![
Variable::named(
"rule0",
Rule::seq(vec![Rule::terminal(10), Rule::non_terminal(2),])
),
Variable::auxiliary(
"rule0_repeat1",
Rule::choice(vec![
Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
Rule::terminal(12),
])
),
Variable::auxiliary(
"rule0_repeat2",
Rule::choice(vec![
Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
Rule::seq(vec![Rule::terminal(11), Rule::non_terminal(1),]),
])
),
meta.kinds,
[
VariableType::Named,
VariableType::Auxiliary,
VariableType::Auxiliary
]
);
// rule0: seq(terminal(10), non_terminal(2))
let e0 = {
let p = &mut g.pool;
let (t, n) = (term(p, 10), non_term(p, 2));
p.seq(&[t, n])
};
assert!(g.pool.subtree_eq(g.variables[0].root, e0));
// rule0_repeat2 (outer): choice(seq(nt2, nt2), seq(terminal(11), nt1))
let e2 = {
let p = &mut g.pool;
let sq1 = {
let (x, y) = (non_term(p, 2), non_term(p, 2));
p.seq(&[x, y])
};
let sq2 = {
let (t, n) = (term(p, 11), non_term(p, 1));
p.seq(&[t, n])
};
p.choice(&[sq1, sq2])
};
assert!(g.pool.subtree_eq(g.variables[2].root, e2));
}
#[test]
fn test_expansion_of_repeats_at_top_of_hidden_rules() {
let grammar = expand_repeats(build_grammar(vec![
Variable::named("rule0", Rule::non_terminal(1)),
Variable::hidden(
"_rule1",
Rule::repeat(Rule::choice(vec![Rule::terminal(11), Rule::terminal(12)])),
),
]));
assert_eq!(
grammar.variables,
vec![
Variable::named("rule0", Rule::non_terminal(1),),
Variable::auxiliary(
"_rule1",
Rule::choice(vec![
Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1)]),
Rule::terminal(11),
Rule::terminal(12),
]),
),
]
// A hidden rule whos whole body is a repeat becomes its own recursive binary
// tree (using its own symbol) instead of gaining a separate aux rule, and is
// reclassified as Axuiliary.
let mut pool = RulePool::default();
let r0 = non_term(&mut pool, 1);
let r1 = {
let ch = {
let (a, b) = (term(&mut pool, 11), term(&mut pool, 12));
pool.choice(&[a, b])
};
pool.repeat(ch)
};
let (n0, n1) = (pool.intern("rule0"), pool.intern("_rule1"));
let variables = vec![
Variable { name: n0, root: r0 },
Variable { name: n1, root: r1 },
];
let (mut g, meta) = expand(
pool,
variables,
vec![VariableType::Named, VariableType::Hidden],
);
// No separate aux rule: _rule1 absorbed during recursion
let names = g
.variables
.iter()
.map(|v| g.pool.resolve(v.name))
.collect::<Vec<_>>();
assert_eq!(names, ["rule0", "_rule1"]);
assert_eq!(meta.kinds, [VariableType::Named, VariableType::Auxiliary]);
// rule0: non_terminal(1) (unchanged)
assert_eq!(
g.pool.node(g.variables[0].root),
Rule::Sym {
kind: SymbolType::NonTerminal,
index: 1
}
);
// _rule1: choice(seq(nt1, nt1), terminal(11), terminal(12)) (inner choice flattened)
let e1 = {
let p = &mut g.pool;
let sq = {
let (x, y) = (non_term(p, 1), non_term(p, 1));
p.seq(&[x, y])
};
let (a, b) = (term(p, 11), term(p, 12));
p.choice(&[sq, a, b])
};
assert!(g.pool.subtree_eq(g.variables[1].root, e1));
}
fn build_grammar(variables: Vec<Variable>) -> ExtractedSyntaxGrammar {
ExtractedSyntaxGrammar {
fn term(p: &mut RulePool, i: u32) -> RuleId {
p.push_node(Rule::Sym {
kind: SymbolType::Terminal,
index: i,
})
}
fn non_term(p: &mut RulePool, i: u32) -> RuleId {
p.push_node(Rule::Sym {
kind: SymbolType::NonTerminal,
index: i,
})
}
fn expand(
pool: RulePool,
variables: Vec<Variable>,
kinds: Vec<VariableType>,
) -> (InputGrammar, ExtractedGrammarMeta) {
let mut grammar = InputGrammar {
pool,
variables,
..Default::default()
}
};
let mut meta = ExtractedGrammarMeta {
kinds,
..Default::default()
};
expand_repeats(&mut grammar, &mut meta);
(grammar, meta)
}
}

1077
crates/generate/src/prepare_grammar/expand_tokens.rs
View file

File diff suppressed because it is too large Load diff

357
crates/generate/src/prepare_grammar/extract_default_aliases.rs
View file

@ -1,5 +1,8 @@
use std::collections::BTreeMap;
use crate::{
grammars::{LexicalGrammar, SyntaxGrammar},
grammars::{InputGrammar, ProductionStore},
prepare_grammar::extract_tokens::ExtractedGrammarMeta,
rules::{Alias, AliasMap, Symbol, SymbolType},
};
@ -19,52 +22,50 @@ struct SymbolStatus {
// ensures that the children of an `ERROR` node have symbols that are consistent with the way that
// they would appear in a valid syntax tree.
pub(super) fn extract_default_aliases(
syntax_grammar: &mut SyntaxGrammar,
lexical_grammar: &LexicalGrammar,
g: &InputGrammar,
meta: &ExtractedGrammarMeta,
out: &mut ProductionStore,
) -> AliasMap {
let mut terminal_status_list = vec![SymbolStatus::default(); lexical_grammar.variables.len()];
let mut non_terminal_status_list =
vec![SymbolStatus::default(); syntax_grammar.variables.len()];
let mut external_status_list =
vec![SymbolStatus::default(); syntax_grammar.external_tokens.len()];
let mut terminal_status_list = vec![SymbolStatus::default(); meta.lexical_variables.len()];
let mut non_terminal_status_list = vec![SymbolStatus::default(); g.variables.len()];
let mut external_status_list = vec![SymbolStatus::default(); meta.external_tokens.len()];
// For each grammar symbol, find all of the aliases under which the symbol appears,
// and determine whether or not the symbol ever appears *unaliased*.
for variable in &syntax_grammar.variables {
for production in &variable.productions {
for step in &production.steps {
let status = match step.symbol.kind {
SymbolType::External => &mut external_status_list[step.symbol.index],
SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
panic!("Unexpected end token")
}
};
// Default aliases don't work for inlined variables.
if syntax_grammar.variables_to_inline.contains(&step.symbol) {
continue;
for prod in &out.productions {
for step in &out.steps[prod.step_range()] {
let symbol = step.symbol();
let status = match symbol.kind {
SymbolType::External => &mut external_status_list[symbol.index],
SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
SymbolType::Terminal => &mut terminal_status_list[symbol.index],
SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
panic!("Unexpected end token")
}
};
if let Some(alias) = &step.alias {
if let Some(count_for_alias) = status
.aliases
.iter_mut()
.find_map(|(a, count)| if a == alias { Some(count) } else { None })
{
*count_for_alias += 1;
} else {
status.aliases.push((alias.clone(), 1));
}
// Default aliases don't work for inlined variables.
if meta.inline.contains(&symbol) {
continue;
}
if let Some(alias) = step.alias() {
if let Some(count_for_alias) = status
.aliases
.iter_mut()
.find_map(|(a, count)| (*a == alias).then_some(count))
{
*count_for_alias += 1;
} else {
status.appears_unaliased = true;
status.aliases.push((alias, 1));
}
} else {
status.appears_unaliased = true;
}
}
}
for symbol in &syntax_grammar.extra_symbols {
for symbol in &meta.extra_symbols {
let status = match symbol.kind {
SymbolType::External => &mut external_status_list[symbol.index],
SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
@ -74,27 +75,27 @@ pub(super) fn extract_default_aliases(
status.appears_unaliased = true;
}
let symbols_with_statuses = (terminal_status_list
let symbols_with_statuses = terminal_status_list
.iter_mut()
.enumerate()
.map(|(i, status)| (Symbol::terminal(i), status)))
.chain(
non_terminal_status_list
.iter_mut()
.enumerate()
.map(|(i, status)| (Symbol::non_terminal(i), status)),
)
.chain(
external_status_list
.iter_mut()
.enumerate()
.map(|(i, status)| (Symbol::external(i), status)),
);
.map(|(i, status)| (Symbol::terminal(i), status))
.chain(
non_terminal_status_list
.iter_mut()
.enumerate()
.map(|(i, status)| (Symbol::non_terminal(i), status)),
)
.chain(
external_status_list
.iter_mut()
.enumerate()
.map(|(i, status)| (Symbol::external(i), status)),
);
// For each symbol that always appears aliased, find the alias that occurs most often,
// and designate that alias as the symbol's "default alias". Store all of these
// default aliases in a map that will be returned.
let mut result = AliasMap::new();
let mut result = BTreeMap::new();
for (symbol, status) in symbols_with_statuses {
if status.appears_unaliased {
status.aliases.clear();
@ -103,10 +104,10 @@ pub(super) fn extract_default_aliases(
.iter()
.enumerate()
.max_by_key(|(i, (_, count))| (count, -(*i as i64)))
.map(|(_, entry)| entry.clone())
.map(|(_, entry)| *entry)
{
status.aliases.clear();
status.aliases.push(default_entry.clone());
status.aliases.push(default_entry);
result.insert(symbol, default_entry.0);
}
}
@ -114,30 +115,31 @@ pub(super) fn extract_default_aliases(
// Wherever a symbol is aliased as its default alias, remove the usage of the alias,
// because it will now be redundant.
let mut alias_positions_to_clear = Vec::new();
for variable in &mut syntax_grammar.variables {
for &(p_start, p_end) in &out.var_prods {
alias_positions_to_clear.clear();
for (i, production) in variable.productions.iter().enumerate() {
for (j, step) in production.steps.iter().enumerate() {
let status = match step.symbol.kind {
SymbolType::External => &mut external_status_list[step.symbol.index],
SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
let productions = &out.productions[p_start as usize..p_end as usize];
for (i, prod) in productions.iter().enumerate() {
for (j, step) in out.steps[prod.step_range()].iter().enumerate() {
let symbol = step.symbol();
let status = match symbol.kind {
SymbolType::External => &external_status_list[symbol.index],
SymbolType::Terminal => &terminal_status_list[symbol.index],
SymbolType::NonTerminal => &non_terminal_status_list[symbol.index],
SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
panic!("Unexpected end token")
}
};
// If this step is aliased as the symbol's default alias, then remove that alias.
if step.alias.is_some()
&& step.alias.as_ref() == status.aliases.first().map(|t| &t.0)
{
if step.alias().is_some() && step.alias() == status.aliases.first().map(|t| t.0) {
let mut other_productions_must_use_this_alias_at_this_index = false;
for (other_i, other_production) in variable.productions.iter().enumerate() {
for (other_i, other_prod) in productions.iter().enumerate() {
let other_steps = &out.steps[other_prod.step_range()];
if other_i != i
&& other_production.steps.len() > j
&& other_production.steps[j].alias == step.alias
&& result.get(&other_production.steps[j].symbol) != step.alias.as_ref()
&& other_steps.len() > j
&& other_steps[j].alias() == step.alias()
&& result.get(&other_steps[j].symbol()) != step.alias().as_ref()
{
other_productions_must_use_this_alias_at_this_index = true;
break;
@ -145,14 +147,14 @@ pub(super) fn extract_default_aliases(
}
if !other_productions_must_use_this_alias_at_this_index {
alias_positions_to_clear.push((i, j));
alias_positions_to_clear.push(prod.steps_start as usize + j);
}
}
}
}
for (production_index, step_index) in &alias_positions_to_clear {
variable.productions[*production_index].steps[*step_index].alias = None;
for &step_index in &alias_positions_to_clear {
out.steps[step_index].set_alias(None);
}
}
@ -163,140 +165,109 @@ pub(super) fn extract_default_aliases(
mod tests {
use super::*;
use crate::{
grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType},
nfa::Nfa,
grammars::{Production, ProductionStep, Variable, VariableType},
prepare_grammar::extract_tokens::LexicalToken,
rules::{Precedence, RulePool},
};
#[test]
fn test_extract_simple_aliases() {
let mut syntax_grammar = SyntaxGrammar {
variables: vec![
SyntaxVariable {
name: "v1".to_owned(),
kind: VariableType::Named,
productions: vec![Production {
dynamic_precedence: 0,
steps: vec![
ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
ProductionStep::new(Symbol::terminal(2)).with_alias("a3", true),
ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
],
}],
},
SyntaxVariable {
name: "v2".to_owned(),
kind: VariableType::Named,
productions: vec![Production {
dynamic_precedence: 0,
steps: vec![
// Token 0 is always aliased as "a1".
ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
// Token 1 is aliased within rule `v1` above, but not here.
ProductionStep::new(Symbol::terminal(1)),
// Token 2 is aliased differently here than in `v1`. The alias from
// `v1` should be promoted to the default alias, because `v1` appears
// first in the grammar.
ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
// Token 3 is also aliased differently here than in `v1`. In this case,
// this alias should be promoted to the default alias, because it is
// used a greater number of times (twice).
ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
],
}],
},
],
let mut pool = RulePool::default();
let dummy = pool.intern("_");
let root = pool.blank();
// v1: every token aliased.
let v1 = vec![
aliased(&mut pool, Symbol::terminal(0), "a1"),
aliased(&mut pool, Symbol::terminal(1), "a2"),
aliased(&mut pool, Symbol::terminal(2), "a3"),
aliased(&mut pool, Symbol::terminal(3), "a4"),
];
// v2: t0 same alias, t1 unaliased, t2 aliased differently, t3 aliased twice as a6
let v2 = vec![
aliased(&mut pool, Symbol::terminal(0), "a1"),
plain(Symbol::terminal(1)),
aliased(&mut pool, Symbol::terminal(2), "a5"),
aliased(&mut pool, Symbol::terminal(3), "a6"),
aliased(&mut pool, Symbol::terminal(3), "a6"),
];
let mut out = ProductionStore::default();
for steps in [v1, v2] {
let ps = out.productions.len() as u32;
let steps_start = out.steps.len() as u32;
out.steps.extend_from_slice(&steps);
out.productions.push(Production {
steps_start,
steps_len: steps.len() as u32,
dynamic_precedence: 0,
});
out.var_prods.push((ps, out.productions.len() as u32));
}
let g = InputGrammar {
variables: (0..2).map(|_| Variable { name: dummy, root }).collect(),
pool,
..Default::default()
};
let meta = ExtractedGrammarMeta {
lexical_variables: (0..4)
.map(|_| LexicalToken {
name: dummy,
kind: VariableType::Anonymous,
root,
})
.collect(),
..Default::default()
};
let lexical_grammar = LexicalGrammar {
nfa: Nfa::new(),
variables: vec![
LexicalVariable {
name: "t0".to_string(),
kind: VariableType::Anonymous,
implicit_precedence: 0,
start_state: 0,
},
LexicalVariable {
name: "t1".to_string(),
kind: VariableType::Anonymous,
implicit_precedence: 0,
start_state: 0,
},
LexicalVariable {
name: "t2".to_string(),
kind: VariableType::Anonymous,
implicit_precedence: 0,
start_state: 0,
},
LexicalVariable {
name: "t3".to_string(),
kind: VariableType::Anonymous,
implicit_precedence: 0,
start_state: 0,
},
],
let pool_map = extract_default_aliases(&g, &meta, &mut out);
// t0 -> a1, t2 -> a3 (v1 wins the tie from appearing first), t3 -> a6 (used twice)
// t1 -> none (appears unaliased in v2).
let default = |s| {
pool_map
.get(&s)
.map(|a| (g.pool.resolve(a.value), a.is_named))
};
assert_eq!(pool_map.len(), 3);
assert_eq!(default(Symbol::terminal(0)), Some(("a1", true)));
assert_eq!(default(Symbol::terminal(2)), Some(("a3", true)));
assert_eq!(default(Symbol::terminal(3)), Some(("a6", true)));
assert_eq!(default(Symbol::terminal(1)), None);
let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
assert_eq!(default_aliases.len(), 3);
// Steps carrying their symbol's default are cleared. The rest keep their alias.
let step_alias = |i: usize| {
out.steps[i]
.alias()
.map(|a| (g.pool.resolve(a.value), a.is_named))
};
assert_eq!(step_alias(0), None); // v1 t0(a1) = default
assert_eq!(step_alias(1), Some(("a2", true))); // v1 t1(a2), no default
assert_eq!(step_alias(2), None); // v1 t2(a3) = default
assert_eq!(step_alias(3), Some(("a4", true))); // v1 t3(a4) != a6
assert_eq!(step_alias(4), None); // v2 t0(a1) = default
assert_eq!(step_alias(5), None); // v2 t1 unaliased
assert_eq!(step_alias(6), Some(("a5", true))); // v2 t2(a5) != a3
assert_eq!(step_alias(7), None); // v2 t3(a6) = default
assert_eq!(step_alias(8), None); // v2 t3(a6) = default
}
assert_eq!(
default_aliases.get(&Symbol::terminal(0)),
Some(&Alias {
value: "a1".to_string(),
fn aliased(pool: &mut RulePool, symbol: Symbol, name: &str) -> ProductionStep {
let value = pool.intern(name);
ProductionStep::pack(
symbol,
Precedence::None,
None,
Some(Alias {
value,
is_named: true,
})
);
assert_eq!(
default_aliases.get(&Symbol::terminal(2)),
Some(&Alias {
value: "a3".to_string(),
is_named: true,
})
);
assert_eq!(
default_aliases.get(&Symbol::terminal(3)),
Some(&Alias {
value: "a6".to_string(),
is_named: true,
})
);
assert_eq!(default_aliases.get(&Symbol::terminal(1)), None);
assert_eq!(
syntax_grammar.variables,
vec![
SyntaxVariable {
name: "v1".to_owned(),
kind: VariableType::Named,
productions: vec![Production {
dynamic_precedence: 0,
steps: vec![
ProductionStep::new(Symbol::terminal(0)),
ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
ProductionStep::new(Symbol::terminal(2)),
ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
],
},],
},
SyntaxVariable {
name: "v2".to_owned(),
kind: VariableType::Named,
productions: vec![Production {
dynamic_precedence: 0,
steps: vec![
ProductionStep::new(Symbol::terminal(0)),
ProductionStep::new(Symbol::terminal(1)),
ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
ProductionStep::new(Symbol::terminal(3)),
ProductionStep::new(Symbol::terminal(3)),
],
},],
},
]
);
}),
None,
0,
)
}
fn plain(symbol: Symbol) -> ProductionStep {
ProductionStep::pack(symbol, Precedence::None, None, None, None, 0)
}
}

1163
crates/generate/src/prepare_grammar/extract_tokens.rs
View file

File diff suppressed because it is too large Load diff

1078
crates/generate/src/prepare_grammar/flatten_grammar.rs
View file

File diff suppressed because it is too large Load diff

601
crates/generate/src/prepare_grammar/intern_symbols.rs
View file

@ -1,16 +1,17 @@
use rustc_hash::{FxBuildHasher, FxHashMap};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use super::InternedGrammar;
use crate::{
Diagnostic,
grammars::{InputGrammar, ReservedWordContext, Variable, VariableType},
rules::{Rule, Symbol},
grammars::{InputGrammar, VariableType},
rules::{Rule, RuleId, RulePool, Symbol},
strpool::StrId,
};
pub type InternSymbolsResult<T> = Result<T, InternSymbolsError>;
#[derive(Debug, Error, Serialize, Deserialize)]
#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
pub enum InternSymbolsError {
#[error("A grammar's start rule must be visible.")]
HiddenStartRule,
@ -24,193 +25,189 @@ pub enum InternSymbolsError {
UndefinedWordToken(String),
}
pub(super) fn intern_symbols(
grammar: &InputGrammar,
diagnostics: &mut Vec<Diagnostic>,
) -> InternSymbolsResult<InternedGrammar> {
let interner = Interner { grammar };
/// The non-pool outputs of the intern pass. The rule bodies are rewritten in place
/// (`NamedSymbol`->`Sym`). This carries the derived metadata that travels alongside
/// them.
#[derive(Debug)]
pub(super) struct InternedGrammarMeta {
pub kinds: Vec<VariableType>,
/// Per external token: its name (if a named symbol) and kind
pub external_tokens: Vec<(Option<StrId>, VariableType)>,
pub supertypes: Vec<Symbol>,
pub conflicts: Vec<Vec<Symbol>>,
pub inline: Vec<Symbol>,
pub word: Option<Symbol>,
}
if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden {
pub(super) fn intern_symbols(
grammar: &mut InputGrammar,
diagnostics: &mut Vec<Diagnostic>,
) -> InternSymbolsResult<InternedGrammarMeta> {
let InputGrammar {
pool,
variables,
external_roots,
extra_roots,
reserved_sets,
..
} = grammar;
// StrId -> the symbol for a variable or external token name. An external whose name
// shadows a variable resolves to the variable (variables are inserted first).
let mut name_of_symbol =
FxHashMap::with_capacity_and_hasher(variables.len() + external_roots.len(), FxBuildHasher);
for (i, v) in variables.iter().enumerate() {
name_of_symbol.insert(v.name, Symbol::non_terminal(i));
}
for (i, &root) in external_roots.iter().enumerate() {
if let Rule::NamedSymbol(sid) = pool.node(root) {
name_of_symbol
.entry(sid)
.or_insert_with(|| Symbol::external(i));
}
}
if variable_type_for_name(pool.resolve(variables[0].name)) == VariableType::Hidden {
Err(InternSymbolsError::HiddenStartRule)?;
}
let mut variables = Vec::with_capacity(grammar.variables.len());
for variable in &grammar.variables {
variables.push(Variable {
name: variable.name.clone(),
kind: variable_type_for_name(&variable.name),
rule: interner.intern_rule(&variable.rule, Some(&variable.name), diagnostics)?,
});
}
// External names/kinds read before the in-place rewrite consumes them.
let external_tokens = external_roots
.iter()
.map(|&root| match pool.node(root) {
Rule::NamedSymbol(sid) => (Some(sid), variable_type_for_name(pool.resolve(sid))),
_ => (None, VariableType::Anonymous),
})
.collect::<Vec<(Option<StrId>, VariableType)>>();
let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len());
for external_token in &grammar.external_tokens {
let rule = interner.intern_rule(external_token, None, diagnostics)?;
let (name, kind) = if let Rule::NamedSymbol(name) = external_token {
(name.clone(), variable_type_for_name(name))
} else {
(String::new(), VariableType::Anonymous)
};
external_tokens.push(Variable { name, kind, rule });
}
let mut kinds = variables
.iter()
.map(|v| variable_type_for_name(pool.resolve(v.name)))
.collect::<Vec<VariableType>>();
let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len());
for extra_token in &grammar.extra_symbols {
extra_symbols.push(interner.intern_rule(extra_token, None, diagnostics)?);
let mut stack = Vec::new();
for v in variables.iter() {
intern_root(
pool,
v.root,
Some(v.name),
&name_of_symbol,
diagnostics,
&mut stack,
)?;
}
let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len());
for supertype_symbol_name in &grammar.supertype_symbols {
supertype_symbols.push(interner.intern_name(supertype_symbol_name).ok_or_else(|| {
InternSymbolsError::UndefinedSupertype(supertype_symbol_name.clone())
})?);
for &root in external_roots.iter().chain(extra_roots.iter()) {
intern_root(pool, root, None, &name_of_symbol, diagnostics, &mut stack)?;
}
let mut reserved_words = Vec::with_capacity(grammar.reserved_words.len());
for reserved_word_set in &grammar.reserved_words {
let mut interned_set = Vec::with_capacity(reserved_word_set.reserved_words.len());
for rule in &reserved_word_set.reserved_words {
interned_set.push(interner.intern_rule(rule, None, diagnostics)?);
}
reserved_words.push(ReservedWordContext {
name: reserved_word_set.name.clone(),
reserved_words: interned_set,
});
}
let mut expected_conflicts = Vec::with_capacity(grammar.expected_conflicts.len());
for conflict in &grammar.expected_conflicts {
let mut interned_conflict = Vec::with_capacity(conflict.len());
for name in conflict {
interned_conflict.push(
interner
.intern_name(name)
.ok_or_else(|| InternSymbolsError::UndefinedConflict(name.clone()))?,
);
}
expected_conflicts.push(interned_conflict);
}
let mut variables_to_inline = Vec::new();
for name in &grammar.variables_to_inline {
if let Some(symbol) = interner.intern_name(name) {
variables_to_inline.push(symbol);
for set in reserved_sets.iter() {
for &root in &set.roots {
intern_root(pool, root, None, &name_of_symbol, diagnostics, &mut stack)?;
}
}
let word_token = if let Some(name) = grammar.word_token.as_ref() {
Some(
interner
.intern_name(name)
.ok_or_else(|| InternSymbolsError::UndefinedWordToken(name.clone()))?,
)
} else {
None
};
let lookup = |sid: StrId| name_of_symbol.get(&sid).copied();
let supertypes = grammar
.supertype_names
.iter()
.map(|&s| {
lookup(s)
.ok_or_else(|| InternSymbolsError::UndefinedSupertype(pool.resolve(s).to_string()))
})
.collect::<InternSymbolsResult<Vec<_>>>()?;
let conflicts = grammar
.conflict_names
.iter()
.map(|c| {
c.iter()
.map(|&s| {
lookup(s).ok_or_else(|| {
InternSymbolsError::UndefinedConflict(pool.resolve(s).to_string())
})
})
.collect::<InternSymbolsResult<Vec<_>>>()
})
.collect::<InternSymbolsResult<Vec<_>>>()?;
for (i, variable) in variables.iter_mut().enumerate() {
if supertype_symbols.contains(&Symbol::non_terminal(i)) {
variable.kind = VariableType::Hidden;
// Unkown inline names are silently skipped.
let inline = grammar
.inline_names
.iter()
.filter_map(|&s| lookup(s))
.collect();
let word = grammar
.word_name
.map(|s| {
lookup(s)
.ok_or_else(|| InternSymbolsError::UndefinedWordToken(pool.resolve(s).to_string()))
})
.transpose()?;
for s in &supertypes {
if s.is_non_terminal() {
kinds[s.index] = VariableType::Hidden;
}
}
Ok(InternedGrammar {
variables,
Ok(InternedGrammarMeta {
kinds,
external_tokens,
extra_symbols,
expected_conflicts,
variables_to_inline,
supertype_symbols,
word_token,
precedence_orderings: grammar.precedence_orderings.clone(),
reserved_word_sets: reserved_words,
supertypes,
conflicts,
inline,
word,
})
}
struct Interner<'a> {
grammar: &'a InputGrammar,
}
// Iterative pre-order walk rewriting `NamedSymbol`->`Sym` in place.
fn intern_root(
pool: &mut RulePool,
root: RuleId,
var_name: Option<StrId>,
name_of_symbol: &FxHashMap<StrId, Symbol>,
diagnostics: &mut Vec<Diagnostic>,
stack: &mut Vec<RuleId>,
) -> InternSymbolsResult<()> {
stack.clear();
stack.push(root);
impl Interner<'_> {
fn intern_rule(
&self,
rule: &Rule,
name: Option<&str>,
diagnostics: &mut Vec<Diagnostic>,
) -> InternSymbolsResult<Rule> {
match rule {
Rule::Choice(elements) => {
Self::check_single(elements, name, true, diagnostics);
let mut result = Vec::with_capacity(elements.len());
for element in elements {
result.push(self.intern_rule(element, name, diagnostics)?);
while let Some(id) = stack.pop() {
match pool.node(id) {
Rule::NamedSymbol(sid) => match name_of_symbol.get(&sid).copied() {
Some(s) => pool.set_node(
id,
Rule::Sym {
kind: s.kind,
index: s.index as u32,
},
),
None => Err(InternSymbolsError::Undefined(pool.resolve(sid).to_string()))?,
},
Rule::Seq(range) | Rule::Choice(range) => {
let children = pool.child_slice(range);
// In the case of a seq or choice rule of 1 element in a hidden rule, weird
// inconsistent behavior with queries can occur. So we should warn the user about it.
if children.len() == 1
&& matches!(pool.node(children[0]), Rule::String(_) | Rule::Pattern(..))
{
let name = var_name.map(|s| pool.resolve(s).to_string());
diagnostics.push(if matches!(pool.node(id), Rule::Choice(_)) {
Diagnostic::UnaryChoice { name }
} else {
Diagnostic::UnarySeq { name }
});
}
Ok(Rule::Choice(result))
let base = stack.len();
stack.extend_from_slice(pool.child_slice(range));
stack[base..].reverse();
}
Rule::Seq(elements) => {
Self::check_single(elements, name, false, diagnostics);
let mut result = Vec::with_capacity(elements.len());
for element in elements {
result.push(self.intern_rule(element, name, diagnostics)?);
}
Ok(Rule::Seq(result))
}
Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(self.intern_rule(
content,
name,
diagnostics,
)?))),
Rule::Metadata { rule, params } => Ok(Rule::Metadata {
rule: Box::new(self.intern_rule(rule, name, diagnostics)?),
params: params.clone(),
}),
Rule::Reserved { rule, context_name } => Ok(Rule::Reserved {
rule: Box::new(self.intern_rule(rule, name, diagnostics)?),
context_name: context_name.clone(),
}),
Rule::NamedSymbol(name) => self.intern_name(name).map_or_else(
|| Err(InternSymbolsError::Undefined(name.clone())),
|symbol| Ok(Rule::Symbol(symbol)),
),
_ => Ok(rule.clone()),
Rule::Repeat(inner)
| Rule::Metadata { rule: inner, .. }
| Rule::Reserved { rule: inner, .. } => stack.push(inner),
_ => {}
}
}
fn intern_name(&self, symbol: &str) -> Option<Symbol> {
for (i, variable) in self.grammar.variables.iter().enumerate() {
if variable.name == symbol {
return Some(Symbol::non_terminal(i));
}
}
for (i, external_token) in self.grammar.external_tokens.iter().enumerate() {
if let Rule::NamedSymbol(name) = external_token
&& name == symbol
{
return Some(Symbol::external(i));
}
}
None
}
// In the case of a seq or choice rule of 1 element in a hidden rule, weird
// inconsistent behavior with queries can occur. So we should warn the user about it.
fn check_single(
elements: &[Rule],
name: Option<&str>,
is_choice: bool,
diagnostics: &mut Vec<Diagnostic>,
) {
if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) {
let name = name.map(str::to_string);
diagnostics.push(if is_choice {
Diagnostic::UnaryChoice { name }
} else {
Diagnostic::UnarySeq { name }
});
}
}
Ok(())
}
fn variable_type_for_name(name: &str) -> VariableType {
@ -223,96 +220,244 @@ fn variable_type_for_name(name: &str) -> VariableType {
#[cfg(test)]
mod tests {
use crate::{grammars::Variable, rules::SymbolType};
use super::*;
#[test]
fn test_basic_repeat_expansion() {
let grammar = intern_symbols(
&build_grammar(vec![
Variable::named("x", Rule::choice(vec![Rule::named("y"), Rule::named("_z")])),
Variable::named("y", Rule::named("_z")),
Variable::named("_z", Rule::string("a")),
]),
&mut Vec::new(),
)
.unwrap();
fn test_basic_interning() {
let mut grammar = {
// x: choice(y, _z)
// y: _z
// _z: "a"
let mut pool = RulePool::default();
let (y_str, z_str) = (pool.intern("y"), pool.intern("_z"));
let y = pool.named_symbol(y_str);
let z = pool.named_symbol(z_str);
let a = pool.intern("a");
let x_root = pool.choice(&[y, z]);
let y_root = pool.named_symbol(z_str);
let z_root = pool.string(a);
let variables = vec![
Variable {
name: pool.intern("x"),
root: x_root,
},
Variable {
name: pool.intern("y"),
root: y_root,
},
Variable {
name: pool.intern("_z"),
root: z_root,
},
];
pool_grammar(pool, variables)
};
let meta = intern_symbols(&mut grammar, &mut Vec::new()).unwrap();
// x's body was `choice(y, _z)` -> `choice(nt1, nt2)`
let x_root = grammar.pool.node(grammar.variables[0].root);
let Rule::Choice(range) = x_root else {
panic!("Expected choice, got {x_root:#?}");
};
let x_children = grammar
.pool
.child_slice(range)
.iter()
.map(|&c| grammar.pool.node(c))
.collect::<Vec<_>>();
assert_eq!(x_children, vec![nt(1), nt(2)]);
// y's body was _z -> nt2
assert_eq!(grammar.pool.node(grammar.variables[1].root), nt(2));
// z's body (a string) is untouched, it remains hidden
assert!(matches!(
grammar.pool.node(grammar.variables[2].root),
Rule::String(_)
));
assert_eq!(
grammar.variables,
meta.kinds,
vec![
Variable::named(
"x",
Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),])
),
Variable::named("y", Rule::non_terminal(2)),
Variable::hidden("_z", Rule::string("a")),
VariableType::Named,
VariableType::Named,
VariableType::Hidden
]
);
}
#[test]
fn test_interning_external_token_names() {
// Variable `y` is both an internal and an external token.
// Variable `z` is just an external token.
let mut input_grammar = build_grammar(vec![
Variable::named(
"w",
Rule::choice(vec![Rule::named("x"), Rule::named("y"), Rule::named("z")]),
),
Variable::named("x", Rule::string("a")),
Variable::named("y", Rule::string("b")),
]);
input_grammar
// w: choice(x, y, z)
// x: "a"
// y: "b"
// externals: [y, z]
let mut grammar = {
let mut pool = RulePool::default();
let (x_str, y_str, z_str) = (pool.intern("x"), pool.intern("y"), pool.intern("z"));
let x_ns = pool.named_symbol(x_str);
let y_ns = pool.named_symbol(y_str);
let z_ns = pool.named_symbol(z_str);
let w_root = pool.choice(&[x_ns, y_ns, z_ns]);
let (a, b) = (pool.intern("a"), pool.intern("b"));
let x_root = pool.string(a);
let y_root = pool.string(b);
let variables = vec![
Variable {
name: pool.intern("w"),
root: w_root,
},
Variable {
name: pool.intern("x"),
root: x_root,
},
Variable {
name: pool.intern("y"),
root: y_root,
},
];
let mut grammar = pool_grammar(pool, variables);
let ext_y = grammar.pool.named_symbol(y_str);
let ext_z = grammar.pool.named_symbol(z_str);
grammar.external_roots = vec![ext_y, ext_z];
grammar
};
let meta = intern_symbols(&mut grammar, &mut Vec::new()).unwrap();
// w: x -> nt1, y -> nt2 (var shadows the external), z -> ext1
let w_root = grammar.pool.node(grammar.variables[0].root);
let Rule::Choice(range) = w_root else {
panic!("Expected choice, got {w_root:#?}");
};
let w_children = grammar
.pool
.child_slice(range)
.iter()
.map(|&c| grammar.pool.node(c))
.collect::<Vec<_>>();
assert_eq!(w_children, vec![nt(1), nt(2), ext(1)]);
// x and y bodies (bare strings) are left alone
assert!(matches!(
grammar.pool.node(grammar.variables[1].root),
Rule::String(_)
));
assert!(matches!(
grammar.pool.node(grammar.variables[2].root),
Rule::String(_)
));
// The external roots resolve the same way: y -> nt2 (shadow), z -> ext1
assert_eq!(grammar.pool.node(grammar.external_roots[0]), nt(2));
assert_eq!(grammar.pool.node(grammar.external_roots[1]), ext(1));
// external token metadata carries the correct names and kinds
let externals = meta
.external_tokens
.extend(vec![Rule::named("y"), Rule::named("z")]);
let grammar = intern_symbols(&input_grammar, &mut Vec::new()).unwrap();
// Variable `y` is referred to by its internal index.
// Variable `z` is referred to by its external index.
.iter()
.map(|&(sid, kind)| (sid.map(|s| grammar.pool.resolve(s)), kind))
.collect::<Vec<_>>();
assert_eq!(
grammar.variables,
externals,
vec![
Variable::named(
"w",
Rule::choice(vec![
Rule::non_terminal(1),
Rule::non_terminal(2),
Rule::external(1),
])
),
Variable::named("x", Rule::string("a")),
Variable::named("y", Rule::string("b")),
]
);
// The external token for `y` refers back to its internal index.
assert_eq!(
grammar.external_tokens,
vec![
Variable::named("y", Rule::non_terminal(2)),
Variable::named("z", Rule::external(1)),
(Some("y"), VariableType::Named),
(Some("z"), VariableType::Named)
]
);
}
#[test]
fn test_grammar_with_undefined_symbols() {
let result = intern_symbols(
&build_grammar(vec![Variable::named("x", Rule::named("y"))]),
&mut Vec::new(),
);
// x references y, which is undefined
let mut pool = RulePool::default();
let y_str = pool.intern("y");
let x_root = pool.named_symbol(y_str);
let variables = vec![Variable {
name: pool.intern("x"),
root: x_root,
}];
let mut grammar = pool_grammar(pool, variables);
let result = intern_symbols(&mut grammar, &mut Vec::new());
assert!(result.is_err(), "Expected an error but got none");
let e = result.err().unwrap();
assert_eq!(e.to_string(), "Undefined symbol `y`");
assert_eq!(
result.unwrap_err(),
InternSymbolsError::Undefined("y".to_string())
);
}
fn build_grammar(variables: Vec<Variable>) -> InputGrammar {
#[test]
fn test_meta_paths() {
// x: "a"
// y: "b"
// supertypes: [y]
// word: y
// conflicts: [x, y]
// inline: [x, nonexistent]
let mut grammar = {
let mut pool = RulePool::default();
let (a, b) = (pool.intern("a"), pool.intern("b"));
let (x, y) = (pool.intern("x"), pool.intern("y"));
let x_root = pool.string(a);
let y_root = pool.string(b);
let variables = vec![
Variable {
name: x,
root: x_root,
},
Variable {
name: y,
root: y_root,
},
];
let mut grammar = pool_grammar(pool, variables);
let nonexistent = grammar.pool.intern("nonexistent");
grammar.supertype_names = vec![y];
grammar.word_name = Some(y);
grammar.conflict_names = vec![vec![x, y]];
grammar.inline_names = vec![x, nonexistent];
grammar
};
let meta = intern_symbols(&mut grammar, &mut Vec::new()).unwrap();
// y is a supertype, so it's reclassified as hidden
assert_eq!(meta.kinds, vec![VariableType::Named, VariableType::Hidden]);
assert_eq!(meta.supertypes, vec![Symbol::non_terminal(1)]);
assert_eq!(meta.word, Some(Symbol::non_terminal(1)));
assert_eq!(
meta.conflicts,
vec![vec![Symbol::non_terminal(0), Symbol::non_terminal(1)]]
);
// unknown inline names are silently skipped
assert_eq!(meta.inline, vec![Symbol::non_terminal(0)]);
}
fn pool_grammar(pool: RulePool, variables: Vec<Variable>) -> InputGrammar {
InputGrammar {
pool,
variables,
name: "the_language".to_string(),
..Default::default()
}
}
fn nt(index: u32) -> Rule {
Rule::Sym {
kind: SymbolType::NonTerminal,
index,
}
}
fn ext(index: u32) -> Rule {
Rule::Sym {
kind: SymbolType::External,
index,
}
}
}

892
crates/generate/src/prepare_grammar/process_inlines.rs
View file

File diff suppressed because it is too large Load diff

188
crates/generate/src/render.rs
View file

@ -7,16 +7,18 @@ use std::{
use rustc_hash::{FxHashMap, FxHashSet};
use crate::LANGUAGE_VERSION;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use super::{
LANGUAGE_VERSION,
build_tables::Tables,
grammars::{ExternalToken, LexicalGrammar, SyntaxGrammar, VariableType},
grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
nfa::CharacterSet,
node_types::ChildType,
rules::{Alias, AliasMap, Symbol, SymbolType, TokenSet},
rules::Alias,
rules::{AliasMap, Symbol, SymbolType, TokenSet},
strpool::{StrId, StrPool},
tables::{
AdvanceAction, FieldLocation, GotoAction, LexState, LexTable, ParseAction, ParseTable,
ParseTableEntry,
@ -98,11 +100,12 @@ struct Generator {
symbol_map: FxHashMap<Symbol, Symbol>,
reserved_word_sets: Vec<TokenSet>,
reserved_word_set_ids_by_parse_state: Vec<usize>,
field_names: Vec<String>,
field_names: Vec<StrId>,
supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
supertype_map: BTreeMap<String, Vec<ChildType>>,
abi_version: usize,
metadata: Option<Metadata>,
str_pool: StrPool,
}
struct LargeCharacterSetInfo {
@ -215,7 +218,7 @@ impl Generator {
if other_symbol < mapping && other_alias == alias {
mapping = other_symbol;
}
} else if self.metadata_for_symbol(*other_symbol) == (&alias.value, kind) {
} else if self.metadata_for_symbol(*other_symbol) == (alias.value, kind) {
mapping = other_symbol;
break;
}
@ -246,37 +249,43 @@ impl Generator {
for production_info in &self.parse_table.production_infos {
// Build a list of all field names
for field_name in production_info.field_map.keys() {
if let Err(i) = self.field_names.binary_search(field_name) {
self.field_names.insert(i, field_name.clone());
for &field_name in production_info.field_map.keys() {
if let Err(i) = self.field_names.binary_search_by(|&sid| {
self.str_pool
.resolve(sid)
.cmp(self.str_pool.resolve(field_name))
}) {
self.field_names.insert(i, field_name);
}
}
for alias in &production_info.alias_sequence {
// Generate a mapping from aliases to C identifiers.
if let Some(alias) = &alias {
// Some aliases match an existing symbol in the grammar.
let alias_id = if let Some(existing_symbol) =
self.symbols_for_alias(alias).first()
{
self.symbol_ids[&self.symbol_map[existing_symbol]].clone()
}
// Other aliases don't match any existing symbol, and need their own
// identifiers.
else {
if let Err(i) = self.unique_aliases.binary_search(alias) {
self.unique_aliases.insert(i, alias.clone());
}
if alias.is_named {
format!("alias_sym_{}", Self::sanitize_identifier(&alias.value))
} else {
format!("anon_alias_sym_{}", Self::sanitize_identifier(&alias.value))
}
};
self.alias_ids.entry(alias.clone()).or_insert(alias_id);
// Generate a mapping from aliases to C identifiers.
for &alias in production_info.alias_sequence.iter().flatten() {
// Some aliases match an existing symbol in the grammar.
let alias_id = if let Some(existing_symbol) = self.symbols_for_alias(alias).first()
{
self.symbol_ids[&self.symbol_map[existing_symbol]].clone()
}
// Other aliases don't match any existing symbol, and need their own
// identifiers.
else {
if let Err(i) = self.unique_aliases.binary_search_by(|candidate| {
self.str_pool
.resolve(candidate.value)
.cmp(self.str_pool.resolve(alias.value))
.then_with(|| candidate.is_named.cmp(&alias.is_named))
}) {
self.unique_aliases.insert(i, alias);
}
if alias.is_named {
format!("alias_sym_{}", self.sanitize_identifier(alias.value))
} else {
format!("anon_alias_sym_{}", self.sanitize_identifier(alias.value))
}
};
self.alias_ids.entry(alias).or_insert(alias_id);
}
}
@ -461,10 +470,11 @@ impl Generator {
add_line!(self, "static const char * const ts_symbol_names[] = {{");
indent!(self);
for symbol in &self.parse_table.symbols {
let name = Self::sanitize_string(self.default_aliases.get(symbol).map_or_else(
|| self.metadata_for_symbol(*symbol).0,
|alias| alias.value.as_str(),
));
let name = self.sanitize_string(
self.default_aliases
.get(symbol)
.map_or_else(|| self.metadata_for_symbol(*symbol).0, |alias| alias.value),
);
add_line!(self, "[{}] = \"{name}\",", self.symbol_ids[symbol]);
}
for alias in &self.unique_aliases {
@ -472,7 +482,7 @@ impl Generator {
self,
"[{}] = \"{}\",",
self.alias_ids[alias],
Self::sanitize_string(&alias.value)
self.sanitize_string(alias.value)
);
}
dedent!(self);
@ -509,8 +519,13 @@ impl Generator {
fn add_field_name_enum(&mut self) {
add_line!(self, "enum ts_field_identifiers {{");
indent!(self);
for (i, field_name) in self.field_names.iter().enumerate() {
add_line!(self, "{} = {},", Self::field_id(field_name), i + 1);
for (i, &field_name) in self.field_names.iter().enumerate() {
add_line!(
self,
"{} = {},",
Self::field_id(self.str_pool.resolve(field_name)),
i + 1
);
}
dedent!(self);
add_line!(self, "}};");
@ -521,7 +536,8 @@ impl Generator {
add_line!(self, "static const char * const ts_field_names[] = {{");
indent!(self);
add_line!(self, "[0] = NULL,");
for field_name in &self.field_names {
for &field_name in &self.field_names {
let field_name = self.str_pool.resolve(field_name);
add_line!(self, "[{}] = \"{field_name}\",", Self::field_id(field_name));
}
dedent!(self);
@ -613,17 +629,18 @@ impl Generator {
fn add_non_terminal_alias_map(&mut self) {
let mut alias_ids_by_symbol = FxHashMap::default();
for variable in &self.syntax_grammar.variables {
for production in &variable.productions {
for step in &production.steps {
if let Some(alias) = &step.alias
&& step.symbol.is_non_terminal()
&& Some(alias) != self.default_aliases.get(&step.symbol)
&& self.symbol_ids.contains_key(&step.symbol)
&& let Some(alias_id) = self.alias_ids.get(alias)
for i in 0..self.syntax_grammar.variables.len() {
for prod_id in self.syntax_grammar.variable_prod_ids(i) {
for step in self.syntax_grammar.production(prod_id).steps {
if let Some(alias) = step.alias()
&& step.symbol().is_non_terminal()
&& Some(alias) != self.default_aliases.get(&step.symbol()).copied()
&& self.symbol_ids.contains_key(&step.symbol())
&& let Some(alias_id) = self.alias_ids.get(&alias)
{
let alias_ids =
alias_ids_by_symbol.entry(step.symbol).or_insert(Vec::new());
let alias_ids = alias_ids_by_symbol
.entry(step.symbol())
.or_insert(Vec::new());
if let Err(i) = alias_ids.binary_search(&alias_id) {
alias_ids.insert(i, alias_id);
}
@ -697,9 +714,12 @@ impl Generator {
let mut flat_field_map = Vec::with_capacity(production_info.field_map.len());
for (field_name, locations) in &production_info.field_map {
for location in locations {
flat_field_map.push((field_name.clone(), *location));
flat_field_map.push((*field_name, *location));
}
}
flat_field_map.sort_by(|(a, _), (b, _)| {
self.str_pool.resolve(*a).cmp(self.str_pool.resolve(*b))
});
let field_map_len = flat_field_map.len();
field_map_ids.push((
Self::get_field_map_id(
@ -742,7 +762,7 @@ impl Generator {
add!(
self,
"{{{}, {}",
Self::field_id(&field_name),
Self::field_id(self.str_pool.resolve(field_name)),
location.index
);
if location.inherited {
@ -788,7 +808,7 @@ impl Generator {
ChildType::Aliased(alias) => {
self.alias_ids.get(alias).cloned().map_or_else(
|| {
self.symbols_for_alias(alias)
self.symbols_for_alias(*alias)
.into_iter()
.map(|s| self.symbol_ids.get(&s).cloned())
.collect()
@ -1227,11 +1247,7 @@ impl Generator {
add_line!(self, "enum ts_external_scanner_symbol_identifiers {{");
indent!(self);
for i in 0..self.syntax_grammar.external_tokens.len() {
add_line!(
self,
"{} = {i},",
Self::external_token_id(&self.syntax_grammar.external_tokens[i]),
);
add_line!(self, "{} = {i},", self.external_token_id(i));
}
dedent!(self);
add_line!(self, "}};");
@ -1252,7 +1268,7 @@ impl Generator {
add_line!(
self,
"[{}] = {},",
Self::external_token_id(token),
self.external_token_id(i),
self.symbol_ids[&id_token],
);
}
@ -1273,11 +1289,7 @@ impl Generator {
add_line!(self, "[{i}] = {{");
indent!(self);
for token in self.parse_table.external_lex_states[i].iter() {
add_line!(
self,
"[{}] = true,",
Self::external_token_id(&self.syntax_grammar.external_tokens[token.index])
);
add_line!(self, "[{}] = true,", self.external_token_id(token.index));
}
dedent!(self);
add_line!(self, "}},");
@ -1697,8 +1709,8 @@ impl Generator {
}
fn get_field_map_id(
flat_field_map: Vec<(String, FieldLocation)>,
flat_field_maps: &mut Vec<(usize, Vec<(String, FieldLocation)>)>,
flat_field_map: Vec<(StrId, FieldLocation)>,
flat_field_maps: &mut Vec<(usize, Vec<(StrId, FieldLocation)>)>,
next_flat_field_map_index: &mut usize,
) -> usize {
if let Some((index, _)) = flat_field_maps.iter().find(|(_, e)| *e == *flat_field_map) {
@ -1711,11 +1723,9 @@ impl Generator {
result
}
fn external_token_id(token: &ExternalToken) -> String {
format!(
"ts_external_token_{}",
Self::sanitize_identifier(&token.name)
)
fn external_token_id(&self, token_idx: usize) -> String {
let token = &self.syntax_grammar.external_tokens[token_idx];
format!("ts_external_token_{}", self.sanitize_identifier(token.name))
}
fn assign_symbol_id(&mut self, symbol: Symbol, used_identifiers: &mut FxHashSet<String>) {
@ -1725,10 +1735,10 @@ impl Generator {
} else {
let (name, kind) = self.metadata_for_symbol(symbol);
id = match kind {
VariableType::Auxiliary => format!("aux_sym_{}", Self::sanitize_identifier(name)),
VariableType::Anonymous => format!("anon_sym_{}", Self::sanitize_identifier(name)),
VariableType::Auxiliary => format!("aux_sym_{}", self.sanitize_identifier(name)),
VariableType::Anonymous => format!("anon_sym_{}", self.sanitize_identifier(name)),
VariableType::Hidden | VariableType::Named => {
format!("sym_{}", Self::sanitize_identifier(name))
format!("sym_{}", self.sanitize_identifier(name))
}
};
@ -1750,25 +1760,27 @@ impl Generator {
format!("field_{field_name}")
}
fn metadata_for_symbol(&self, symbol: Symbol) -> (&str, VariableType) {
fn metadata_for_symbol(&self, symbol: Symbol) -> (StrId, VariableType) {
match symbol.kind {
SymbolType::End | SymbolType::EndOfNonTerminalExtra => ("end", VariableType::Hidden),
SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
(StrPool::END_NAME_ID, VariableType::Hidden)
}
SymbolType::NonTerminal => {
let variable = &self.syntax_grammar.variables[symbol.index];
(&variable.name, variable.kind)
(variable.name, variable.kind)
}
SymbolType::Terminal => {
let variable = &self.lexical_grammar.variables[symbol.index];
(&variable.name, variable.kind)
(variable.name, variable.kind)
}
SymbolType::External => {
let token = &self.syntax_grammar.external_tokens[symbol.index];
(&token.name, token.kind)
(token.name, token.kind)
}
}
}
fn symbols_for_alias(&self, alias: &Alias) -> Vec<Symbol> {
fn symbols_for_alias(&self, alias: Alias) -> Vec<Symbol> {
self.parse_table
.symbols
.iter()
@ -1779,13 +1791,14 @@ impl Generator {
let (name, kind) = self.metadata_for_symbol(*symbol);
name == alias.value && kind == alias.kind()
},
|default_alias| default_alias == alias,
|&default_alias| default_alias == alias,
)
})
.collect()
}
fn sanitize_identifier(name: &str) -> String {
fn sanitize_identifier(&self, name: StrId) -> String {
let name = self.str_pool.resolve(name);
let mut result = String::with_capacity(name.len());
for c in name.chars() {
if c.is_ascii_alphanumeric() || c == '_' {
@ -1880,7 +1893,8 @@ impl Generator {
result
}
fn sanitize_string(name: &str) -> String {
fn sanitize_string(&self, name: StrId) -> String {
let name = self.str_pool.resolve(name);
let mut result = String::with_capacity(name.len());
for c in name.chars() {
match c {
@ -1939,6 +1953,8 @@ impl Generator {
/// * `default_aliases` - A map describing the global rename rules that should apply. the keys are
/// symbols that are *always* aliased in the same way, and the values are the aliases that are
/// applied to those symbols.
/// * `str_pool` - A backing pool for `StrId`-identified strings within `syntax_grammar`,
/// `lexical_grammar`, and `default_aliases`.
/// * `abi_version` - The language ABI version that should be generated. Usually you want
/// Tree-sitter's current version, but right after making an ABI change, it may be useful to
/// generate code with the previous ABI.
@ -1947,11 +1963,12 @@ impl Generator {
reason = "all parameters are required for code generation"
)]
pub fn render_c_code(
name: &str,
name: StrId,
tables: Tables,
syntax_grammar: SyntaxGrammar,
lexical_grammar: LexicalGrammar,
default_aliases: AliasMap,
str_pool: StrPool,
abi_version: usize,
semantic_version: Option<(u8, u8, u8)>,
supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
@ -1961,7 +1978,7 @@ pub fn render_c_code(
}
Generator {
language_name: name.to_string(),
language_name: str_pool.resolve(name).to_string(),
parse_table: tables.parse_table,
main_lex_table: tables.main_lex_table,
keyword_lex_table: tables.keyword_lex_table,
@ -1977,6 +1994,7 @@ pub fn render_c_code(
patch,
}),
supertype_symbol_map,
str_pool,
..Default::default()
}
.generate()

732
crates/generate/src/rules.rs
View file

@ -1,9 +1,15 @@
use std::{collections::BTreeMap, fmt, hash::Hash};
//! Flat, pool-indexed rule IR for the grammar backend.
//!
//! Nodes live in one append-only arena and reference children by index, so passes
//! are iterative pool walks with in-place rewrites.
use serde::{Deserialize, Serialize};
use super::bitvec::{BitVec, SetBitsIter};
use super::grammars::VariableType;
use crate::{
bitvec::{BitVec, SetBitsIter},
grammars::VariableType,
strpool::{StrId, StrPool},
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
pub enum SymbolType {
@ -20,191 +26,15 @@ pub enum Associativity {
Right,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
pub struct Alias {
pub value: String,
pub value: StrId,
pub is_named: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default, Serialize, Deserialize)]
pub enum Precedence {
#[default]
None,
Integer(i32),
Name(String),
}
pub type AliasMap = BTreeMap<Symbol, Alias>;
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct MetadataParams {
pub precedence: Precedence,
pub dynamic_precedence: i32,
pub associativity: Option<Associativity>,
pub is_token: bool,
pub is_main_token: bool,
pub alias: Option<Alias>,
pub field_name: Option<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
pub struct Symbol {
pub kind: SymbolType,
pub index: usize,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum Rule {
Blank,
String(String),
Pattern(String, String),
NamedSymbol(String),
Symbol(Symbol),
Choice(Vec<Self>),
Metadata {
params: MetadataParams,
rule: Box<Self>,
},
Repeat(Box<Self>),
Seq(Vec<Self>),
Reserved {
rule: Box<Self>,
context_name: String,
},
}
// Because tokens are represented as small (~400 max) unsigned integers,
// sets of tokens can be efficiently represented as bit vectors with each
// index corresponding to a token, and each value representing whether or not
// the token is present in the set.
#[derive(Default, Clone, PartialEq, Eq, Hash)]
pub struct TokenSet {
terminal_bits: BitVec,
external_bits: BitVec,
eof: bool,
end_of_nonterminal_extra: bool,
}
impl fmt::Debug for TokenSet {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_list().entries(self.iter()).finish()
}
}
impl PartialOrd for TokenSet {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for TokenSet {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.terminal_bits
.cmp(&other.terminal_bits)
.then_with(|| self.external_bits.cmp(&other.external_bits))
.then_with(|| self.eof.cmp(&other.eof))
.then_with(|| {
self.end_of_nonterminal_extra
.cmp(&other.end_of_nonterminal_extra)
})
}
}
impl Rule {
#[must_use]
pub fn field(name: String, content: Self) -> Self {
add_metadata(content, move |params| {
params.field_name = Some(name);
})
}
#[must_use]
pub fn alias(content: Self, value: String, is_named: bool) -> Self {
add_metadata(content, move |params| {
params.alias = Some(Alias { value, is_named });
})
}
#[must_use]
pub fn token(content: Self) -> Self {
add_metadata(content, |params| {
params.is_token = true;
})
}
#[must_use]
pub fn immediate_token(content: Self) -> Self {
add_metadata(content, |params| {
params.is_token = true;
params.is_main_token = true;
})
}
#[must_use]
pub fn prec(value: Precedence, content: Self) -> Self {
add_metadata(content, |params| {
params.precedence = value;
})
}
#[must_use]
pub fn prec_left(value: Precedence, content: Self) -> Self {
add_metadata(content, |params| {
params.associativity = Some(Associativity::Left);
params.precedence = value;
})
}
#[must_use]
pub fn prec_right(value: Precedence, content: Self) -> Self {
add_metadata(content, |params| {
params.associativity = Some(Associativity::Right);
params.precedence = value;
})
}
#[must_use]
pub fn prec_dynamic(value: i32, content: Self) -> Self {
add_metadata(content, |params| {
params.dynamic_precedence = value;
})
}
#[must_use]
pub fn repeat(rule: Self) -> Self {
Self::Repeat(Box::new(rule))
}
#[must_use]
pub fn choice(rules: Vec<Self>) -> Self {
let mut elements = Vec::with_capacity(rules.len());
for rule in rules {
choice_helper(&mut elements, rule);
}
Self::Choice(elements)
}
#[must_use]
pub const fn seq(rules: Vec<Self>) -> Self {
Self::Seq(rules)
}
pub fn is_empty(&self) -> bool {
match self {
Self::Blank | Self::Pattern(..) | Self::NamedSymbol(_) | Self::Symbol(_) => false,
Self::String(string) => string.is_empty(),
Self::Metadata { rule, .. } | Self::Repeat(rule) | Self::Reserved { rule, .. } => {
rule.is_empty()
}
Self::Choice(rules) => rules.iter().any(Self::is_empty),
Self::Seq(rules) => rules.iter().all(Self::is_empty),
}
}
}
impl Alias {
#[must_use]
pub const fn kind(&self) -> VariableType {
pub const fn kind(self) -> VariableType {
if self.is_named {
VariableType::Named
} else {
@ -213,46 +43,30 @@ impl Alias {
}
}
impl Precedence {
#[must_use]
pub const fn is_none(&self) -> bool {
matches!(self, Self::None)
}
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)]
pub enum Precedence {
#[default]
None,
Integer(i32),
Name(StrId),
}
#[cfg(test)]
impl Rule {
#[must_use]
pub const fn terminal(index: usize) -> Self {
Self::Symbol(Symbol::terminal(index))
}
#[must_use]
pub const fn non_terminal(index: usize) -> Self {
Self::Symbol(Symbol::non_terminal(index))
}
#[must_use]
pub const fn external(index: usize) -> Self {
Self::Symbol(Symbol::external(index))
}
#[must_use]
pub fn named(name: &'static str) -> Self {
Self::NamedSymbol(name.to_string())
}
#[must_use]
pub fn string(value: &'static str) -> Self {
Self::String(value.to_string())
}
#[must_use]
pub fn pattern(value: &'static str, flags: &'static str) -> Self {
Self::Pattern(value.to_string(), flags.to_string())
}
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)]
pub struct MetadataParams {
pub precedence: Precedence,
pub associativity: Option<Associativity>,
pub dynamic_precedence: i32,
pub alias: Option<Alias>,
pub field: Option<StrId>,
pub is_token: bool,
pub is_main_token: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
pub struct Symbol {
pub kind: SymbolType,
pub index: usize,
}
impl Symbol {
#[must_use]
pub fn is_terminal(&self) -> bool {
@ -315,9 +129,445 @@ impl Symbol {
}
}
/// A pooled rule node.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Rule {
Blank,
String(StrId),
Pattern(StrId, StrId),
NamedSymbol(StrId),
Sym { kind: SymbolType, index: u32 },
Seq(RuleIdRange),
Choice(RuleIdRange),
Repeat(RuleId),
Metadata { params: ParamsId, rule: RuleId },
Reserved { rule: RuleId, ctx: StrId },
}
const _: () = assert!(std::mem::size_of::<Rule>() <= 12);
impl From<Symbol> for Rule {
fn from(symbol: Symbol) -> Self {
Self::Symbol(symbol)
fn from(value: Symbol) -> Self {
Self::Sym {
kind: value.kind,
index: value.index as u32,
}
}
}
impl Rule {
pub const fn symbol(self) -> Option<Symbol> {
match self {
Self::Sym { kind, index } => Some(Symbol {
kind,
index: index as usize,
}),
_ => None,
}
}
}
/// The arena: nodes, their pooled children/params, and the string interner.
/// Storage is append-only, passes rewrite nodes in place and may orphan subtrees.
#[derive(Clone, Default)]
pub struct RulePool {
nodes: Vec<Rule>,
children: Vec<RuleId>,
params: Vec<MetadataParams>,
str_pool: StrPool,
}
impl RulePool {
#[must_use]
pub fn node(&self, id: RuleId) -> Rule {
self.nodes[id.index()]
}
pub fn set_node(&mut self, id: RuleId, node: Rule) {
self.nodes[id.index()] = node;
}
pub fn push_node(&mut self, node: Rule) -> RuleId {
let id = RuleId(self.nodes.len() as u32);
self.nodes.push(node);
id
}
#[must_use]
pub fn child_slice(&self, range: RuleIdRange) -> &[RuleId] {
&self.children[range.as_range()]
}
pub fn push_children(&mut self, ids: &[RuleId]) -> RuleIdRange {
let start = self.children.len() as u32;
self.children.extend_from_slice(ids);
RuleIdRange {
start,
len: ids.len() as u32,
}
}
#[must_use]
pub fn params(&self, id: ParamsId) -> MetadataParams {
self.params[id.0 as usize]
}
pub fn push_params(&mut self, params: MetadataParams) -> ParamsId {
let id = ParamsId(self.params.len() as u32);
self.params.push(params);
id
}
pub fn set_params(&mut self, id: ParamsId, params: MetadataParams) {
self.params[id.0 as usize] = params;
}
fn metadata_with(&mut self, content: RuleId, f: impl FnOnce(&mut MetadataParams)) -> RuleId {
if let Rule::Metadata { params, .. } = self.node(content) {
let mut p = self.params(params);
if !p.is_token {
f(&mut p);
self.set_params(params, p);
return content;
}
}
let mut p = MetadataParams::default();
f(&mut p);
let params = self.push_params(p);
self.push_node(Rule::Metadata {
params,
rule: content,
})
}
pub fn blank(&mut self) -> RuleId {
self.push_node(Rule::Blank)
}
pub fn string(&mut self, value: StrId) -> RuleId {
self.push_node(Rule::String(value))
}
pub fn pattern(&mut self, value: StrId, flags: StrId) -> RuleId {
self.push_node(Rule::Pattern(value, flags))
}
pub fn named_symbol(&mut self, name: StrId) -> RuleId {
self.push_node(Rule::NamedSymbol(name))
}
pub fn field(&mut self, name: StrId, content: RuleId) -> RuleId {
self.metadata_with(content, |p| p.field = Some(name))
}
pub fn alias(&mut self, content: RuleId, value: StrId, is_named: bool) -> RuleId {
self.metadata_with(content, |p| p.alias = Some(Alias { value, is_named }))
}
pub fn token(&mut self, content: RuleId) -> RuleId {
self.metadata_with(content, |p| p.is_token = true)
}
pub fn immediate_token(&mut self, content: RuleId) -> RuleId {
self.metadata_with(content, |p| {
p.is_token = true;
p.is_main_token = true;
})
}
pub fn prec(&mut self, value: Precedence, content: RuleId) -> RuleId {
self.metadata_with(content, |p| p.precedence = value)
}
pub fn prec_left(&mut self, value: Precedence, content: RuleId) -> RuleId {
self.metadata_with(content, |p| {
p.associativity = Some(Associativity::Left);
p.precedence = value;
})
}
pub fn prec_right(&mut self, value: Precedence, content: RuleId) -> RuleId {
self.metadata_with(content, |p| {
p.associativity = Some(Associativity::Right);
p.precedence = value;
})
}
pub fn prec_dynamic(&mut self, value: i32, content: RuleId) -> RuleId {
self.metadata_with(content, |p| p.dynamic_precedence = value)
}
pub fn repeat(&mut self, content: RuleId) -> RuleId {
self.push_node(Rule::Repeat(content))
}
pub fn seq(&mut self, ids: &[RuleId]) -> RuleId {
let range = self.push_children(ids);
self.push_node(Rule::Seq(range))
}
/// Flatten nested choices and de-dup structurally, keeping a `Choice` node
/// event for a single element
pub fn choice(&mut self, ids: &[RuleId]) -> RuleId {
let mut elements: Vec<RuleId> = Vec::with_capacity(ids.len());
let mut stack: Vec<RuleId> = Vec::with_capacity(ids.len());
stack.extend(ids.iter().rev());
while let Some(id) = stack.pop() {
if let Rule::Choice(range) = self.node(id) {
let base = stack.len();
stack.extend_from_slice(self.child_slice(range));
stack[base..].reverse();
} else if !elements.iter().any(|&e| self.subtree_eq(e, id)) {
elements.push(id);
}
}
let range = self.push_children(&elements);
self.push_node(Rule::Choice(range))
}
pub fn reserved(&mut self, content: RuleId, ctx: StrId) -> RuleId {
self.push_node(Rule::Reserved { rule: content, ctx })
}
#[inline]
#[must_use]
pub fn intern(&mut self, s: &str) -> StrId {
self.str_pool.intern(s)
}
#[inline]
#[must_use]
pub fn resolve(&self, id: StrId) -> &str {
self.str_pool.resolve(id)
}
#[must_use]
pub fn into_interner(self) -> StrPool {
self.str_pool
}
#[must_use]
pub fn subtree_hash(&self, root: RuleId) -> u64 {
use std::hash::{Hash, Hasher};
let mut hasher = rustc_hash::FxHasher::default();
let mut stack = vec![root];
while let Some(id) = stack.pop() {
let node = self.node(id);
std::mem::discriminant(&node).hash(&mut hasher);
match node {
Rule::Blank => {}
Rule::String(s) | Rule::NamedSymbol(s) => s.hash(&mut hasher),
Rule::Pattern(p, f) => {
p.hash(&mut hasher);
f.hash(&mut hasher);
}
Rule::Sym { kind, index } => {
(kind as u8).hash(&mut hasher);
index.hash(&mut hasher);
}
Rule::Seq(range) | Rule::Choice(range) => {
hasher.write_u32(range.len);
let base = stack.len();
stack.extend_from_slice(self.child_slice(range));
stack[base..].reverse();
}
Rule::Repeat(inner) => stack.push(inner),
Rule::Metadata { params, rule } => {
let p = self.params(params);
(p.precedence, p.associativity, p.dynamic_precedence).hash(&mut hasher);
p.alias.map(|a| (a.value, a.is_named)).hash(&mut hasher);
(p.field, p.is_token, p.is_main_token).hash(&mut hasher);
stack.push(rule);
}
Rule::Reserved { rule, ctx } => {
ctx.hash(&mut hasher);
stack.push(rule);
}
}
}
hasher.finish()
}
pub fn subtree_eq(&self, a: RuleId, b: RuleId) -> bool {
let mut stack = vec![(a, b)];
while let Some((a, b)) = stack.pop() {
match (self.node(a), self.node(b)) {
(Rule::Blank, Rule::Blank) => {}
(Rule::String(x), Rule::String(y))
| (Rule::NamedSymbol(x), Rule::NamedSymbol(y)) => {
if x != y {
return false;
}
}
(Rule::Pattern(p1, f1), Rule::Pattern(p2, f2)) => {
if p1 != p2 || f1 != f2 {
return false;
}
}
(n1 @ Rule::Sym { .. }, n2 @ Rule::Sym { .. }) => {
if n1 != n2 {
return false;
}
}
(Rule::Seq(r1), Rule::Seq(r2)) | (Rule::Choice(r1), Rule::Choice(r2)) => {
if r1.len != r2.len {
return false;
}
stack.extend(
self.child_slice(r1)
.iter()
.copied()
.zip(self.child_slice(r2).iter().copied()),
);
}
(Rule::Repeat(i1), Rule::Repeat(i2)) => stack.push((i1, i2)),
#[rustfmt::skip]
(Rule::Metadata { params: p1, rule: r1 }, Rule::Metadata { params: p2, rule: r2 }) => {
if self.params(p1) != self.params(p2) {
return false;
}
stack.push((r1, r2));
}
(Rule::Reserved { rule: r1, ctx: c1 }, Rule::Reserved { rule: r2, ctx: c2 }) => {
if c1 != c2 {
return false;
}
stack.push((r1, r2));
}
_ => return false,
}
}
true
}
/// Whether the subtree at `id` can only match the empty string.
pub fn subtree_matches_empty_str(&self, id: RuleId) -> bool {
match self.node(id) {
Rule::String(sid) => self.resolve(sid).is_empty(),
Rule::Metadata { rule, .. } | Rule::Repeat(rule) | Rule::Reserved { rule, .. } => {
self.subtree_matches_empty_str(rule)
}
Rule::Choice(range) => self
.child_slice(range)
.iter()
.any(|&c| self.subtree_matches_empty_str(c)),
Rule::Seq(range) => self
.child_slice(range)
.iter()
.all(|&c| self.subtree_matches_empty_str(c)),
_ => false,
}
}
/// Check if a rule is referenced by another rule.
///
/// This function is used to determine if a variable is used in a given rule,
/// and `is_external` indicates if the rule is an external, and if it is,
/// to not assume that a named symbol that is equal to itself means it's being referenced.
///
/// For example, if we have an external rule **and** a normal rule both called `foo`,
/// `foo` should not be thought of as directly used unless it's used within another rule.
pub fn rule_is_referenced(&self, id: RuleId, target: StrId, is_external: bool) -> bool {
match self.node(id) {
Rule::NamedSymbol(name) => name == target && !is_external,
Rule::Choice(range) | Rule::Seq(range) => self
.child_slice(range)
.iter()
.any(|&c| self.rule_is_referenced(c, target, false)),
Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
self.rule_is_referenced(rule, target, is_external)
}
Rule::Repeat(inner) => self.rule_is_referenced(inner, target, false),
Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => false,
}
}
/// Append every `NamedSymbol` name reachable in `id` to `out`. If
/// `skip_top_level` is true, a `NamedSymbol` at the root of `id` is
/// ignored (used for externals entries, which name themselves).
pub fn collect_referenced_ids(&self, id: RuleId, skip_top_level: bool, out: &mut Vec<StrId>) {
match self.node(id) {
Rule::NamedSymbol(name) => {
if !skip_top_level {
out.push(name);
}
}
Rule::Choice(range) | Rule::Seq(range) => {
for &c in self.child_slice(range) {
self.collect_referenced_ids(c, false, out);
}
}
Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
self.collect_referenced_ids(rule, skip_top_level, out);
}
Rule::Repeat(inner) => self.collect_referenced_ids(inner, false, out),
Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {}
}
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct RuleId(u32);
impl RuleId {
#[must_use]
pub const fn index(self) -> usize {
self.0 as usize
}
}
/// Index into [`RulePool::params`].
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct ParamsId(u32);
/// A `[start, start + len)` slice of [`RulePool::children`].
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct RuleIdRange {
pub start: u32,
pub len: u32,
}
impl RuleIdRange {
#[must_use]
pub const fn as_range(self) -> std::ops::Range<usize> {
self.start as usize..(self.start + self.len) as usize
}
}
// Because tokens are represented as small (~400 max) unsigned integers,
// sets of tokens can be efficiently represented as bit vectors with each
// index corresponding to a token, and each value representing whether or not
// the token is present in the set.
#[derive(Default, Clone, PartialEq, Eq, Hash)]
pub struct TokenSet {
terminal_bits: BitVec,
external_bits: BitVec,
eof: bool,
end_of_nonterminal_extra: bool,
}
impl std::fmt::Debug for TokenSet {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_list().entries(self.iter()).finish()
}
}
impl PartialOrd for TokenSet {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for TokenSet {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.terminal_bits
.cmp(&other.terminal_bits)
.then_with(|| self.external_bits.cmp(&other.external_bits))
.then_with(|| self.eof.cmp(&other.eof))
.then_with(|| {
self.end_of_nonterminal_extra
.cmp(&other.end_of_nonterminal_extra)
})
}
}
@ -495,44 +745,4 @@ impl FromIterator<Symbol> for TokenSet {
}
}
fn add_metadata<T: FnOnce(&mut MetadataParams)>(input: Rule, f: T) -> Rule {
match input {
Rule::Metadata { rule, mut params } if !params.is_token => {
f(&mut params);
Rule::Metadata { rule, params }
}
_ => {
let mut params = MetadataParams::default();
f(&mut params);
Rule::Metadata {
rule: Box::new(input),
params,
}
}
}
}
fn choice_helper(result: &mut Vec<Rule>, rule: Rule) {
match rule {
Rule::Choice(elements) => {
for element in elements {
choice_helper(result, element);
}
}
_ => {
if !result.contains(&rule) {
result.push(rule);
}
}
}
}
impl fmt::Display for Precedence {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Integer(i) => write!(f, "{i}"),
Self::Name(s) => write!(f, "'{s}'"),
Self::None => write!(f, "none"),
}
}
}
pub type AliasMap = std::collections::BTreeMap<Symbol, Alias>;

74
crates/generate/src/strpool.rs Normal file
View file

@ -0,0 +1,74 @@
use std::{num::NonZeroU32, rc::Rc};
use rustc_hash::FxHashMap;
/// Interned string id, a 1-based index into the pool's string table
#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Hash, Debug, Eq)]
pub struct StrId(NonZeroU32);
impl StrId {
/// Dense 0-based index (ids are 1-based).
#[must_use]
pub const fn index(self) -> usize {
self.0.get() as usize - 1
}
/// The raw 1-based id, for packed encodings where 0 means "none".
#[must_use]
pub const fn raw(self) -> u32 {
self.0.get()
}
/// Inverse of [`Self::raw`]. Caller must pass a value produced by `raw`.
#[must_use]
pub const fn from_raw(raw: u32) -> Self {
Self(NonZeroU32::new(raw).unwrap())
}
}
impl Default for StrId {
fn default() -> Self {
StrPool::EMPTY_STR_ID
}
}
#[derive(Clone, Debug)]
pub struct StrPool {
strs: Vec<Rc<str>>,
str_ids: FxHashMap<Rc<str>, StrId>,
}
impl Default for StrPool {
fn default() -> Self {
let mut pool = Self {
strs: Vec::default(),
str_ids: FxHashMap::default(),
};
let empty_id = pool.intern("");
debug_assert_eq!(empty_id, Self::EMPTY_STR_ID);
let end_id = pool.intern("end");
debug_assert_eq!(end_id, Self::END_NAME_ID);
pool
}
}
impl StrPool {
pub const EMPTY_STR_ID: StrId = StrId::from_raw(1);
pub const END_NAME_ID: StrId = StrId::from_raw(2);
pub fn intern(&mut self, s: &str) -> StrId {
if let Some(&id) = self.str_ids.get(s) {
return id;
}
let owned: Rc<str> = Rc::from(s);
let id = StrId(NonZeroU32::new(self.strs.len() as u32 + 1).unwrap());
self.strs.push(Rc::clone(&owned));
self.str_ids.insert(owned, id);
id
}
#[must_use]
pub fn resolve(&self, id: StrId) -> &str {
&self.strs[id.index()]
}
}

6
crates/generate/src/tables.rs
View file

@ -2,7 +2,9 @@ use std::collections::BTreeMap;
use super::{
nfa::CharacterSet,
rules::{Alias, Symbol, TokenSet},
rules::Alias,
rules::{Symbol, TokenSet},
strpool::StrId,
};
pub type ProductionInfoId = usize;
pub type ParseStateId = usize;
@ -62,7 +64,7 @@ pub struct FieldLocation {
#[derive(Debug, Default, PartialEq, Eq)]
pub struct ProductionInfo {
pub alias_sequence: Vec<Option<Alias>>,
pub field_map: BTreeMap<String, Vec<FieldLocation>>,
pub field_map: BTreeMap<StrId, Vec<FieldLocation>>,
}
#[derive(Debug, Default, PartialEq, Eq)]

2
test/fixtures/test_grammars/indirect_recursion_in_transitions/expected_error.txt vendored
View file

@ -1 +1 @@
Grammar contains an indirectly recursive rule: type_expression -> _expression -> identifier_expression -> type_expression
Grammar contains an indirectly recursive rule: type_expression -> _expression -> type_expression