libs-code/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:26:23 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

perf(generate): intern lookahead TokenSets

Browse source 
`ParseItemSetEntry` stores a `LookaheadSetId` into a `LookaheadSetPool` instead
of an owned `TokenSet`. Ids are canonical, so entry hash/eq/clone are integer
ops and state dedup stops walking set words. Unions and single-token inserts
are memoized by id, so the transitive closure and successor-kernel construction
stop re-materializing the same unions per state. Closure additions carry interned
ids with word-token membership precomputed.

For cpp, 3566 distinct lookahead sets back all 49,992 states. For ruby 2000 sets
for 91,991 states, and rust 646 for 16,796.

Reduces wall time by 10-15%, peak rss by 3-5%.
This commit is contained in:
Will Lillis 2026-08-09 11:51:45 -05:00
parent 6659ff57eb
commit d6ae19cc6d
4 changed files with 185 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
crates/generate/src/build_tables.rs
View file

@ -547,7 +547,13 @@ fn report_state_info<'a>(
);
info!(
"\nitems:\n{}",
item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar, str_pool),
item::ParseItemSetDisplay(
item_set,
syntax_grammar,
lexical_grammar,
str_pool,
&parse_state_info.lookaheads
),
);
}
}

36
crates/generate/src/build_tables/build_parse_table.rs
View file

@ -15,7 +15,7 @@ use super::{
};
use crate::{
Diagnostic,
build_tables::item::prec_display,
build_tables::item::{LookaheadSetPool, prec_display},
grammars::{LexicalGrammar, PrecedenceEntry, ReservedWordSetId, SyntaxGrammar, VariableType},
node_types::VariableInfo,
rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
@ -35,6 +35,7 @@ type AuxiliarySymbolSequence = Vec<AuxiliarySymbolInfo>;
pub struct ParseStateInfo<'a> {
pub preceding_symbols_by_id: Vec<SymbolSequence>,
item_sets_by_ids: IndexMap<ParseItemSet<'a>, ParseStateId, BuildHasherDefault<FxHasher>>,
pub lookaheads: LookaheadSetPool,
}
impl<'a> ParseStateInfo<'a> {
@ -295,13 +296,14 @@ impl<'a> ParseTableBuilder<'a> {
self.add_parse_state(&Vec::new(), &Vec::new(), ParseItemSet::default());
// Add the starting state at index 1.
let end_lookaheads = self.item_set_builder.lookaheads.singleton(Symbol::end());
self.add_parse_state(
&Vec::new(),
&Vec::new(),
ParseItemSet {
entries: vec![ParseItemSetEntry {
item: ParseItem::start(self.item_set_builder.key_map),
lookaheads: std::iter::once(Symbol::end()).collect(),
lookaheads: end_lookaheads,
following_reserved_word_set: ReservedWordSetId::default(),
}],
},
@ -320,7 +322,7 @@ impl<'a> ParseTableBuilder<'a> {
.variable_prod_ids(extra_non_terminal.index as usize)
{
let production = self.syntax_grammar.production(prod_id);
non_terminal_extra_item_sets_by_first_terminal
let entry = non_terminal_extra_item_sets_by_first_terminal
.entry(production.first_symbol().unwrap())
.or_insert_with(ParseItemSet::default)
.insert(ParseItem {
@ -329,9 +331,11 @@ impl<'a> ParseTableBuilder<'a> {
step_index: 1,
keys: self.item_set_builder.key_map.keys_for(prod_id),
has_preceding_inherited_fields: false,
})
});
entry.lookaheads = self
.item_set_builder
.lookaheads
.insert(Symbol::end_of_nonterminal_extra());
.insert(entry.lookaheads, Symbol::end_of_nonterminal_extra());
}
}
@ -389,6 +393,7 @@ impl<'a> ParseTableBuilder<'a> {
ParseStateInfo {
preceding_symbols_by_id: self.preceding_symbols_by_id,
item_sets_by_ids: self.state_ids_by_item_set,
lookaheads: self.item_set_builder.lookaheads,
},
))
}
@ -495,7 +500,10 @@ impl<'a> ParseTableBuilder<'a> {
.or_insert_with(ParseItemSet::default)
};
let successor_entry = successor_set.insert(successor);
successor_entry.lookaheads.insert_all(lookaheads);
successor_entry.lookaheads = self
.item_set_builder
.lookaheads
.union(successor_entry.lookaheads, *lookaheads);
successor_entry.following_reserved_word_set = successor_entry
.following_reserved_word_set
.max(*reserved_lookaheads);
@ -528,7 +536,7 @@ impl<'a> ParseTableBuilder<'a> {
let precedence = item.precedence(self.syntax_grammar);
let associativity = item.associativity(self.syntax_grammar);
for lookahead in lookaheads.iter() {
for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() {
let table_entry = self.parse_table.states[state_id]
.terminal_entries
.entry(lookahead)
@ -716,7 +724,12 @@ impl<'a> ParseTableBuilder<'a> {
} else {
None
}
} else if entry.lookaheads.contains(keyword_capture_token) {
} else if self
.item_set_builder
.lookaheads
.get(entry.lookaheads)
.contains(keyword_capture_token)
{
Some(entry.following_reserved_word_set)
} else {
None
@ -778,7 +791,12 @@ impl<'a> ParseTableBuilder<'a> {
shift_precedence.insert(i, p);
}
}
} else if lookaheads.contains(conflicting_lookahead) && item.variable_index != u32::MAX
} else if self
.item_set_builder
.lookaheads
.get(*lookaheads)
.contains(conflicting_lookahead)
&& item.variable_index != u32::MAX
{
conflicting_items.insert(item);
}

99
crates/generate/src/build_tables/item.rs
View file

@ -1,10 +1,11 @@
use std::{
cmp::Ordering,
fmt,
hash::{Hash, Hasher},
hash::{BuildHasherDefault, Hash, Hasher},
};
use rustc_hash::FxHashMap;
use indexmap::IndexSet;
use rustc_hash::{FxHashMap, FxHasher};
use crate::{
grammars::{LexicalGrammar, ProdRef, ProductionStep, ReservedWordSetId, SyntaxGrammar},
@ -262,6 +263,93 @@ pub struct ParseItem<'a> {
pub has_preceding_inherited_fields: bool,
}
/// Interned lookahead set. An index into a [`LookaheadPool`]. Ids are canonical,
/// so id equality is equivalent to underlying [`TokenSet`] equality.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct LookaheadSetId(u32);
impl Default for LookaheadSetId {
fn default() -> Self {
LookaheadSetPool::EMPTY
}
}
/// Interner for lookahead [`TokenSet`]s. Item-set entries store ids.
pub struct LookaheadSetPool {
sets: IndexSet<TokenSet, BuildHasherDefault<FxHasher>>,
union_memo: FxHashMap<(LookaheadSetId, LookaheadSetId), LookaheadSetId>,
insert_memo: FxHashMap<(LookaheadSetId, Symbol), LookaheadSetId>,
}
impl LookaheadSetPool {
pub const EMPTY: LookaheadSetId = LookaheadSetId(0);
#[must_use]
pub fn new() -> Self {
let mut pool = Self {
sets: IndexSet::default(),
union_memo: FxHashMap::default(),
insert_memo: FxHashMap::default(),
};
let empty = pool.intern(TokenSet::new());
debug_assert_eq!(empty, Self::EMPTY);
pool
}
#[must_use]
pub fn get(&self, id: LookaheadSetId) -> &TokenSet {
&self.sets[id.0 as usize]
}
pub fn intern(&mut self, set: TokenSet) -> LookaheadSetId {
let (index, _) = self.sets.insert_full(set);
LookaheadSetId(index as u32)
}
pub fn intern_ref(&mut self, set: &TokenSet) -> LookaheadSetId {
if let Some(index) = self.sets.get_index_of(set) {
return LookaheadSetId(index as u32);
}
self.intern(set.clone())
}
pub fn singleton(&mut self, symbol: Symbol) -> LookaheadSetId {
self.insert(Self::EMPTY, symbol)
}
pub fn insert(&mut self, id: LookaheadSetId, symbol: Symbol) -> LookaheadSetId {
if self.get(id).contains(symbol) {
return id;
}
if let Some(&result) = self.insert_memo.get(&(id, symbol)) {
return result;
}
let mut set = self.get(id).clone();
set.insert(symbol);
let result = self.intern(set);
self.insert_memo.insert((id, symbol), result);
result
}
pub fn union(&mut self, left: LookaheadSetId, right: LookaheadSetId) -> LookaheadSetId {
if left == right || right == Self::EMPTY {
return left;
}
if left == Self::EMPTY {
return right;
}
let key = (left.min(right), left.max(right));
if let Some(&result) = self.union_memo.get(&key) {
return result;
}
let mut set = self.get(key.0).clone();
set.insert_all(self.get(key.1));
let result = self.intern(set);
self.union_memo.insert(key, result);
result
}
}
/// Represents a set of in-progress matches of productions in a grammar.
///
/// For each in-progress match, a set of "lookaheads" (tokens that are allowed to
@ -275,7 +363,7 @@ pub struct ParseItemSet<'a> {
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ParseItemSetEntry<'a> {
pub item: ParseItem<'a>,
pub lookaheads: TokenSet,
pub lookaheads: LookaheadSetId,
pub following_reserved_word_set: ReservedWordSetId,
}
@ -305,6 +393,7 @@ pub struct ParseItemSetDisplay<'a>(
pub &'a SyntaxGrammar,
pub &'a LexicalGrammar,
pub &'a StrPool,
pub &'a LookaheadSetPool,
);
impl<'a> ParseItem<'a> {
@ -411,7 +500,7 @@ impl<'a> ParseItemSet<'a> {
i,
ParseItemSetEntry {
item,
lookaheads: TokenSet::new(),
lookaheads: LookaheadSetPool::EMPTY,
following_reserved_word_set: ReservedWordSetId::default(),
},
);
@ -594,7 +683,7 @@ impl fmt::Display for ParseItemSetDisplay<'_> {
f,
"{}\t{}",
ParseItemDisplay(&entry.item, self.1, self.2, self.3),
TokenSetDisplay(&entry.lookaheads, self.1, self.2, self.3),
TokenSetDisplay(self.4.get(entry.lookaheads), self.1, self.2, self.3),
)?;
if entry.following_reserved_word_set != ReservedWordSetId::default() {
write!(

86
crates/generate/src/build_tables/item_set_builder.rs
View file

@ -6,17 +6,30 @@ use super::item::{
ItemKeyMap, ParseItem, ParseItemDisplay, ParseItemSet, ParseItemSetEntry, TokenSetDisplay,
};
use crate::{
build_tables::item::{LookaheadSetId, LookaheadSetPool},
grammars::{InlinedProductionMap, LexicalGrammar, ReservedWordSetId, SyntaxGrammar},
rules::{Symbol, SymbolType, TokenSet},
strpool::StrPool,
};
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct TransitiveClosureAddition<'a> {
item: ParseItem<'a>,
info: FollowSetInfo,
info: AdditionInfo,
}
/// [`FollowSetInfo`] with an interned lookahead set and word-token membership precomputed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct AdditionInfo {
lookaheads: LookaheadSetId,
reserved_lookaheads: ReservedWordSetId,
propagates_lookaheads: bool,
contains_word: bool,
}
/// the mutable accumulator for a non-terminal's follow set. `lookaheads` is a
/// working `TokenSet` that can be unioned incrementally during traversal, to be
/// interned into [`AdditionInfo`] once complete.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
struct FollowSetInfo {
lookaheads: TokenSet,
@ -27,10 +40,13 @@ struct FollowSetInfo {
pub struct ParseItemSetBuilder<'a> {
syntax_grammar: &'a SyntaxGrammar,
first_sets: FxHashMap<Symbol, TokenSet>,
/// Each FIRST set interned, for closure propagation by id
first_set_ids: FxHashMap<Symbol, LookaheadSetId>,
reserved_first_sets: FxHashMap<Symbol, ReservedWordSetId>,
last_sets: FxHashMap<Symbol, TokenSet>,
inlines: &'a InlinedProductionMap,
pub key_map: &'a ItemKeyMap,
pub lookaheads: LookaheadSetPool,
transitive_closure_additions: Vec<Vec<TransitiveClosureAddition<'a>>>,
}
@ -51,10 +67,12 @@ impl<'a> ParseItemSetBuilder<'a> {
let mut result = Self {
syntax_grammar,
first_sets: FxHashMap::default(),
first_set_ids: FxHashMap::default(),
reserved_first_sets: FxHashMap::default(),
last_sets: FxHashMap::default(),
inlines,
key_map,
lookaheads: LookaheadSetPool::new(),
transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()],
};
@ -139,6 +157,13 @@ impl<'a> ParseItemSetBuilder<'a> {
}
}
// Intern each FIRST set so closure propagation can union by id.
for (symbol, set) in &result.first_sets {
result
.first_set_ids
.insert(*symbol, result.lookaheads.intern_ref(set));
}
// To compute an item set's transitive closure, we find each item in the set
// whose next symbol is a non-terminal, and we add new items to the set for
// each of that symbol's productions. These productions might themselves begin
@ -219,16 +244,23 @@ impl<'a> ParseItemSetBuilder<'a> {
// Store all of those non-terminals' productions, along with their associated
// lookahead info, as *additions* associated with non-terminal `i`.
let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
for (&variable_index, follow_set_info) in &follow_set_info_by_non_terminal {
let non_terminal = Symbol::non_terminal(variable_index);
let variable_index = variable_index as u32;
if syntax_grammar.variables_to_inline.contains(&non_terminal) {
continue;
}
for prod_id in syntax_grammar.variable_prod_ids(variable_index as usize) {
let info = AdditionInfo {
lookaheads: result.lookaheads.intern_ref(&follow_set_info.lookaheads),
reserved_lookaheads: follow_set_info.reserved_lookaheads,
propagates_lookaheads: follow_set_info.propagates_lookaheads,
contains_word: syntax_grammar
.word_token
.is_some_and(|w| follow_set_info.lookaheads.contains(w)),
};
let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
for prod_id in syntax_grammar.variable_prod_ids(variable_index) {
let item = ParseItem {
variable_index,
variable_index: variable_index as u32,
prod_id,
keys: key_map.keys_for(prod_id),
step_index: 0,
@ -241,17 +273,14 @@ impl<'a> ParseItemSetBuilder<'a> {
additions_for_non_terminal,
TransitiveClosureAddition {
item: item.substitute_production(id, key_map.keys_for(id)),
info: follow_set_info.clone(),
info,
},
);
}
} else {
find_or_push(
additions_for_non_terminal,
TransitiveClosureAddition {
item,
info: follow_set_info.clone(),
},
TransitiveClosureAddition { item, info },
);
}
}
@ -262,7 +291,7 @@ impl<'a> ParseItemSetBuilder<'a> {
}
#[must_use]
pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
pub fn transitive_closure(&mut self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
let mut result = ParseItemSet::default();
for entry in &item_set.entries {
if let Some(ids) = self
@ -276,7 +305,7 @@ impl<'a> ParseItemSetBuilder<'a> {
item: entry
.item
.substitute_production(id, self.key_map.keys_for(id)),
lookaheads: entry.lookaheads.clone(),
lookaheads: entry.lookaheads,
following_reserved_word_set: entry.following_reserved_word_set,
},
);
@ -304,7 +333,7 @@ impl<'a> ParseItemSetBuilder<'a> {
&self.last_sets[&symbol]
}
fn add_item(&self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
fn add_item(&mut self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
if let Some(step) = entry.item.step(self.syntax_grammar)
&& step.symbol().is_non_terminal()
{
@ -313,42 +342,41 @@ impl<'a> ParseItemSetBuilder<'a> {
// Determine which tokens can follow this non-terminal.
let (following_tokens, following_reserved_tokens) = if let Some(next_step) = next_step {
(
self.first_sets.get(&next_step.symbol()).unwrap(),
*self.reserved_first_sets.get(&next_step.symbol()).unwrap(),
self.first_set_ids[&next_step.symbol()],
self.reserved_first_sets[&next_step.symbol()],
)
} else {
(&entry.lookaheads, entry.following_reserved_word_set)
(entry.lookaheads, entry.following_reserved_word_set)
};
// Use the pre-computed *additions* to expand the non-terminal.
for addition in &self.transitive_closure_additions[step.symbol().index as usize] {
let entry = set.insert(addition.item);
entry.lookaheads.insert_all(&addition.info.lookaheads);
let e = set.insert(addition.item);
e.lookaheads = self
.lookaheads
.union(e.lookaheads, addition.info.lookaheads);
if let Some(word_token) = self.syntax_grammar.word_token
&& addition.info.lookaheads.contains(word_token)
{
entry.following_reserved_word_set = entry
if addition.info.contains_word {
e.following_reserved_word_set = e
.following_reserved_word_set
.max(addition.info.reserved_lookaheads);
}
if addition.info.propagates_lookaheads {
entry.lookaheads.insert_all(following_tokens);
e.lookaheads = self.lookaheads.union(e.lookaheads, following_tokens);
if let Some(word_token) = self.syntax_grammar.word_token
&& following_tokens.contains(word_token)
&& self.lookaheads.get(following_tokens).contains(word_token)
{
entry.following_reserved_word_set = entry
.following_reserved_word_set
.max(following_reserved_tokens);
e.following_reserved_word_set =
e.following_reserved_word_set.max(following_reserved_tokens);
}
}
}
}
let e = set.insert(entry.item);
e.lookaheads.insert_all(&entry.lookaheads);
e.lookaheads = self.lookaheads.union(e.lookaheads, entry.lookaheads);
e.following_reserved_word_set = e
.following_reserved_word_set
.max(entry.following_reserved_word_set);