emacs/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:36:22 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

perf(generate): shrink parse/lex/production ids from usize to u32

Browse source 
`ParseStateId`, `LexStateId`, `ProductionInfoId`, and `ReservedWordSetId`
each held a `usize`, while the runtime stores state/field ids as `u16` (so
`u32` has plenty of headroom). Shrinking them saves space for every parse-table
entry, and a number of other data structures.

Reduces wall time by 2-5%, peak rss by 9-12%.
This commit is contained in:
Will Lillis 2026-08-09 11:51:45 -05:00
parent 87d14ddfc6
commit dbcdd0416b
9 changed files with 109 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
crates/generate/src/build_tables.rs
View file

@ -331,7 +331,7 @@ fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &S
.unwrap_or_else(|| {
parse_table.external_lex_states.push(external_tokens);
parse_table.external_lex_states.len() - 1
});
}) as u32;
}
}
@ -524,7 +524,7 @@ fn report_state_info<'a>(
for state_index in state_indices {
let id = parse_table.states[state_index].id;
let preceding_symbols = &parse_state_info.preceding_symbols_by_id[id];
let preceding_symbols = &parse_state_info.preceding_symbols_by_id[id as usize];
let item_set = parse_state_info.item_set(id);
info!("state index: {state_index}");
info!("state id: {id}");

46
crates/generate/src/build_tables/build_lex_table.rs
View file

@ -71,13 +71,13 @@ pub fn build_lex_table(
coincident_token_index,
) {
did_merge = true;
entry.1.push(i);
entry.1.push(i as u32);
break;
}
}
if !did_merge {
parse_state_ids_by_token_set.push((tokens, vec![i]));
parse_state_ids_by_token_set.push((tokens, vec![i as u32]));
}
}
@ -85,7 +85,7 @@ pub fn build_lex_table(
for (tokens, parse_state_ids) in parse_state_ids_by_token_set {
let lex_state_id = builder.add_state_for_tokens(&tokens);
for id in parse_state_ids {
parse_table.states[id].lex_state_id = lex_state_id;
parse_table.states[id as usize].lex_state_id = lex_state_id;
}
}
@ -141,7 +141,7 @@ struct LexTableBuilder<'a> {
cursor: NfaCursor<'a>,
table: LexTable,
state_queue: VecDeque<QueueEntry>,
state_ids_by_nfa_state_set: FxHashMap<(Vec<u32>, bool), usize>,
state_ids_by_nfa_state_set: FxHashMap<(Vec<u32>, bool), LexStateId>,
}
impl<'a> LexTableBuilder<'a> {
@ -205,7 +205,7 @@ impl<'a> LexTableBuilder<'a> {
{
Entry::Occupied(o) => (*o.get(), false),
Entry::Vacant(v) => {
let state_id = self.table.states.len();
let state_id = self.table.states.len() as u32;
self.table.states.push(LexState::default());
self.state_queue.push_back(QueueEntry {
state_id,
@ -242,7 +242,7 @@ impl<'a> LexTableBuilder<'a> {
// character that leads to the empty set of NFA states.
if eof_valid {
let (next_state_id, _) = self.add_state(Vec::new(), false);
self.table.states[state_id].eof_action = Some(AdvanceAction {
self.table.states[state_id as usize].eof_action = Some(AdvanceAction {
state: next_state_id,
in_main_token: true,
});
@ -263,7 +263,7 @@ impl<'a> LexTableBuilder<'a> {
let (next_state_id, _) =
self.add_state(transition.states, eof_valid && transition.is_separator);
self.table.states[state_id].advance_actions.push((
self.table.states[state_id as usize].advance_actions.push((
transition.characters,
AdvanceAction {
state: next_state_id,
@ -273,9 +273,10 @@ impl<'a> LexTableBuilder<'a> {
}
if let Some((complete_id, _)) = completion {
self.table.states[state_id].accept_action = Some(Symbol::terminal(complete_id));
self.table.states[state_id as usize].accept_action =
Some(Symbol::terminal(complete_id));
} else if self.cursor.state_ids.is_empty() {
self.table.states[state_id].accept_action = Some(Symbol::end());
self.table.states[state_id as usize].accept_action = Some(Symbol::end());
}
}
}
@ -368,7 +369,7 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
state_ids_by_signature
.entry(signature)
.or_insert(Vec::new())
.push(i);
.push(i as u32);
}
let mut state_ids_by_group_id = state_ids_by_signature
.into_iter()
@ -381,10 +382,10 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
.unwrap();
state_ids_by_group_id.swap(error_group_index, 0);
let mut group_ids_by_state_id = vec![0; table.states.len()];
let mut group_ids_by_state_id = vec![0u32; table.states.len()];
for (group_id, state_ids) in state_ids_by_group_id.iter().enumerate() {
for state_id in state_ids {
group_ids_by_state_id[*state_id] = group_id;
group_ids_by_state_id[*state_id as usize] = group_id as u32;
}
}
@ -399,19 +400,19 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
for state_ids in &state_ids_by_group_id {
let mut new_state = LexState::default();
mem::swap(&mut new_state, &mut table.states[state_ids[0]]);
mem::swap(&mut new_state, &mut table.states[state_ids[0] as usize]);
for (_, advance_action) in &mut new_state.advance_actions {
advance_action.state = group_ids_by_state_id[advance_action.state];
advance_action.state = group_ids_by_state_id[advance_action.state as usize];
}
if let Some(eof_action) = &mut new_state.eof_action {
eof_action.state = group_ids_by_state_id[eof_action.state];
eof_action.state = group_ids_by_state_id[eof_action.state as usize];
}
new_states.push(new_state);
}
for state in &mut parse_table.states {
state.lex_state_id = group_ids_by_state_id[state.lex_state_id];
state.lex_state_id = group_ids_by_state_id[state.lex_state_id as usize];
}
table.states = new_states;
@ -426,7 +427,8 @@ fn lex_states_differ(
.iter()
.zip(right.advance_actions.iter())
.any(|(left, right)| {
group_ids_by_state_id[left.1.state] != group_ids_by_state_id[right.1.state]
group_ids_by_state_id[left.1.state as usize]
!= group_ids_by_state_id[right.1.state as usize]
})
}
@ -436,9 +438,9 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
old_ids_by_new_id[1..].sort_by_key(|id| &table.states[*id]);
// Get the inverse mapping
let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
let mut new_ids_by_old_id = vec![0u32; old_ids_by_new_id.len()];
for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
new_ids_by_old_id[*old_id] = id;
new_ids_by_old_id[*old_id] = id as u32;
}
// Reorder the parse states and update their references to reflect
@ -449,10 +451,10 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
let mut state = LexState::default();
mem::swap(&mut state, &mut table.states[*old_id]);
for (_, advance_action) in &mut state.advance_actions {
advance_action.state = new_ids_by_old_id[advance_action.state];
advance_action.state = new_ids_by_old_id[advance_action.state as usize];
}
if let Some(eof_action) = &mut state.eof_action {
eof_action.state = new_ids_by_old_id[eof_action.state];
eof_action.state = new_ids_by_old_id[eof_action.state as usize];
}
state
})
@ -460,6 +462,6 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
// Update the parse table's lex state references
for state in &mut parse_table.states {
state.lex_state_id = new_ids_by_old_id[state.lex_state_id];
state.lex_state_id = new_ids_by_old_id[state.lex_state_id as usize];
}
}

36
crates/generate/src/build_tables/build_parse_table.rs
View file

@ -41,7 +41,7 @@ pub struct ParseStateInfo<'a> {
impl<'a> ParseStateInfo<'a> {
#[must_use]
pub fn item_set(&self, id: ParseStateId) -> &ParseItemSet<'a> {
self.item_sets_by_ids.get_index(id).unwrap().0
self.item_sets_by_ids.get_index(id as usize).unwrap().0
}
}
@ -70,12 +70,12 @@ struct ParseTableBuilder<'a> {
syntax_grammar: &'a SyntaxGrammar,
lexical_grammar: &'a LexicalGrammar,
variable_info: &'a [VariableInfo],
core_ids_by_core: FxHashMap<ParseItemSetCore<'a>, usize>,
core_ids_by_core: FxHashMap<ParseItemSetCore<'a>, u32>,
state_ids_by_item_set: IndexMap<ParseItemSet<'a>, ParseStateId, BuildHasherDefault<FxHasher>>,
preceding_symbols_by_id: Vec<SymbolSequence>,
production_info_ids_by_prod_id: Vec<Option<ProductionInfoId>>,
parse_state_queue: VecDeque<ParseStateQueueEntry>,
non_terminal_extra_states: Vec<(Symbol, usize)>,
non_terminal_extra_states: Vec<(Symbol, ParseStateId)>,
actual_conflicts: FxHashSet<Vec<Symbol>>,
parse_table: ParseTable,
str_pool: &'a StrPool,
@ -366,14 +366,14 @@ impl<'a> ParseTableBuilder<'a> {
// Two states are identical iff their kernels match.
let kernel = self
.state_ids_by_item_set
.get_index(entry.state_id)
.get_index(entry.state_id as usize)
// Invariant: `state_id` is the map's insertion index
.unwrap()
.0;
let item_set = self.item_set_builder.transitive_closure(kernel);
self.add_actions(
self.preceding_symbols_by_id[entry.state_id].clone(),
self.preceding_symbols_by_id[entry.state_id as usize].clone(),
entry.preceding_auxiliary_symbols,
entry.state_id,
&item_set,
@ -415,10 +415,10 @@ impl<'a> ParseTableBuilder<'a> {
// parse states to populate.
Entry::Vacant(v) => {
let core = v.key().core();
let core_count = self.core_ids_by_core.len();
let core_count = self.core_ids_by_core.len() as u32;
let core_id = *self.core_ids_by_core.entry(core).or_insert(core_count);
let state_id = self.parse_table.states.len();
let state_id = self.parse_table.states.len() as u32;
self.preceding_symbols_by_id.push(preceding_symbols.clone());
self.parse_table.states.push(ParseState {
@ -549,7 +549,7 @@ impl<'a> ParseTableBuilder<'a> {
let precedence = item.precedence(self.syntax_grammar);
let associativity = item.associativity(self.syntax_grammar);
for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() {
let table_entry = self.parse_table.states[state_id]
let table_entry = self.parse_table.states[state_id as usize]
.terminal_entries
.entry(lookahead)
.or_insert_with(ParseTableEntry::new);
@ -609,7 +609,7 @@ impl<'a> ParseTableBuilder<'a> {
);
preceding_symbols.pop();
let entry = self.parse_table.states[state_id]
let entry = self.parse_table.states[state_id as usize]
.terminal_entries
.entry(symbol);
if let Entry::Occupied(e) = &entry
@ -635,7 +635,7 @@ impl<'a> ParseTableBuilder<'a> {
next_item_set,
);
preceding_symbols.pop();
self.parse_table.states[state_id]
self.parse_table.states[state_id as usize]
.nonterminal_entries
.insert(symbol, GotoAction::Goto(next_state_id));
}
@ -657,7 +657,7 @@ impl<'a> ParseTableBuilder<'a> {
}
// Add actions for the grammar's `extra` symbols.
let state = &mut self.parse_table.states[state_id];
let state = &mut self.parse_table.states[state_id as usize];
let is_end_of_non_terminal_extra = state.is_end_of_non_terminal_extra();
// If this state represents the end of a non-terminal extra rule, then make sure that
@ -732,7 +732,7 @@ impl<'a> ParseTableBuilder<'a> {
.filter_map(|entry| {
if let Some(next_step) = entry.item.step(self.syntax_grammar) {
if next_step.symbol() == keyword_capture_token {
Some(ReservedWordSetId(usize::from(next_step.reserved)))
Some(ReservedWordSetId(u32::from(next_step.reserved)))
} else {
None
}
@ -750,7 +750,7 @@ impl<'a> ParseTableBuilder<'a> {
.max();
if let Some(reserved_word_set_id) = reserved_word_set_id {
state.reserved_words =
self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0].clone();
self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0 as usize].clone();
}
}
@ -766,7 +766,7 @@ impl<'a> ParseTableBuilder<'a> {
conflicting_lookahead: Symbol,
reduction_info: &ReductionInfo,
) -> BuildTableResult<()> {
let entry = self.parse_table.states[state_id]
let entry = self.parse_table.states[state_id as usize]
.terminal_entries
.get_mut(&conflicting_lookahead)
.unwrap();
@ -902,7 +902,7 @@ impl<'a> ParseTableBuilder<'a> {
}
// If all of the actions but one have been eliminated, then there's no problem.
let entry = self.parse_table.states[state_id]
let entry = self.parse_table.states[state_id as usize]
.terminal_entries
.get_mut(&conflicting_lookahead)
.unwrap();
@ -1172,7 +1172,7 @@ impl<'a> ParseTableBuilder<'a> {
.entry(field_name)
.or_default()
.push(FieldLocation {
index: i,
index: i as u32,
inherited: false,
});
}
@ -1189,7 +1189,7 @@ impl<'a> ParseTableBuilder<'a> {
.entry(field_name)
.or_default()
.push(FieldLocation {
index: i,
index: i as u32,
inherited: true,
});
}
@ -1217,7 +1217,7 @@ impl<'a> ParseTableBuilder<'a> {
} else {
self.parse_table.production_infos.push(production_info);
self.parse_table.production_infos.len() - 1
};
} as ProductionInfoId;
self.production_info_ids_by_prod_id[item.prod_id as usize] = Some(id);
id
}

4
crates/generate/src/build_tables/item_set_builder.rs
View file

@ -133,7 +133,7 @@ impl<'a> ParseItemSetBuilder<'a> {
symbols_to_process.push(symbol);
}
*reserved_first_set =
(*reserved_first_set).max(ReservedWordSetId(step.reserved as usize));
(*reserved_first_set).max(ReservedWordSetId(u32::from(step.reserved)));
}
}
}
@ -325,7 +325,7 @@ impl<'a> ParseItemSetBuilder<'a> {
#[must_use]
pub fn reserved_first_set(&self, symbol: Symbol) -> Option<&TokenSet> {
let id = *self.reserved_first_sets.get(&symbol)?;
Some(&self.syntax_grammar.reserved_word_sets[id.0])
Some(&self.syntax_grammar.reserved_word_sets[id.0 as usize])
}
#[must_use]

57
crates/generate/src/build_tables/minimize_parse_table.rs
View file

@ -16,7 +16,7 @@ use crate::{
/// Index into [`SyntaxGrammar::variables`]. All nonterminal [`Symbol`]s share
/// the same `kind`, so storing the index alone is sufficient for ordering.
type NonterminalIndex = usize;
type NonterminalIndex = u32;
/// A [`Symbol`] packed into a `u64` for O(1) sort-key comparison.
///
@ -194,7 +194,7 @@ impl Minimizer<'_> {
if let Some(symbol) = unit_reduction_symbol
&& only_unit_reductions
{
unit_reduction_symbols_by_state.insert(i, *symbol);
unit_reduction_symbols_by_state.insert(i as u32, *symbol);
}
}
@ -232,9 +232,9 @@ impl Minimizer<'_> {
// Pre-allocate for the maximum possible number of groups (one per state) to
// avoid reallocs as split_state_id_groups pushes new groups.
let mut state_ids_by_group_id = Vec::with_capacity(self.parse_table.states.len());
state_ids_by_group_id.resize(core_count, Vec::new());
state_ids_by_group_id.resize(core_count as usize, Vec::new());
for (i, state) in self.parse_table.states.iter().enumerate() {
state_ids_by_group_id[state.core_id].push(i);
state_ids_by_group_id[state.core_id as usize].push(i as u32);
group_ids_by_state_id.push(state.core_id);
}
@ -353,7 +353,7 @@ impl Minimizer<'_> {
let mut entries = state
.nonterminal_entries
.iter()
.map(|(sym, action)| (sym.index as usize, *action))
.map(|(sym, action)| (sym.index, *action))
.collect::<Vec<(NonterminalIndex, GotoAction)>>();
entries.sort_unstable_by_key(|&(idx, _)| idx);
entries
@ -385,12 +385,12 @@ impl Minimizer<'_> {
let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
for state_ids in &state_ids_by_group_id {
// Initialize the new state based on the first old state in the group.
let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0]]);
let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0] as usize]);
// Extend the new state with all of the actions from the other old states
// in the group.
for state_id in &state_ids[1..] {
let other_parse_state = mem::take(&mut self.parse_table.states[*state_id]);
let other_parse_state = mem::take(&mut self.parse_table.states[*state_id as usize]);
parse_state
.terminal_entries
@ -407,7 +407,8 @@ impl Minimizer<'_> {
}
// Update the new state's outgoing references using the new grouping.
parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
parse_state
.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id as usize]);
new_states.push(parse_state);
}
@ -422,8 +423,8 @@ impl Minimizer<'_> {
entry_maps: &[Vec<(SymbolKey, &ParseTableEntry)>],
bits: &ConflictBits,
) -> bool {
let entries1 = &entry_maps[state1.id];
let entries2 = &entry_maps[state2.id];
let entries1 = &entry_maps[state1.id as usize];
let entries2 = &entry_maps[state2.id as usize];
let len1 = entries1.len();
let len2 = entries2.len();
let mut i = 0;
@ -488,10 +489,10 @@ impl Minimizer<'_> {
state2: &ParseState,
group_ids_by_state_id: &[ParseStateId],
shift_maps: &[Vec<(SymbolKey, ParseStateId)>],
nonterminal_maps: &[Vec<(usize, GotoAction)>],
nonterminal_maps: &[Vec<(NonterminalIndex, GotoAction)>],
) -> bool {
let shifts1 = &shift_maps[state1.id];
let shifts2 = &shift_maps[state2.id];
let shifts1 = &shift_maps[state1.id as usize];
let shifts2 = &shift_maps[state2.id as usize];
let mut i = 0;
let mut j = 0;
while i < shifts1.len() && j < shifts2.len() {
@ -502,8 +503,8 @@ impl Minimizer<'_> {
Ordering::Less => i += 1,
Ordering::Greater => j += 1,
Ordering::Equal => {
let group1 = group_ids_by_state_id[s1];
let group2 = group_ids_by_state_id[s2];
let group1 = group_ids_by_state_id[s1 as usize];
let group2 = group_ids_by_state_id[s2 as usize];
if group1 != group2 {
debug!(
"split states {} {} - successors for {} are split: {s1} {s2}",
@ -519,8 +520,8 @@ impl Minimizer<'_> {
}
}
let nonterms1 = &nonterminal_maps[state1.id];
let nonterms2 = &nonterminal_maps[state2.id];
let nonterms1 = &nonterminal_maps[state1.id as usize];
let nonterms2 = &nonterminal_maps[state2.id as usize];
let mut i = 0;
let mut j = 0;
while i < nonterms1.len() && j < nonterms2.len() {
@ -534,15 +535,15 @@ impl Minimizer<'_> {
match (s1, s2) {
(GotoAction::ShiftExtra, GotoAction::ShiftExtra) => {}
(GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
let group1 = group_ids_by_state_id[s1];
let group2 = group_ids_by_state_id[s2];
let group1 = group_ids_by_state_id[s1 as usize];
let group2 = group_ids_by_state_id[s2 as usize];
if group1 != group2 {
debug!(
"split states {} {} - successors for {} are split: {s1} {s2}",
state1.id,
state2.id,
self.str_pool
.resolve(self.syntax_grammar.variables[idx1].name),
.resolve(self.syntax_grammar.variables[idx1 as usize].name),
);
return true;
}
@ -591,8 +592,8 @@ impl Minimizer<'_> {
},
) = (action1, action2)
{
let group1 = group_ids_by_state_id[*s1];
let group2 = group_ids_by_state_id[*s2];
let group1 = group_ids_by_state_id[*s1 as usize];
let group2 = group_ids_by_state_id[*s2 as usize];
if group1 == group2 && is_repetition1 == is_repetition2 {
continue;
}
@ -663,7 +664,7 @@ impl Minimizer<'_> {
// conflict row against the state's terminal bits, masking out the word/keyword
// exemptions.
let row = bits.get_conflict_row(new_token.index as usize);
let right_terminal_bits = bits.get_state_row(right_state.id);
let right_terminal_bits = bits.get_state_row(right_state.id as usize);
for (w, &row_word) in row.iter().enumerate() {
let mut candidates = right_terminal_bits[w] & row_word;
if new_token_is_keyword
@ -714,13 +715,13 @@ impl Minimizer<'_> {
for state in &self.parse_table.states {
for referenced_state in state.referenced_states() {
state_usage_map[referenced_state] = true;
state_usage_map[referenced_state as usize] = true;
}
}
let mut removed_predecessor_count = 0;
let mut state_replacement_map = vec![0; self.parse_table.states.len()];
for state_id in 0..self.parse_table.states.len() {
state_replacement_map[state_id] = state_id - removed_predecessor_count;
state_replacement_map[state_id] = (state_id - removed_predecessor_count) as u32;
if !state_usage_map[state_id] {
removed_predecessor_count += 1;
}
@ -730,7 +731,7 @@ impl Minimizer<'_> {
while state_id < self.parse_table.states.len() {
if state_usage_map[original_state_id] {
self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
state_replacement_map[other_state_id]
state_replacement_map[other_state_id as usize]
});
state_id += 1;
} else {
@ -757,7 +758,7 @@ impl Minimizer<'_> {
// Get the inverse mapping
let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
new_ids_by_old_id[*old_id] = id;
new_ids_by_old_id[*old_id] = id as u32;
}
// Reorder the parse states and update their references to reflect
@ -767,7 +768,7 @@ impl Minimizer<'_> {
.map(|old_id| {
let mut state = ParseState::default();
mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
state.update_referenced_states(|id, _| new_ids_by_old_id[id as usize]);
state
})
.collect();

26
crates/generate/src/dedup.rs
View file

@ -1,10 +1,10 @@
#[inline]
pub fn split_state_id_groups<S>(
states: &[S],
state_ids_by_group_id: &mut Vec<Vec<usize>>,
group_ids_by_state_id: &mut [usize],
start_group_id: usize,
mut should_split: impl FnMut(&S, &S, &[usize]) -> bool,
state_ids_by_group_id: &mut Vec<Vec<u32>>,
group_ids_by_state_id: &mut [u32],
start_group_id: u32,
mut should_split: impl FnMut(&S, &S, &[u32]) -> bool,
) -> bool {
let mut result = false;
@ -12,13 +12,13 @@ pub fn split_state_id_groups<S>(
let mut is_split = vec![false; states.len()];
let mut group_id = start_group_id;
while group_id < state_ids_by_group_id.len() {
let state_ids = &state_ids_by_group_id[group_id];
while (group_id as usize) < state_ids_by_group_id.len() {
let state_ids = &state_ids_by_group_id[group_id as usize];
let mut split_state_ids = Vec::new();
let mut i = 0;
while i < state_ids.len() {
let left_state_id = state_ids[i];
let left_state_id = state_ids[i] as usize;
if is_split[left_state_id] {
i += 1;
continue;
@ -30,7 +30,7 @@ pub fn split_state_id_groups<S>(
// this state.
let mut j = i + 1;
while j < state_ids.len() {
let right_state_id = state_ids[j];
let right_state_id = state_ids[j] as usize;
if is_split[right_state_id] {
j += 1;
continue;
@ -38,7 +38,7 @@ pub fn split_state_id_groups<S>(
let right_state = &states[right_state_id];
if should_split(left_state, right_state, group_ids_by_state_id) {
split_state_ids.push(right_state_id);
split_state_ids.push(right_state_id as u32);
is_split[right_state_id] = true;
}
@ -51,12 +51,12 @@ pub fn split_state_id_groups<S>(
// If any states were removed from the group, add them all as a new group.
if !split_state_ids.is_empty() {
result = true;
state_ids_by_group_id[group_id].retain(|i| !is_split[*i]);
state_ids_by_group_id[group_id as usize].retain(|i| !is_split[*i as usize]);
let new_group_id = state_ids_by_group_id.len();
let new_group_id = state_ids_by_group_id.len() as u32;
for id in &split_state_ids {
group_ids_by_state_id[*id] = new_group_id;
is_split[*id] = false;
group_ids_by_state_id[*id as usize] = new_group_id;
is_split[*id as usize] = false;
}
state_ids_by_group_id.push(split_state_ids);

2
crates/generate/src/grammars.rs
View file

@ -217,7 +217,7 @@ impl ProductionStep {
// Extracted syntax grammar
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ReservedWordSetId(pub usize);
pub struct ReservedWordSetId(pub u32);
impl std::fmt::Display for ReservedWordSetId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

24
crates/generate/src/render.rs
View file

@ -1306,7 +1306,7 @@ impl Generator {
fn add_parse_table(&mut self) -> RenderResult<()> {
let mut parse_table_entries = FxHashMap::default();
let mut next_parse_action_list_index = 0;
let mut next_parse_action_list_index = 0u32;
// Parse action lists zero is for the default value, when a symbol is not valid.
Self::get_parse_action_list_id(
@ -1352,7 +1352,7 @@ impl Generator {
"[{}] = STATE({}),",
self.symbol_ids[symbol],
match action {
GotoAction::Goto(state) => *state,
GotoAction::Goto(state) => *state as usize,
GotoAction::ShiftExtra => i,
}
);
@ -1386,7 +1386,7 @@ impl Generator {
.len()
.saturating_sub(self.large_state_count),
);
let mut symbols_by_value = FxHashMap::<(usize, SymbolType), Vec<Symbol>>::default();
let mut symbols_by_value = FxHashMap::<(u32, SymbolType), Vec<Symbol>>::default();
for state in self.parse_table.states.iter().skip(self.large_state_count) {
small_state_indices.push(next_table_index);
symbols_by_value.clear();
@ -1413,7 +1413,7 @@ impl Generator {
let state_id = match action {
GotoAction::Goto(i) => *i,
GotoAction::ShiftExtra => {
self.large_state_count + small_state_indices.len() - 1
(self.large_state_count + small_state_indices.len() - 1) as u32
}
};
symbols_by_value
@ -1474,8 +1474,10 @@ impl Generator {
add_line!(self, "}};");
add_line!(self, "");
}
if next_parse_action_list_index >= usize::from(u16::MAX) {
Err(RenderError::ParseTable(next_parse_action_list_index))?;
if next_parse_action_list_index >= u32::from(u16::MAX) {
Err(RenderError::ParseTable(
next_parse_action_list_index as usize,
))?;
}
let mut parse_table_entries = parse_table_entries
@ -1488,7 +1490,7 @@ impl Generator {
Ok(())
}
fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
fn add_parse_action_list(&mut self, parse_table_entries: Vec<(u32, ParseTableEntry)>) {
add_line!(
self,
"static const TSParseActionEntry ts_parse_actions[] = {{"
@ -1699,15 +1701,15 @@ impl Generator {
fn get_parse_action_list_id(
entry: &ParseTableEntry,
parse_table_entries: &mut FxHashMap<ParseTableEntry, usize>,
next_parse_action_list_index: &mut usize,
) -> usize {
parse_table_entries: &mut FxHashMap<ParseTableEntry, u32>,
next_parse_action_list_index: &mut u32,
) -> u32 {
if let Some(&index) = parse_table_entries.get(entry) {
index
} else {
let result = *next_parse_action_list_index;
parse_table_entries.insert(entry.clone(), result);
*next_parse_action_list_index += 1 + entry.actions.len();
*next_parse_action_list_index += 1 + entry.actions.len() as u32;
result
}
}

13
crates/generate/src/tables.rs
View file

@ -6,9 +6,10 @@ use super::{
rules::{Symbol, TokenSet},
strpool::StrId,
};
pub type ProductionInfoId = usize;
pub type ParseStateId = usize;
pub type LexStateId = usize;
pub type ProductionInfoId = u32;
pub type ParseStateId = u32;
pub type LexStateId = u32;
use std::hash::BuildHasherDefault;
@ -136,12 +137,12 @@ pub struct ParseState {
pub reserved_words: TokenSet,
pub lex_state_id: LexStateId,
pub external_lex_state_id: LexStateId,
pub core_id: usize,
pub core_id: u32,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct FieldLocation {
pub index: usize,
pub index: u32,
pub inherited: bool,
}
@ -216,7 +217,7 @@ impl ParseState {
pub fn update_referenced_states<F>(&mut self, mut f: F)
where
F: FnMut(usize, &Self) -> usize,
F: FnMut(ParseStateId, &Self) -> ParseStateId,
{
let mut updates = Vec::new();
for (symbol, entry) in &self.terminal_entries {