libs-code/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:26:23 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat!: add an eof function

Browse source 
This commit adds an `eof()` function for grammars, which is easier to
use than the NUL byte directly. It compiles down to a constraint that
the enclosing production can only reduce at end of input, not to a
shiftable token.

BREAKING CHANGE: Public error types for `tree-sitter-generate` were modified.

Co-authored-by: Will Lillis <will.lillis24@gmail.com>
This commit is contained in:
Amaan Qureshi 2026-08-16 00:31:39 -04:00 committed by Will Lillis
parent baad4174e5
commit 74b7d0c951
35 changed files with 802 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
crates/cli/npm/dsl.d.ts vendored
View file

@ -15,6 +15,8 @@ type SeqRule = { type: 'SEQ'; members: Rule[] };
type StringRule = { type: 'STRING'; value: string };
type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
type TokenRule = { type: 'TOKEN'; content: Rule };
type EOFRule = { type: 'EOF' };
type Rule =
| AliasRule
@ -33,7 +35,8 @@ type Rule =
| SeqRule
| StringRule
| SymbolRule<string>
| TokenRule;
| TokenRule
| EOFRule;
declare class RustRegex {
value: string;
@ -382,6 +385,19 @@ declare const token: {
immediate(rule: RuleOrLiteral): ImmediateTokenRule;
};
/**
* Matches the end of input. May only appear as the final symbol of a
* (possibly nested) sequence; a production ending in `eof()` reduces only
* when the lookahead is end-of-input, rather than shifting a token.
*
* Choice branches that continue past `eof()` are dropped as unreachable,
* and `eof()` is not allowed inside `token()`.
*
* Useful when a rule should match either an explicit terminator (e.g. a
* newline) or the end of the file.
*/
declare function eof(): EOFRule;
/**
* Creates a new language grammar with the provided schema.
*

44
crates/generate/src/build_tables/build_parse_table.rs
View file

@ -115,6 +115,7 @@ pub struct Interpretation {
pub conflicting_lookahead: String,
pub precedence: Option<String>,
pub associativity: Option<String>,
pub requires_eof_lookahead: bool,
}
#[derive(Debug, Serialize, Deserialize)]
@ -142,16 +143,21 @@ impl std::fmt::Display for ConflictError {
.iter()
.map(|i| {
let line = i.to_string();
let prec_line = if let (Some(precedence), Some(associativity)) =
(&i.precedence, &i.associativity)
{
Some(format!(
let mut annotations = Vec::new();
if let (Some(precedence), Some(associativity)) = (&i.precedence, &i.associativity) {
annotations.push(format!(
"(precedence: {precedence}, associativity: {associativity})",
))
));
} else if let Some(precedence) = &i.precedence {
annotations.push(format!("(precedence: {precedence})"));
}
if i.requires_eof_lookahead {
annotations.push("(reduces only at end of input)".to_string());
}
let prec_line = if annotations.is_empty() {
None
} else {
i.precedence
.as_ref()
.map(|precedence| format!("(precedence: {precedence})"))
Some(annotations.join(" "))
};
(line, prec_line)
@ -430,6 +436,7 @@ impl<'a> ParseTableBuilder<'a> {
nonterminal_entries: IndexMap::default(),
reserved_words: TokenSet::default(),
core_id,
has_eof_gated_reduce: false,
});
self.parse_state_queue.push_back(ParseStateQueueEntry {
state_id,
@ -549,7 +556,15 @@ impl<'a> ParseTableBuilder<'a> {
let precedence = item.precedence(self.syntax_grammar);
let associativity = item.associativity(self.syntax_grammar);
if item.production(self.syntax_grammar).requires_eof_lookahead {
self.parse_table.states[state_id as usize].has_eof_gated_reduce = true;
}
for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() {
if item.production(self.syntax_grammar).requires_eof_lookahead
&& lookahead != Symbol::end()
{
continue;
}
let table_entry = self.parse_table.states[state_id as usize]
.terminal_entries
.entry(lookahead)
@ -575,9 +590,15 @@ impl<'a> ParseTableBuilder<'a> {
lookaheads_with_conflicts.remove(lookahead);
*reduction_info = ReductionInfo::default();
}
// Two items that reduce identically build the same tree, so
// there is nothing for the user to resolve. Precedence is
// still compared first, because an item that only repeats an
// existing action can still outrank it and clear the entry.
Ordering::Equal => {
table_entry.actions.push(action);
lookaheads_with_conflicts.insert(lookahead);
if !table_entry.actions.contains(&action) {
table_entry.actions.push(action);
lookaheads_with_conflicts.insert(lookahead);
}
}
Ordering::Less => continue,
}
@ -997,6 +1018,9 @@ impl<'a> ParseTableBuilder<'a> {
conflicting_lookahead: self.symbol_name(conflicting_lookahead),
precedence,
associativity,
requires_eof_lookahead: item
.production(self.syntax_grammar)
.requires_eof_lookahead,
}
})
.collect::<Vec<_>>();

17
crates/generate/src/build_tables/item.rs
View file

@ -28,17 +28,19 @@ const fn start_production() -> ProdRef<'static> {
ProdRef {
steps: &START_STEPS,
dynamic_precedence: 0,
requires_eof_lookahead: false,
}
}
/// Precomputed identity keys for one `(production, dot)` pair.
///
/// `cmp` is the rank of the content tuple `Ord` compared (dynamic precedence,
/// length, precedence/associativity at the dot, then completed steps' aliases and fields
/// and remaining steps in full). Equal ranks hold _exactly_ when the tuple is equal,
/// so it doubles as the equality class for items without preceding inherited fields.
/// `eq_with_syms` subdivides `cmp` by the completed steps' symbols, which participate
/// in equality only when `has_preceding_inherited_fields` is set.
/// `requires_eof_lookahead`, length, precedence/associativity at the dot, then
/// completed steps' aliases and fields and remaining steps in full). Equal ranks
/// hold _exactly_ when the tuple is equal, so it doubles as the equality class
/// for items without preceding inherited fields. `eq_with_syms` subdivides `cmp`
/// by the completed steps' symbols, which participate in equality only when
/// `has_preceding_inherited_fields` is set.
#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
pub struct DotKeys {
pub cmp: u32,
@ -198,6 +200,11 @@ impl Ord for ItemContent<'_> {
self.production
.dynamic_precedence
.cmp(&other.production.dynamic_precedence)
.then_with(|| {
self.production
.requires_eof_lookahead
.cmp(&other.production.requires_eof_lookahead)
})
.then_with(|| {
self.production
.steps

31
crates/generate/src/build_tables/item_set_builder.rs
View file

@ -193,6 +193,8 @@ impl<'a> ParseItemSetBuilder<'a> {
//
// Rather than computing these additions recursively, we use an explicit stack.
let empty_lookaheads = TokenSet::new();
let mut eof_lookaheads = TokenSet::new();
eof_lookaheads.insert(Symbol::end());
let mut stack = Vec::new();
let mut follow_set_info_by_non_terminal = FxHashMap::<usize, FollowSetInfo>::default();
for i in 0..syntax_grammar.variables.len() {
@ -230,6 +232,13 @@ impl<'a> ParseItemSetBuilder<'a> {
result.reserved_first_sets[&next_step.symbol()],
false,
));
} else if production.requires_eof_lookahead {
stack.push((
symbol.index as usize,
&eof_lookaheads,
ReservedWordSetId::default(),
false,
));
} else {
stack.push((
symbol.index as usize,
@ -269,18 +278,36 @@ impl<'a> ParseItemSetBuilder<'a> {
if let Some(ids) = inlines.inlined_prod_ids(item.prod_id, item.step_index) {
for &id in ids {
let mut item_info = info;
if syntax_grammar.production(id).requires_eof_lookahead {
item_info.lookaheads =
result.lookaheads.intern_ref(&eof_lookaheads);
item_info.reserved_lookaheads = ReservedWordSetId::default();
item_info.propagates_lookaheads = false;
item_info.contains_word = false;
}
find_or_push(
additions_for_non_terminal,
TransitiveClosureAddition {
item: item.substitute_production(id, key_map.keys_for(id)),
info,
info: item_info,
},
);
}
} else {
let mut item_info = info;
if syntax_grammar.production(prod_id).requires_eof_lookahead {
item_info.lookaheads = result.lookaheads.intern_ref(&eof_lookaheads);
item_info.reserved_lookaheads = ReservedWordSetId::default();
item_info.propagates_lookaheads = false;
item_info.contains_word = false;
}
find_or_push(
additions_for_non_terminal,
TransitiveClosureAddition { item, info },
TransitiveClosureAddition {
item,
info: item_info,
},
);
}
}

4
crates/generate/src/build_tables/minimize_parse_table.rs
View file

@ -159,6 +159,9 @@ impl Minimizer<'_> {
let mut unit_reduction_symbols_by_state = FxHashMap::default();
for (i, state) in self.parse_table.states.iter().enumerate() {
if state.has_eof_gated_reduce {
continue;
}
let mut only_unit_reductions = true;
let mut unit_reduction_symbol = None;
for (_, id) in &state.terminal_entries {
@ -431,6 +434,7 @@ impl Minimizer<'_> {
for state_id in &state_ids[1..] {
let other_parse_state = mem::take(&mut self.parse_table.states[*state_id as usize]);
parse_state.has_eof_gated_reduce |= other_parse_state.has_eof_gated_reduce;
parse_state
.terminal_entries
.extend(other_parse_state.terminal_entries);

7
crates/generate/src/dsl.js
View file

@ -33,6 +33,12 @@ function blank() {
};
}
function eof() {
return {
type: "EOF"
};
}
function field(name, rule) {
if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`);
@ -528,6 +534,7 @@ function getEnv(name) {
globalThis.alias = alias;
globalThis.blank = blank;
globalThis.eof = eof;
globalThis.choice = choice;
globalThis.optional = optional;
globalThis.prec = prec;

9
crates/generate/src/grammars.rs
View file

@ -225,12 +225,17 @@ impl std::fmt::Display for ReservedWordSetId {
}
}
/// A flattened production consisting of a step range and its dynamic precedence
/// A flattened production consisting of a step range, its dynamic precedence, and
/// whether its reduce action is gated on end-of-input lookahead.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct Production {
pub steps_start: u32,
pub steps_len: u32,
pub dynamic_precedence: i32,
/// True when the production was written ending in `eof()`. The reduce
/// action for this production is only emitted under the end-of-input
/// lookahead, never as a shift.
pub requires_eof_lookahead: bool,
}
impl Production {
@ -291,6 +296,7 @@ impl SyntaxGrammar {
ProdRef {
steps: &self.steps[p.step_range()],
dynamic_precedence: p.dynamic_precedence,
requires_eof_lookahead: p.requires_eof_lookahead,
}
}
@ -307,6 +313,7 @@ impl SyntaxGrammar {
pub struct ProdRef<'a> {
pub steps: &'a [ProductionStep],
pub dynamic_precedence: i32,
pub requires_eof_lookahead: bool,
}
impl ProdRef<'_> {

1
crates/generate/src/node_types.rs
View file

@ -2847,6 +2847,7 @@ mod tests {
steps_start,
steps_len: steps.len() as u32 - steps_start,
dynamic_precedence: 0,
requires_eof_lookahead: false,
});
}
var_prods.push((prod_start, productions.len() as u32));

2
crates/generate/src/parse_grammar.rs
View file

@ -81,6 +81,7 @@ enum RuleJSON {
context_name: String,
content: Box<Self>,
},
EOF,
}
#[derive(Deserialize)]
@ -462,6 +463,7 @@ impl RulePool {
let sid = self.intern(&value);
Ok(self.string(sid))
}
RuleJSON::EOF => Ok(self.eof()),
}
}
}

12
crates/generate/src/prepare_grammar.rs
View file

@ -32,9 +32,12 @@ use crate::{
pub use self::expand_tokens::expand_tokens;
use self::{
expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
intern_symbols::intern_symbols, process_inlines::process_inlines,
expand_repeats::{ExpandRepeatsError, expand_repeats},
extract_default_aliases::extract_default_aliases,
extract_tokens::extract_tokens,
flatten_grammar::flatten_grammar,
intern_symbols::intern_symbols,
process_inlines::process_inlines,
};
use super::{
Diagnostic,
@ -51,6 +54,7 @@ pub type PrepareGrammarResult<T> = Result<T, PrepareGrammarError>;
pub enum PrepareGrammarError {
ValidatePrecedences(#[from] ValidatePrecedenceError),
ValidateIndirectRecursion(#[from] IndirectRecursionError),
ExpandRepeats(#[from] ExpandRepeatsError),
InternSymbols(#[from] InternSymbolsError),
ExtractTokens(#[from] ExtractTokensError),
FlattenGrammar(#[from] FlattenGrammarError),
@ -116,7 +120,7 @@ pub fn prepare_grammar(
let interned_meta = intern_symbols(&mut g, diagnostics)?;
let mut ext_meta = extract_tokens(&mut g, &interned_meta)?;
expand_repeats(&mut g, &mut ext_meta);
expand_repeats(&mut g, &mut ext_meta)?;
let mut state = FlattenState::default();
let mut out = ProductionStore::default();

252
crates/generate/src/prepare_grammar/expand_repeats.rs
View file

@ -1,10 +1,12 @@
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
grammars::{InputGrammar, Variable, VariableType},
prepare_grammar::extract_tokens::ExtractedGrammarMeta,
rules::{Rule, RuleId, RulePool, Symbol},
strpool::StrId,
rules::{Rule, RuleId, RulePool, Symbol, SymbolType},
strpool::{StrId, StrPool},
};
#[derive(Default)]
@ -13,6 +15,7 @@ struct Expander {
aux: Vec<Variable>,
memo: FxHashMap<u64, Vec<(RuleId, Symbol)>>,
stack: Vec<Task>,
zero_width: ZeroWidth,
}
#[derive(Copy, Clone)]
@ -21,6 +24,10 @@ enum Task {
Expand { id: RuleId, content: RuleId },
}
#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
#[error("Rule `{0}` contains a repetition that can match the empty string at end of input")]
pub struct ExpandRepeatsError(pub String);
impl Expander {
/// Post-order repeat expansion over one root. Children expand first, and `Reserved`
/// nodes are not descended.
@ -30,7 +37,7 @@ impl Expander {
root: RuleId,
var_name: StrId,
aux_repeat_counter: &mut u32,
) {
) -> Result<(), ExpandRepeatsError> {
self.stack.clear();
self.stack.push(Task::Visit(root));
'walk: while let Some(task) = self.stack.pop() {
@ -53,6 +60,10 @@ impl Expander {
_ => {} // For primitive rules, don't change anything.
},
Task::Expand { id, content } => {
let width = self.zero_width.eval(pool, content);
if width.eof_nullable {
return Err(ExpandRepeatsError(pool.resolve(var_name).to_string()));
}
// For repetitions, introduce an auxiliary rule that contains the
// repeated content, but can also contain a recursive binary tree structure.
let hash = pool.subtree_hash(content);
@ -68,8 +79,10 @@ impl Expander {
let name = format!("{}_repeat{aux_repeat_counter}", pool.resolve(var_name));
let name = pool.intern(&name);
// Aux rules are appended after the original variables, so they occupy
// non-terminal indices `preceding..`.
// non-terminal indices `preceding..`. The aux stands for `Repeat(content)`,
// which matches zero width exactly when `content` does.
let symbol = Symbol::non_terminal(self.preceding + self.aux.len());
self.zero_width.push_variable(width);
self.memo.entry(hash).or_default().push((content, symbol));
let root = wrap_in_binary_tree(pool, symbol, content);
self.aux.push(Variable { name, root });
@ -77,6 +90,189 @@ impl Expander {
}
}
}
Ok(())
}
}
/// Whether a rule can match zero characters, with and without crossing an `eof()`.
#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
struct Width {
nullable: bool,
eof_nullable: bool,
}
impl Width {
/// Fold `other` in, reporting whether that set anything new to `true`.
fn merge(&mut self, other: Self) -> bool {
let before = *self;
self.nullable |= other.nullable;
self.eof_nullable |= other.eof_nullable;
*self != before
}
}
#[derive(Clone, Copy)]
enum Visit {
Enter(RuleId),
Exit(RuleId),
}
/// Zero width analysis over the rule pool.
///
/// `by_variable` is seeded by a fixpoint over the whole grammar before any expansion,
/// which makes results indepedent of the order in which rules are declared.
#[derive(Default)]
struct ZeroWidth {
by_variable: Vec<Width>,
stack: Vec<Visit>,
values: Vec<Width>,
}
impl ZeroWidth {
/// Seed `by_variable` with every rule's width, as a least fixpoint.
///
/// [`Self::eval`] reads a nonterminal's width from this table, but rules can
/// reference each other cyclically. Every rule starts at "matches nothing", and
/// all of them are re-evaluated against the current table until a full pass
/// changes nothing. Starting there prevents a cycle from marking itself. A rule
/// that reaches itself reads back "matches nothing", so flags are only set by
/// a path that truly matches zero width.
fn new(pool: &RulePool, variables: &[Variable]) -> Self {
let mut this = Self {
by_variable: vec![Width::default(); variables.len()],
stack: Vec::new(),
values: Vec::new(),
};
loop {
let mut changed = false;
for (i, v) in variables.iter().enumerate() {
let width = this.eval(pool, v.root);
changed |= this.by_variable[i].merge(width);
}
if !changed {
break;
}
}
this
}
/// Record the width of a newly created auxiliary rule.
fn push_variable(&mut self, width: Width) {
self.by_variable.push(width);
}
/// Post-order walk of the subtree at `root`, combining children into their parent.
fn eval(&mut self, pool: &RulePool, root: RuleId) -> Width {
self.stack.clear();
self.values.clear();
self.stack.push(Visit::Enter(root));
while let Some(step) = self.stack.pop() {
match step {
Visit::Enter(id) => match pool.node(id) {
Rule::Seq(range) | Rule::Choice(range) => {
self.stack.push(Visit::Exit(id));
for &c in pool.child_slice(range) {
self.stack.push(Visit::Enter(c));
}
}
Rule::Repeat(rule)
| Rule::Metadata { rule, .. }
| Rule::Reserved { rule, .. } => {
self.stack.push(Visit::Exit(id));
self.stack.push(Visit::Enter(rule));
}
Rule::Blank => self.values.push(Width {
nullable: true,
eof_nullable: false,
}),
Rule::String(s) => self.values.push(Width {
nullable: s == StrPool::EMPTY_STR_ID,
eof_nullable: false,
}),
Rule::Eof => self.values.push(Width {
nullable: false,
eof_nullable: true,
}),
Rule::Sym { kind, index } => {
let width = match kind {
SymbolType::End => Width {
nullable: false,
eof_nullable: true,
},
SymbolType::NonTerminal => self
.by_variable
.get(index as usize)
.copied()
.unwrap_or_default(),
// External scanners decide at runtime how far to advance
// and may return a zero width token, so this must count
// as nullable.
//
// A scanner may gate itself on `lexer->eof`, but there's
// no way to determine that here. Assuming so would reject
// every grammar that `repeat`s an external.
SymbolType::External => Width {
nullable: true,
eof_nullable: false,
},
// `expand_tokens` rejects tokens that match the empty string
SymbolType::Terminal => Width::default(),
// Lookahead marker that `build_parse_table` inserts for nonterminal
// extras _after_ this pass runs.
SymbolType::EndOfNonTerminalExtra => unreachable!(),
};
self.values.push(width);
}
// `extract_tokens` hoists every `Pattern` into the lexical grammar and
// leaves a terminal symbol in its place. Only syntactic (non
// lexical grammar) rules are walked here.
Rule::Pattern(..)
// `intern_symbols` resolves every `NamedSymbol` to a `Sym`
| Rule::NamedSymbol(_) => unreachable!(),
},
Visit::Exit(id) => match pool.node(id) {
Rule::Choice(range) => {
let base = self.values.len() - range.len as usize;
let mut width = Width::default();
for child in self.values.drain(base..) {
width.nullable |= child.nullable;
width.eof_nullable |= child.eof_nullable;
}
self.values.push(width);
}
Rule::Seq(range) => {
let base = self.values.len() - range.len as usize;
let mut nullable = true; // matches empty iff every element does
let mut any_eof = false; // matches eof if any element does
let mut all_zero_width = true; // every element matches empty or via `eof()`
for child in self.values.drain(base..) {
nullable &= child.nullable;
any_eof |= child.eof_nullable;
all_zero_width &= child.nullable || child.eof_nullable;
}
self.values.push(Width {
nullable,
eof_nullable: all_zero_width && any_eof,
});
}
// Each of these wraps a single child whose width is both already on the
// stack and exactly this (the wrapper) node's width:
//
// - `Repeat` is one or more, so it matches zero width exactly when its
// content does. Zero or more comes in as `Choice(Repeat, Blank)` from
// `parse_grammar`, so that blank gives the nullable case instead of here.
// - `Metadata` carries wrapping data that doesn't change how much input is
// matched. Its `token` forms are already replaced by terminals in
// `extrac_tokens`.
// - `Reserved` only names the reserved word set for a its child.
Rule::Repeat(_) | Rule::Metadata { .. } | Rule::Reserved { .. } => {}
// Every other rule is a leaf. `Enter` pushed its width directly and
// never queued an `Exit`.
_ => unreachable!(),
},
}
}
self.values.pop().unwrap_or_default()
}
}
@ -105,9 +301,13 @@ fn wrap_in_binary_tree(pool: &mut RulePool, symbol: Symbol, inner: RuleId) -> Ru
}
}
pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGrammarMeta) {
pub(super) fn expand_repeats(
grammar: &mut InputGrammar,
meta: &mut ExtractedGrammarMeta,
) -> Result<(), ExpandRepeatsError> {
let mut expander = Expander {
preceding: grammar.variables.len(),
zero_width: ZeroWidth::new(&grammar.pool, &grammar.variables),
..Default::default()
};
for i in 0..grammar.variables.len() {
@ -119,7 +319,14 @@ pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGra
if meta.kinds[i] == VariableType::Hidden
&& let Rule::Repeat(content) = grammar.pool.node(root)
{
expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count);
if expander
.zero_width
.eval(&grammar.pool, content)
.eof_nullable
{
return Err(ExpandRepeatsError(grammar.pool.resolve(name).to_string()));
}
expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count)?;
grammar.variables[i].root =
wrap_in_binary_tree(&mut grammar.pool, Symbol::non_terminal(i), content);
meta.kinds[i] = VariableType::Auxiliary;
@ -127,12 +334,13 @@ pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGra
continue;
}
expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count);
expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count)?;
}
for var in expander.aux {
grammar.variables.push(var);
meta.kinds.push(VariableType::Auxiliary);
}
Ok(())
}
#[cfg(test)]
@ -426,6 +634,34 @@ mod tests {
assert!(g.pool.subtree_eq(g.variables[1].root, e1));
}
#[test]
fn test_rejects_repeat_of_eof_helper_rule() {
// rule0: repeat(non_terminal(1)); rule1: eof()
let mut pool = RulePool::default();
let r0 = {
let n = non_term(&mut pool, 1);
pool.repeat(n)
};
let r1 = pool.push_node(Rule::Eof);
let (n0, n1) = (pool.intern("rule0"), pool.intern("rule1"));
let mut grammar = InputGrammar {
pool,
variables: vec![
Variable { name: n0, root: r0 },
Variable { name: n1, root: r1 },
],
..Default::default()
};
let mut meta = ExtractedGrammarMeta {
kinds: vec![VariableType::Named; 2],
..Default::default()
};
assert_eq!(
expand_repeats(&mut grammar, &mut meta).unwrap_err(),
ExpandRepeatsError("rule0".to_string())
);
}
fn term(p: &mut RulePool, i: u32) -> RuleId {
p.push_node(Rule::Sym {
kind: SymbolType::Terminal,
@ -453,7 +689,7 @@ mod tests {
kinds,
..Default::default()
};
expand_repeats(&mut grammar, &mut meta);
expand_repeats(&mut grammar, &mut meta).unwrap();
(grammar, meta)
}
}

7
crates/generate/src/prepare_grammar/expand_tokens.rs
View file

@ -162,6 +162,12 @@ pub enum ExpandRuleError {
UnexpectedSymbol(Symbol),
#[error("unexpected reserved-word context {0}")]
UnexpectedReserved(String),
#[error(
"`eof()` cannot be used inside a token. \
A lexical rule cannot check for end of input, \
so use `eof()` only at the end of a syntactic rule."
)]
UnexpectedEof,
#[error("{0}")]
Parse(String),
#[error(transparent)]
@ -297,6 +303,7 @@ impl NfaBuilder {
result
}
Rule::Blank => Ok(false),
Rule::Eof => Err(ExpandRuleError::UnexpectedEof)?,
Rule::Sym { kind, index } => {
Err(ExpandRuleError::UnexpectedSymbol(Symbol { kind, index }))?
}

5
crates/generate/src/prepare_grammar/extract_default_aliases.rs
View file

@ -72,7 +72,9 @@ pub(super) fn extract_default_aliases(
SymbolType::External => &mut external_status_list[symbol_index],
SymbolType::NonTerminal => &mut non_terminal_status_list[symbol_index],
SymbolType::Terminal => &mut terminal_status_list[symbol_index],
SymbolType::End | SymbolType::EndOfNonTerminalExtra => panic!("Unexpected end token"),
SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
panic!("Unexpected end token")
}
};
status.appears_unaliased = true;
}
@ -204,6 +206,7 @@ mod tests {
steps_start,
steps_len: steps.len() as u32,
dynamic_precedence: 0,
requires_eof_lookahead: false,
});
out.var_prods.push((ps, out.productions.len() as u32));
}

52
crates/generate/src/prepare_grammar/flatten_grammar.rs
View file

@ -31,6 +31,8 @@ unless they are used only as the grammar's start rule.
EmptyString(String),
#[error("Rule `{0}` cannot be inlined because it contains a reference to itself")]
RecursiveInline(String),
#[error("Rule `{0}` has no reachable productions.")]
NoReachableProductions(String),
}
#[derive(Clone, Copy, Default)]
@ -179,6 +181,11 @@ fn apply(
let child = pool.child_slice(range)[selected as usize];
apply(pool, reserved_ids, child, f_ctx, at_end, st)
}
Rule::Eof => {
let symbol = Symbol::end();
st.push_step(symbol.kind, symbol.index, f_ctx);
Ok(true)
}
Rule::Metadata { params, rule } => {
let params = pool.params(params);
let mut inner_ctx = f_ctx;
@ -227,11 +234,36 @@ fn apply(
}
/// Append the completed path as a production unless this variable already has an
/// identical one.
fn emit(st: &FlattenState, out: &mut ProductionStore, prod_start: u32) {
/// identical one. A path with `eof()` anywhere but the final step is discarded,
/// since such a production could never be completed.
fn emit(st: &mut FlattenState, out: &mut ProductionStore, prod_start: u32) -> bool {
let last = st.steps.len().saturating_sub(1);
let Some(eof_index) = st
.steps
.iter()
.position(|step| step.symbol() == Symbol::end())
else {
return emit_ready(st, out, prod_start, false);
};
if eof_index != last {
return false;
}
st.steps.pop();
emit_ready(st, out, prod_start, true)
}
fn emit_ready(
st: &FlattenState,
out: &mut ProductionStore,
prod_start: u32,
requires_eof_lookahead: bool,
) -> bool {
for p in &out.productions[prod_start as usize..] {
if p.dynamic_precedence == st.dyn_prec && out.steps[p.step_range()] == st.steps[..] {
return;
if p.dynamic_precedence == st.dyn_prec
&& p.requires_eof_lookahead == requires_eof_lookahead
&& out.steps[p.step_range()] == st.steps[..]
{
return true;
}
}
let steps_start = out.steps.len() as u32;
@ -240,7 +272,9 @@ fn emit(st: &FlattenState, out: &mut ProductionStore, prod_start: u32) {
steps_start,
steps_len: st.steps.len() as u32,
dynamic_precedence: st.dyn_prec,
requires_eof_lookahead,
});
true
}
pub(super) fn flatten_grammar(
@ -263,6 +297,7 @@ pub(super) fn flatten_grammar(
.collect();
for v in &g.variables {
let prod_start = out.productions.len() as u32;
let mut dropped_for_eof = false;
st.reset_variable();
loop {
apply(
@ -274,13 +309,18 @@ pub(super) fn flatten_grammar(
st,
)?;
if !st.dead {
emit(st, out, prod_start);
dropped_for_eof |= !emit(st, out, prod_start);
}
if !st.choices.advance() {
break;
}
st.reset_path();
}
if dropped_for_eof && prod_start == out.productions.len() as u32 {
return Err(FlattenGrammarError::NoReachableProductions(
g.pool.resolve(v.name).to_string(),
));
}
out.var_prods
.push((prod_start, out.productions.len() as u32));
}
@ -298,7 +338,7 @@ fn check(
let used = out.steps.iter().any(|s| s.symbol() == symbol);
let inlined = meta.inline.contains(&symbol);
for p in &out.productions[p_start as usize..p_end as usize] {
if used && p.steps_len == 0 {
if used && p.steps_len == 0 && !p.requires_eof_lookahead {
Err(FlattenGrammarError::EmptyString(
g.pool.resolve(g.variables[i].name).to_string(),
))?;

99
crates/generate/src/prepare_grammar/process_inlines.rs
View file

@ -1,6 +1,6 @@
use std::hash::{Hash as _, Hasher as _};
use rustc_hash::{FxHashMap, FxHasher};
use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
use serde::{Deserialize, Serialize};
use thiserror::Error;
@ -22,12 +22,17 @@ struct InlineBuilder<'a> {
struct ScratchProd {
steps: Vec<ProductionStep>,
dynamic_precedence: i32,
requires_eof_lookahead: bool,
}
impl InlineBuilder<'_> {
fn build(mut self) -> InlinedProductionMap {
/// Returns the inline map along with the ids of productions whose every expansion
/// was dropped.
fn build(mut self) -> (InlinedProductionMap, FxHashSet<u32>) {
let mut dead = FxHashSet::default();
let mut worklist = Vec::new();
for prod_id in 0..self.first_inlined {
let mut survived = false;
worklist.push((prod_id, 0u32));
while !worklist.is_empty() {
let mut i = 0;
@ -43,12 +48,16 @@ impl InlineBuilder<'_> {
}
} else {
worklist.remove(i);
survived = true;
}
}
}
if !survived {
dead.insert(prod_id);
}
}
InlinedProductionMap { map: self.map }
(InlinedProductionMap { map: self.map }, dead)
}
/// Expand inlining at one step of one production, dedup, and store the results
@ -62,6 +71,7 @@ impl InlineBuilder<'_> {
let mut scratch = vec![ScratchProd {
steps: self.out.steps[src.step_range()].to_vec(),
dynamic_precedence: src.dynamic_precedence,
requires_eof_lookahead: src.requires_eof_lookahead,
}];
let mut i = 0;
while i < scratch.len() {
@ -77,8 +87,11 @@ impl InlineBuilder<'_> {
let removed_step = removed_prod.steps[si];
let (v_start, v_end) = self.out.var_prods[symbol.index as usize];
let replacements = (v_start..v_end)
.map(|p_idx| {
.filter_map(|p_idx| {
let p = self.out.productions[p_idx as usize];
if p.requires_eof_lookahead && si + 1 < removed_prod.steps.len() {
return None;
}
let mut production = removed_prod.clone();
production
.steps
@ -105,7 +118,8 @@ impl InlineBuilder<'_> {
if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() {
production.dynamic_precedence = p.dynamic_precedence;
}
production
production.requires_eof_lookahead |= p.requires_eof_lookahead;
Some(production)
})
.collect::<Vec<_>>();
scratch.splice(i..=i, replacements);
@ -115,11 +129,13 @@ impl InlineBuilder<'_> {
for sp in scratch {
let mut hasher = FxHasher::default();
sp.dynamic_precedence.hash(&mut hasher);
sp.requires_eof_lookahead.hash(&mut hasher);
sp.steps.hash(&mut hasher);
let candidates = self.memo.entry(hasher.finish()).or_default();
let existing = candidates.iter().copied().find(|&id| {
let p = self.out.productions[id as usize];
p.dynamic_precedence == sp.dynamic_precedence
&& p.requires_eof_lookahead == sp.requires_eof_lookahead
&& self.out.steps[p.step_range()] == sp.steps
});
result.push(existing.unwrap_or_else(|| {
@ -129,6 +145,7 @@ impl InlineBuilder<'_> {
steps_start,
steps_len: sp.steps.len() as u32,
dynamic_precedence: sp.dynamic_precedence,
requires_eof_lookahead: sp.requires_eof_lookahead,
});
let id = (self.out.productions.len() - 1) as u32;
candidates.push(id);
@ -156,6 +173,8 @@ pub enum ProcessInlinesError {
Token(String),
#[error("Rule `{0}` cannot be inlined because it is the first rule")]
FirstRule(String),
#[error("Rule `{0}` has no reachable productions after inlining")]
NoReachableProductions(String),
}
pub(super) fn process_inlines(
@ -185,14 +204,30 @@ pub(super) fn process_inlines(
}
}
Ok(InlineBuilder {
let (map, dead) = InlineBuilder {
first_inlined: out.productions.len() as u32,
out,
out: &mut *out,
inline: &meta.inline,
map: FxHashMap::default(),
memo: FxHashMap::default(),
}
.build())
.build();
if !dead.is_empty() {
for (i, &(p_start, p_end)) in out.var_prods.iter().enumerate() {
if p_start == p_end || !(p_start..p_end).all(|p| dead.contains(&p)) {
continue;
}
let symbol = Symbol::non_terminal(i);
if i == 0 || out.steps.iter().any(|s| s.symbol() == symbol) {
Err(ProcessInlinesError::NoReachableProductions(
g.pool.resolve(g.variables[i].name).to_string(),
))?;
}
}
}
Ok(map)
}
#[cfg(test)]
@ -518,6 +553,7 @@ mod tests {
steps_start,
steps_len: steps.len() as u32,
dynamic_precedence: *dynamic_prc,
requires_eof_lookahead: false,
});
}
out.var_prods.push((start, out.productions.len() as u32));
@ -546,6 +582,53 @@ mod tests {
})
}
#[test]
fn test_error_when_inlining_removes_all_productions() {
// var0: [nt1]
// var1: [nt2, t10] (nt2 is inlined)
// var2: one empty eof-gated production, like a flattened `_eof: _ => eof()`
// Inlining var2 into var1 drops var1's only production.
let mut out = ProductionStore::default();
add_variable(&mut out, &[(vec![plain(Symbol::non_terminal(1))], 0)]);
add_variable(
&mut out,
&[(
vec![plain(Symbol::non_terminal(2)), plain(Symbol::terminal(10))],
0,
)],
);
let start = out.productions.len() as u32;
out.productions.push(Production {
steps_start: out.steps.len() as u32,
steps_len: 0,
dynamic_precedence: 0,
requires_eof_lookahead: true,
});
out.var_prods.push((start, start + 1));
let mut pool = RulePool::default();
let variables = ["rule0", "rule1", "rule2"]
.map(|name| {
let name = pool.intern(name);
let root = pool.push_node(crate::rules::Rule::Blank);
crate::grammars::Variable { name, root }
})
.to_vec();
let g = InputGrammar {
pool,
variables,
..Default::default()
};
let meta = ExtractedGrammarMeta {
inline: vec![Symbol::non_terminal(2)],
..Default::default()
};
assert_eq!(
process_inlines(&g, &meta, &mut out).unwrap_err(),
ProcessInlinesError::NoReachableProductions("rule1".to_string())
);
}
fn plain(symbol: Symbol) -> ProductionStep {
ProductionStep::pack(symbol, Precedence::None, None, None, None, 0)
}

15
crates/generate/src/rules.rs
View file

@ -141,6 +141,7 @@ pub enum Rule {
Seq(RuleIdRange),
Choice(RuleIdRange),
Repeat(RuleId),
Eof,
Metadata { params: ParamsId, rule: RuleId },
Reserved { rule: RuleId, ctx: StrId },
}
@ -299,6 +300,10 @@ impl RulePool {
self.push_node(Rule::Repeat(content))
}
pub fn eof(&mut self) -> RuleId {
self.push_node(Rule::Eof)
}
pub fn seq(&mut self, ids: &[RuleId]) -> RuleId {
let range = self.push_children(ids);
self.push_node(Rule::Seq(range))
@ -353,7 +358,7 @@ impl RulePool {
let node = self.node(id);
std::mem::discriminant(&node).hash(&mut hasher);
match node {
Rule::Blank => {}
Rule::Blank | Rule::Eof => {}
Rule::String(s) | Rule::NamedSymbol(s) => s.hash(&mut hasher),
Rule::Pattern(p, f) => {
p.hash(&mut hasher);
@ -390,7 +395,7 @@ impl RulePool {
let mut stack = vec![(a, b)];
while let Some((a, b)) = stack.pop() {
match (self.node(a), self.node(b)) {
(Rule::Blank, Rule::Blank) => {}
(Rule::Blank, Rule::Blank) | (Rule::Eof, Rule::Eof) => {}
(Rule::String(x), Rule::String(y))
| (Rule::NamedSymbol(x), Rule::NamedSymbol(y)) => {
if x != y {
@ -476,7 +481,9 @@ impl RulePool {
self.rule_is_referenced(rule, target, is_external)
}
Rule::Repeat(inner) => self.rule_is_referenced(inner, target, false),
Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => false,
Rule::Blank | Rule::Eof | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {
false
}
}
}
@ -499,7 +506,7 @@ impl RulePool {
self.collect_referenced_ids(rule, skip_top_level, out);
}
Rule::Repeat(inner) => self.collect_referenced_ids(inner, false, out),
Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {}
Rule::Blank | Rule::Eof | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {}
}
}
}

2
crates/generate/src/tables.rs
View file

@ -238,6 +238,7 @@ impl ActionListPool {
lex_state_id: state.lex_state_id,
external_lex_state_id: state.external_lex_state_id,
core_id: state.core_id,
has_eof_gated_reduce: state.has_eof_gated_reduce,
}
})
.collect();
@ -293,6 +294,7 @@ pub struct ParseState<T = ActionListId> {
pub lex_state_id: LexStateId,
pub external_lex_state_id: LexStateId,
pub core_id: u32,
pub has_eof_gated_reduce: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]

12
docs/src/assets/schemas/grammar.schema.json
View file

@ -129,6 +129,17 @@
"required": ["type"]
},
"eof-rule": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "EOF"
}
},
"required": ["type"]
},
"string-rule": {
"type": "object",
"properties": {
@ -321,6 +332,7 @@
"oneOf": [
{ "$ref": "#/definitions/alias-rule" },
{ "$ref": "#/definitions/blank-rule" },
{ "$ref": "#/definitions/eof-rule" },
{ "$ref": "#/definitions/string-rule" },
{ "$ref": "#/definitions/pattern-rule" },
{ "$ref": "#/definitions/symbol-rule" },

5
docs/src/creating-parsers/2-the-grammar-dsl.md
View file

@ -101,6 +101,11 @@ rule. In the resulting syntax tree, you can then use that field name to access s
one passed into the `wordset` parameter. This is useful for contextual keywords, such as `if` in JavaScript, which cannot
be used as a variable name in most contexts, but can be used as a property name.
- **End of Input : `eof()`** — This function creates a rule that matches the end of the input, without consuming any
characters. It may only appear as the final symbol of a rule, and is useful when a rule should match either an explicit
terminator (such as a newline) or the end of the file. Choice branches where other symbols follow `eof()` can never
match, so they are dropped, and `eof()` is not allowed inside `token()`.
In addition to the `name` and `rules` fields, grammars have a few other optional public fields that influence the behavior
of the parser. Each of these fields is a function that accepts the grammar object (`$`) as its only parameter, like the
grammar rules themselves. These fields are:

10
test/fixtures/test_grammars/duplicate_reduction_precedence/corpus.txt vendored Normal file
View file

@ -0,0 +1,10 @@
======================
higher precedence wins
======================
x
---
(start
(a
(x)))

13
test/fixtures/test_grammars/duplicate_reduction_precedence/grammar.js vendored Normal file
View file

@ -0,0 +1,13 @@
// `a` has two productions that reduce identically and differ only in precedence.
// The higher one has to outrank `c`, which means the duplicate action still has
// to be compared against rather than skipped.
export default grammar({
name: 'duplicate_reduction_precedence',
rules: {
start: $ => choice($.a, $.c),
a: $ => choice($.x, prec(2, $.x)),
c: $ => $.x,
x: _ => 'x',
}
})

34
test/fixtures/test_grammars/eof_basic/corpus.txt vendored Normal file
View file

@ -0,0 +1,34 @@
============
trailing newline
============
text
text
---
(source_file
(line)
(line))
============
no trailing newline
============
text
text
---
(source_file
(line)
(line))
============
single line, no newline
============
text
---
(source_file
(line))

12
test/fixtures/test_grammars/eof_basic/grammar.js vendored Normal file
View file

@ -0,0 +1,12 @@
export default grammar({
name: 'eof_basic',
rules: {
source_file: $ => repeat($.line),
line: $ => choice(
seq('text', '\n'),
seq('text', eof()),
),
}
});

46
test/fixtures/test_grammars/eof_dropped_branch/corpus.txt vendored Normal file
View file

@ -0,0 +1,46 @@
============
no items
============
()
---
(source_file)
============
items separated by newlines
============
(x
x
x)
---
(source_file (item) (item) (item))
============
items separated by semicolons
============
(x;x;x)
---
(source_file (item) (item) (item))
============
trailing separator
============
(x;)
---
(source_file (item))
============
eof in an inline before a later symbol is unreachable
============
ok
---
(source_file (inline_test))

30
test/fixtures/test_grammars/eof_dropped_branch/grammar.js vendored Normal file
View file

@ -0,0 +1,30 @@
// The `eof()` alternative of `terminator` is unreachable inside the parens
// (because `)` always has to come after) so the generator silently drops
// that production. The grammar still parses because the newline and `;`
// alternatives remain.
const terminator = choice('\n', ';', eof());
export default grammar({
name: 'eof_dropped_branch',
inline: $ => [$._inline_eof],
rules: {
source_file: $ => choice(
seq(
'(',
optional(seq($.item, repeat(seq(terminator, $.item)), optional(terminator))),
')',
),
$.inline_test,
),
item: _ => 'x',
// Inlining the second alternative would put `bad` after EOF, so that
// alternative must be discarded while the reachable one remains.
inline_test: $ => choice('ok', seq($._inline_eof, 'bad')),
_inline_eof: _ => eof(),
}
});

20
test/fixtures/test_grammars/eof_duplicate_reduction/corpus.txt vendored Normal file
View file

@ -0,0 +1,20 @@
===========================
single line at end of input
===========================
text
---
(source_file
(line))
=============
several lines
=============
texttext
---
(source_file
(line)
(line))

11
test/fixtures/test_grammars/eof_duplicate_reduction/grammar.js vendored Normal file
View file

@ -0,0 +1,11 @@
// Both branches reduce `line` to the same single `'text'` child, differing only in
// whether the reduce is gated on end of input. Identical actions are not a conflict,
// but they still have to be compared on precedence rather than skipped.
export default grammar({
name: 'eof_duplicate_reduction',
rules: {
source_file: $ => repeat($.line),
line: _ => choice('text', seq('text', eof())),
}
});

1
test/fixtures/test_grammars/eof_misplaced/expected_error.txt vendored Normal file
View file

@ -0,0 +1 @@
Rule `source_file` has no reachable productions.

7
test/fixtures/test_grammars/eof_misplaced/grammar.js vendored Normal file
View file

@ -0,0 +1,7 @@
export default grammar({
name: 'eof_misplaced',
rules: {
source_file: $ => seq(eof(), 'after'),
}
});

18
test/fixtures/test_grammars/eof_repeat_terminated/corpus.txt vendored Normal file
View file

@ -0,0 +1,18 @@
============
single item at end of input
============
text
---
(source_file)
============
item not at end of input
============
texttext
---
(source_file
(ERROR))

10
test/fixtures/test_grammars/eof_repeat_terminated/grammar.js vendored Normal file
View file

@ -0,0 +1,10 @@
// A repeat item ending in `eof()` can match at most once. The EOF-gated
// reduce must survive table minimization, so a second `text` is a parse
// error rather than being silently accepted.
export default grammar({
name: 'eof_repeat_terminated',
rules: {
source_file: $ => repeat(seq('text', eof())),
}
});

1
test/fixtures/test_grammars/eof_repeat_via_nullable_rule/expected_error.txt vendored Normal file
View file

@ -0,0 +1 @@
Rule `body` contains a repetition that can match the empty string at end of input

11
test/fixtures/test_grammars/eof_repeat_via_nullable_rule/grammar.js vendored Normal file
View file

@ -0,0 +1,11 @@
// `n` matches the empty string, so `seq($.n, eof())` can match nothing at all at
// the end of input and the outer `repeat` would spin.
export default grammar({
name: 'eof_repeat_via_nullable_rule',
rules: {
start: $ => $.body,
body: $ => repeat1(seq($.n, eof())),
n: _ => optional('a'),
}
})

23
test/fixtures/test_grammars/eof_repeat_via_rule/corpus.txt vendored Normal file
View file

@ -0,0 +1,23 @@
========================
one item at end of input
========================
aaa
---
(start
(body
(b)))
========================
item not at end of input
========================
aab
---
(start
(body
(b))
(ERROR
(UNEXPECTED 'b')))

11
test/fixtures/test_grammars/eof_repeat_via_rule/grammar.js vendored Normal file
View file

@ -0,0 +1,11 @@
// The repeated content ends in `eof()`, so it can match at most once, but it
// still has to consume some input to get there.
export default grammar({
name: 'eof_repeat_via_rule',
rules: {
start: $ => $.body,
body: $ => repeat1(seq($.b, eof())),
b: _ => repeat1('a'),
}
})