diff --git a/crates/cli/npm/dsl.d.ts b/crates/cli/npm/dsl.d.ts index accdb95ff..3dac495f0 100644 --- a/crates/cli/npm/dsl.d.ts +++ b/crates/cli/npm/dsl.d.ts @@ -15,6 +15,8 @@ type SeqRule = { type: 'SEQ'; members: Rule[] }; type StringRule = { type: 'STRING'; value: string }; type SymbolRule = { type: 'SYMBOL'; name: Name }; type TokenRule = { type: 'TOKEN'; content: Rule }; +type EOFRule = { type: 'EOF' }; + type Rule = | AliasRule @@ -33,7 +35,8 @@ type Rule = | SeqRule | StringRule | SymbolRule - | TokenRule; + | TokenRule + | EOFRule; declare class RustRegex { value: string; @@ -382,6 +385,19 @@ declare const token: { immediate(rule: RuleOrLiteral): ImmediateTokenRule; }; +/** + * Matches the end of input. May only appear as the final symbol of a + * (possibly nested) sequence; a production ending in `eof()` reduces only + * when the lookahead is end-of-input, rather than shifting a token. + * + * Choice branches that continue past `eof()` are dropped as unreachable, + * and `eof()` is not allowed inside `token()`. + * + * Useful when a rule should match either an explicit terminator (e.g. a + * newline) or the end of the file. + */ +declare function eof(): EOFRule; + /** * Creates a new language grammar with the provided schema. * diff --git a/crates/generate/src/build_tables/build_parse_table.rs b/crates/generate/src/build_tables/build_parse_table.rs index ff59fd1c0..20617627f 100644 --- a/crates/generate/src/build_tables/build_parse_table.rs +++ b/crates/generate/src/build_tables/build_parse_table.rs @@ -115,6 +115,7 @@ pub struct Interpretation { pub conflicting_lookahead: String, pub precedence: Option, pub associativity: Option, + pub requires_eof_lookahead: bool, } #[derive(Debug, Serialize, Deserialize)] @@ -142,16 +143,21 @@ impl std::fmt::Display for ConflictError { .iter() .map(|i| { let line = i.to_string(); - let prec_line = if let (Some(precedence), Some(associativity)) = - (&i.precedence, &i.associativity) - { - Some(format!( + let mut annotations = Vec::new(); + if let (Some(precedence), Some(associativity)) = (&i.precedence, &i.associativity) { + annotations.push(format!( "(precedence: {precedence}, associativity: {associativity})", - )) + )); + } else if let Some(precedence) = &i.precedence { + annotations.push(format!("(precedence: {precedence})")); + } + if i.requires_eof_lookahead { + annotations.push("(reduces only at end of input)".to_string()); + } + let prec_line = if annotations.is_empty() { + None } else { - i.precedence - .as_ref() - .map(|precedence| format!("(precedence: {precedence})")) + Some(annotations.join(" ")) }; (line, prec_line) @@ -430,6 +436,7 @@ impl<'a> ParseTableBuilder<'a> { nonterminal_entries: IndexMap::default(), reserved_words: TokenSet::default(), core_id, + has_eof_gated_reduce: false, }); self.parse_state_queue.push_back(ParseStateQueueEntry { state_id, @@ -549,7 +556,15 @@ impl<'a> ParseTableBuilder<'a> { let precedence = item.precedence(self.syntax_grammar); let associativity = item.associativity(self.syntax_grammar); + if item.production(self.syntax_grammar).requires_eof_lookahead { + self.parse_table.states[state_id as usize].has_eof_gated_reduce = true; + } for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() { + if item.production(self.syntax_grammar).requires_eof_lookahead + && lookahead != Symbol::end() + { + continue; + } let table_entry = self.parse_table.states[state_id as usize] .terminal_entries .entry(lookahead) @@ -575,9 +590,15 @@ impl<'a> ParseTableBuilder<'a> { lookaheads_with_conflicts.remove(lookahead); *reduction_info = ReductionInfo::default(); } + // Two items that reduce identically build the same tree, so + // there is nothing for the user to resolve. Precedence is + // still compared first, because an item that only repeats an + // existing action can still outrank it and clear the entry. Ordering::Equal => { - table_entry.actions.push(action); - lookaheads_with_conflicts.insert(lookahead); + if !table_entry.actions.contains(&action) { + table_entry.actions.push(action); + lookaheads_with_conflicts.insert(lookahead); + } } Ordering::Less => continue, } @@ -997,6 +1018,9 @@ impl<'a> ParseTableBuilder<'a> { conflicting_lookahead: self.symbol_name(conflicting_lookahead), precedence, associativity, + requires_eof_lookahead: item + .production(self.syntax_grammar) + .requires_eof_lookahead, } }) .collect::>(); diff --git a/crates/generate/src/build_tables/item.rs b/crates/generate/src/build_tables/item.rs index f8e3494b9..9dc8d3efe 100644 --- a/crates/generate/src/build_tables/item.rs +++ b/crates/generate/src/build_tables/item.rs @@ -28,17 +28,19 @@ const fn start_production() -> ProdRef<'static> { ProdRef { steps: &START_STEPS, dynamic_precedence: 0, + requires_eof_lookahead: false, } } /// Precomputed identity keys for one `(production, dot)` pair. /// /// `cmp` is the rank of the content tuple `Ord` compared (dynamic precedence, -/// length, precedence/associativity at the dot, then completed steps' aliases and fields -/// and remaining steps in full). Equal ranks hold _exactly_ when the tuple is equal, -/// so it doubles as the equality class for items without preceding inherited fields. -/// `eq_with_syms` subdivides `cmp` by the completed steps' symbols, which participate -/// in equality only when `has_preceding_inherited_fields` is set. +/// `requires_eof_lookahead`, length, precedence/associativity at the dot, then +/// completed steps' aliases and fields and remaining steps in full). Equal ranks +/// hold _exactly_ when the tuple is equal, so it doubles as the equality class +/// for items without preceding inherited fields. `eq_with_syms` subdivides `cmp` +/// by the completed steps' symbols, which participate in equality only when +/// `has_preceding_inherited_fields` is set. #[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] pub struct DotKeys { pub cmp: u32, @@ -198,6 +200,11 @@ impl Ord for ItemContent<'_> { self.production .dynamic_precedence .cmp(&other.production.dynamic_precedence) + .then_with(|| { + self.production + .requires_eof_lookahead + .cmp(&other.production.requires_eof_lookahead) + }) .then_with(|| { self.production .steps diff --git a/crates/generate/src/build_tables/item_set_builder.rs b/crates/generate/src/build_tables/item_set_builder.rs index 193e4d8de..f7b3ea801 100644 --- a/crates/generate/src/build_tables/item_set_builder.rs +++ b/crates/generate/src/build_tables/item_set_builder.rs @@ -193,6 +193,8 @@ impl<'a> ParseItemSetBuilder<'a> { // // Rather than computing these additions recursively, we use an explicit stack. let empty_lookaheads = TokenSet::new(); + let mut eof_lookaheads = TokenSet::new(); + eof_lookaheads.insert(Symbol::end()); let mut stack = Vec::new(); let mut follow_set_info_by_non_terminal = FxHashMap::::default(); for i in 0..syntax_grammar.variables.len() { @@ -230,6 +232,13 @@ impl<'a> ParseItemSetBuilder<'a> { result.reserved_first_sets[&next_step.symbol()], false, )); + } else if production.requires_eof_lookahead { + stack.push(( + symbol.index as usize, + &eof_lookaheads, + ReservedWordSetId::default(), + false, + )); } else { stack.push(( symbol.index as usize, @@ -269,18 +278,36 @@ impl<'a> ParseItemSetBuilder<'a> { if let Some(ids) = inlines.inlined_prod_ids(item.prod_id, item.step_index) { for &id in ids { + let mut item_info = info; + if syntax_grammar.production(id).requires_eof_lookahead { + item_info.lookaheads = + result.lookaheads.intern_ref(&eof_lookaheads); + item_info.reserved_lookaheads = ReservedWordSetId::default(); + item_info.propagates_lookaheads = false; + item_info.contains_word = false; + } find_or_push( additions_for_non_terminal, TransitiveClosureAddition { item: item.substitute_production(id, key_map.keys_for(id)), - info, + info: item_info, }, ); } } else { + let mut item_info = info; + if syntax_grammar.production(prod_id).requires_eof_lookahead { + item_info.lookaheads = result.lookaheads.intern_ref(&eof_lookaheads); + item_info.reserved_lookaheads = ReservedWordSetId::default(); + item_info.propagates_lookaheads = false; + item_info.contains_word = false; + } find_or_push( additions_for_non_terminal, - TransitiveClosureAddition { item, info }, + TransitiveClosureAddition { + item, + info: item_info, + }, ); } } diff --git a/crates/generate/src/build_tables/minimize_parse_table.rs b/crates/generate/src/build_tables/minimize_parse_table.rs index 7011e550c..924a72775 100644 --- a/crates/generate/src/build_tables/minimize_parse_table.rs +++ b/crates/generate/src/build_tables/minimize_parse_table.rs @@ -159,6 +159,9 @@ impl Minimizer<'_> { let mut unit_reduction_symbols_by_state = FxHashMap::default(); for (i, state) in self.parse_table.states.iter().enumerate() { + if state.has_eof_gated_reduce { + continue; + } let mut only_unit_reductions = true; let mut unit_reduction_symbol = None; for (_, id) in &state.terminal_entries { @@ -431,6 +434,7 @@ impl Minimizer<'_> { for state_id in &state_ids[1..] { let other_parse_state = mem::take(&mut self.parse_table.states[*state_id as usize]); + parse_state.has_eof_gated_reduce |= other_parse_state.has_eof_gated_reduce; parse_state .terminal_entries .extend(other_parse_state.terminal_entries); diff --git a/crates/generate/src/dsl.js b/crates/generate/src/dsl.js index dcaa4127f..10cd7ea96 100644 --- a/crates/generate/src/dsl.js +++ b/crates/generate/src/dsl.js @@ -33,6 +33,12 @@ function blank() { }; } +function eof() { + return { + type: "EOF" + }; +} + function field(name, rule) { if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`); @@ -528,6 +534,7 @@ function getEnv(name) { globalThis.alias = alias; globalThis.blank = blank; +globalThis.eof = eof; globalThis.choice = choice; globalThis.optional = optional; globalThis.prec = prec; diff --git a/crates/generate/src/grammars.rs b/crates/generate/src/grammars.rs index be73cd3af..0c42ab58c 100644 --- a/crates/generate/src/grammars.rs +++ b/crates/generate/src/grammars.rs @@ -225,12 +225,17 @@ impl std::fmt::Display for ReservedWordSetId { } } -/// A flattened production consisting of a step range and its dynamic precedence +/// A flattened production consisting of a step range, its dynamic precedence, and +/// whether its reduce action is gated on end-of-input lookahead. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Production { pub steps_start: u32, pub steps_len: u32, pub dynamic_precedence: i32, + /// True when the production was written ending in `eof()`. The reduce + /// action for this production is only emitted under the end-of-input + /// lookahead, never as a shift. + pub requires_eof_lookahead: bool, } impl Production { @@ -291,6 +296,7 @@ impl SyntaxGrammar { ProdRef { steps: &self.steps[p.step_range()], dynamic_precedence: p.dynamic_precedence, + requires_eof_lookahead: p.requires_eof_lookahead, } } @@ -307,6 +313,7 @@ impl SyntaxGrammar { pub struct ProdRef<'a> { pub steps: &'a [ProductionStep], pub dynamic_precedence: i32, + pub requires_eof_lookahead: bool, } impl ProdRef<'_> { diff --git a/crates/generate/src/node_types.rs b/crates/generate/src/node_types.rs index a103d1299..0c776fa05 100644 --- a/crates/generate/src/node_types.rs +++ b/crates/generate/src/node_types.rs @@ -2847,6 +2847,7 @@ mod tests { steps_start, steps_len: steps.len() as u32 - steps_start, dynamic_precedence: 0, + requires_eof_lookahead: false, }); } var_prods.push((prod_start, productions.len() as u32)); diff --git a/crates/generate/src/parse_grammar.rs b/crates/generate/src/parse_grammar.rs index e9805e76b..ed163d51c 100644 --- a/crates/generate/src/parse_grammar.rs +++ b/crates/generate/src/parse_grammar.rs @@ -81,6 +81,7 @@ enum RuleJSON { context_name: String, content: Box, }, + EOF, } #[derive(Deserialize)] @@ -462,6 +463,7 @@ impl RulePool { let sid = self.intern(&value); Ok(self.string(sid)) } + RuleJSON::EOF => Ok(self.eof()), } } } diff --git a/crates/generate/src/prepare_grammar.rs b/crates/generate/src/prepare_grammar.rs index 663f44df5..e940b5730 100644 --- a/crates/generate/src/prepare_grammar.rs +++ b/crates/generate/src/prepare_grammar.rs @@ -32,9 +32,12 @@ use crate::{ pub use self::expand_tokens::expand_tokens; use self::{ - expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases, - extract_tokens::extract_tokens, flatten_grammar::flatten_grammar, - intern_symbols::intern_symbols, process_inlines::process_inlines, + expand_repeats::{ExpandRepeatsError, expand_repeats}, + extract_default_aliases::extract_default_aliases, + extract_tokens::extract_tokens, + flatten_grammar::flatten_grammar, + intern_symbols::intern_symbols, + process_inlines::process_inlines, }; use super::{ Diagnostic, @@ -51,6 +54,7 @@ pub type PrepareGrammarResult = Result; pub enum PrepareGrammarError { ValidatePrecedences(#[from] ValidatePrecedenceError), ValidateIndirectRecursion(#[from] IndirectRecursionError), + ExpandRepeats(#[from] ExpandRepeatsError), InternSymbols(#[from] InternSymbolsError), ExtractTokens(#[from] ExtractTokensError), FlattenGrammar(#[from] FlattenGrammarError), @@ -116,7 +120,7 @@ pub fn prepare_grammar( let interned_meta = intern_symbols(&mut g, diagnostics)?; let mut ext_meta = extract_tokens(&mut g, &interned_meta)?; - expand_repeats(&mut g, &mut ext_meta); + expand_repeats(&mut g, &mut ext_meta)?; let mut state = FlattenState::default(); let mut out = ProductionStore::default(); diff --git a/crates/generate/src/prepare_grammar/expand_repeats.rs b/crates/generate/src/prepare_grammar/expand_repeats.rs index 7a8d69d79..2f732d84c 100644 --- a/crates/generate/src/prepare_grammar/expand_repeats.rs +++ b/crates/generate/src/prepare_grammar/expand_repeats.rs @@ -1,10 +1,12 @@ use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use thiserror::Error; use crate::{ grammars::{InputGrammar, Variable, VariableType}, prepare_grammar::extract_tokens::ExtractedGrammarMeta, - rules::{Rule, RuleId, RulePool, Symbol}, - strpool::StrId, + rules::{Rule, RuleId, RulePool, Symbol, SymbolType}, + strpool::{StrId, StrPool}, }; #[derive(Default)] @@ -13,6 +15,7 @@ struct Expander { aux: Vec, memo: FxHashMap>, stack: Vec, + zero_width: ZeroWidth, } #[derive(Copy, Clone)] @@ -21,6 +24,10 @@ enum Task { Expand { id: RuleId, content: RuleId }, } +#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)] +#[error("Rule `{0}` contains a repetition that can match the empty string at end of input")] +pub struct ExpandRepeatsError(pub String); + impl Expander { /// Post-order repeat expansion over one root. Children expand first, and `Reserved` /// nodes are not descended. @@ -30,7 +37,7 @@ impl Expander { root: RuleId, var_name: StrId, aux_repeat_counter: &mut u32, - ) { + ) -> Result<(), ExpandRepeatsError> { self.stack.clear(); self.stack.push(Task::Visit(root)); 'walk: while let Some(task) = self.stack.pop() { @@ -53,6 +60,10 @@ impl Expander { _ => {} // For primitive rules, don't change anything. }, Task::Expand { id, content } => { + let width = self.zero_width.eval(pool, content); + if width.eof_nullable { + return Err(ExpandRepeatsError(pool.resolve(var_name).to_string())); + } // For repetitions, introduce an auxiliary rule that contains the // repeated content, but can also contain a recursive binary tree structure. let hash = pool.subtree_hash(content); @@ -68,8 +79,10 @@ impl Expander { let name = format!("{}_repeat{aux_repeat_counter}", pool.resolve(var_name)); let name = pool.intern(&name); // Aux rules are appended after the original variables, so they occupy - // non-terminal indices `preceding..`. + // non-terminal indices `preceding..`. The aux stands for `Repeat(content)`, + // which matches zero width exactly when `content` does. let symbol = Symbol::non_terminal(self.preceding + self.aux.len()); + self.zero_width.push_variable(width); self.memo.entry(hash).or_default().push((content, symbol)); let root = wrap_in_binary_tree(pool, symbol, content); self.aux.push(Variable { name, root }); @@ -77,6 +90,189 @@ impl Expander { } } } + Ok(()) + } +} + +/// Whether a rule can match zero characters, with and without crossing an `eof()`. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +struct Width { + nullable: bool, + eof_nullable: bool, +} + +impl Width { + /// Fold `other` in, reporting whether that set anything new to `true`. + fn merge(&mut self, other: Self) -> bool { + let before = *self; + self.nullable |= other.nullable; + self.eof_nullable |= other.eof_nullable; + *self != before + } +} + +#[derive(Clone, Copy)] +enum Visit { + Enter(RuleId), + Exit(RuleId), +} + +/// Zero width analysis over the rule pool. +/// +/// `by_variable` is seeded by a fixpoint over the whole grammar before any expansion, +/// which makes results indepedent of the order in which rules are declared. +#[derive(Default)] +struct ZeroWidth { + by_variable: Vec, + stack: Vec, + values: Vec, +} + +impl ZeroWidth { + /// Seed `by_variable` with every rule's width, as a least fixpoint. + /// + /// [`Self::eval`] reads a nonterminal's width from this table, but rules can + /// reference each other cyclically. Every rule starts at "matches nothing", and + /// all of them are re-evaluated against the current table until a full pass + /// changes nothing. Starting there prevents a cycle from marking itself. A rule + /// that reaches itself reads back "matches nothing", so flags are only set by + /// a path that truly matches zero width. + fn new(pool: &RulePool, variables: &[Variable]) -> Self { + let mut this = Self { + by_variable: vec![Width::default(); variables.len()], + stack: Vec::new(), + values: Vec::new(), + }; + loop { + let mut changed = false; + for (i, v) in variables.iter().enumerate() { + let width = this.eval(pool, v.root); + changed |= this.by_variable[i].merge(width); + } + if !changed { + break; + } + } + this + } + + /// Record the width of a newly created auxiliary rule. + fn push_variable(&mut self, width: Width) { + self.by_variable.push(width); + } + + /// Post-order walk of the subtree at `root`, combining children into their parent. + fn eval(&mut self, pool: &RulePool, root: RuleId) -> Width { + self.stack.clear(); + self.values.clear(); + self.stack.push(Visit::Enter(root)); + while let Some(step) = self.stack.pop() { + match step { + Visit::Enter(id) => match pool.node(id) { + Rule::Seq(range) | Rule::Choice(range) => { + self.stack.push(Visit::Exit(id)); + for &c in pool.child_slice(range) { + self.stack.push(Visit::Enter(c)); + } + } + Rule::Repeat(rule) + | Rule::Metadata { rule, .. } + | Rule::Reserved { rule, .. } => { + self.stack.push(Visit::Exit(id)); + self.stack.push(Visit::Enter(rule)); + } + Rule::Blank => self.values.push(Width { + nullable: true, + eof_nullable: false, + }), + Rule::String(s) => self.values.push(Width { + nullable: s == StrPool::EMPTY_STR_ID, + eof_nullable: false, + }), + Rule::Eof => self.values.push(Width { + nullable: false, + eof_nullable: true, + }), + Rule::Sym { kind, index } => { + let width = match kind { + SymbolType::End => Width { + nullable: false, + eof_nullable: true, + }, + SymbolType::NonTerminal => self + .by_variable + .get(index as usize) + .copied() + .unwrap_or_default(), + // External scanners decide at runtime how far to advance + // and may return a zero width token, so this must count + // as nullable. + // + // A scanner may gate itself on `lexer->eof`, but there's + // no way to determine that here. Assuming so would reject + // every grammar that `repeat`s an external. + SymbolType::External => Width { + nullable: true, + eof_nullable: false, + }, + // `expand_tokens` rejects tokens that match the empty string + SymbolType::Terminal => Width::default(), + // Lookahead marker that `build_parse_table` inserts for nonterminal + // extras _after_ this pass runs. + SymbolType::EndOfNonTerminalExtra => unreachable!(), + }; + self.values.push(width); + } + // `extract_tokens` hoists every `Pattern` into the lexical grammar and + // leaves a terminal symbol in its place. Only syntactic (non + // lexical grammar) rules are walked here. + Rule::Pattern(..) + // `intern_symbols` resolves every `NamedSymbol` to a `Sym` + | Rule::NamedSymbol(_) => unreachable!(), + }, + Visit::Exit(id) => match pool.node(id) { + Rule::Choice(range) => { + let base = self.values.len() - range.len as usize; + let mut width = Width::default(); + for child in self.values.drain(base..) { + width.nullable |= child.nullable; + width.eof_nullable |= child.eof_nullable; + } + self.values.push(width); + } + Rule::Seq(range) => { + let base = self.values.len() - range.len as usize; + let mut nullable = true; // matches empty iff every element does + let mut any_eof = false; // matches eof if any element does + let mut all_zero_width = true; // every element matches empty or via `eof()` + for child in self.values.drain(base..) { + nullable &= child.nullable; + any_eof |= child.eof_nullable; + all_zero_width &= child.nullable || child.eof_nullable; + } + self.values.push(Width { + nullable, + eof_nullable: all_zero_width && any_eof, + }); + } + // Each of these wraps a single child whose width is both already on the + // stack and exactly this (the wrapper) node's width: + // + // - `Repeat` is one or more, so it matches zero width exactly when its + // content does. Zero or more comes in as `Choice(Repeat, Blank)` from + // `parse_grammar`, so that blank gives the nullable case instead of here. + // - `Metadata` carries wrapping data that doesn't change how much input is + // matched. Its `token` forms are already replaced by terminals in + // `extrac_tokens`. + // - `Reserved` only names the reserved word set for a its child. + Rule::Repeat(_) | Rule::Metadata { .. } | Rule::Reserved { .. } => {} + // Every other rule is a leaf. `Enter` pushed its width directly and + // never queued an `Exit`. + _ => unreachable!(), + }, + } + } + self.values.pop().unwrap_or_default() } } @@ -105,9 +301,13 @@ fn wrap_in_binary_tree(pool: &mut RulePool, symbol: Symbol, inner: RuleId) -> Ru } } -pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGrammarMeta) { +pub(super) fn expand_repeats( + grammar: &mut InputGrammar, + meta: &mut ExtractedGrammarMeta, +) -> Result<(), ExpandRepeatsError> { let mut expander = Expander { preceding: grammar.variables.len(), + zero_width: ZeroWidth::new(&grammar.pool, &grammar.variables), ..Default::default() }; for i in 0..grammar.variables.len() { @@ -119,7 +319,14 @@ pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGra if meta.kinds[i] == VariableType::Hidden && let Rule::Repeat(content) = grammar.pool.node(root) { - expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count); + if expander + .zero_width + .eval(&grammar.pool, content) + .eof_nullable + { + return Err(ExpandRepeatsError(grammar.pool.resolve(name).to_string())); + } + expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count)?; grammar.variables[i].root = wrap_in_binary_tree(&mut grammar.pool, Symbol::non_terminal(i), content); meta.kinds[i] = VariableType::Auxiliary; @@ -127,12 +334,13 @@ pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGra continue; } - expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count); + expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count)?; } for var in expander.aux { grammar.variables.push(var); meta.kinds.push(VariableType::Auxiliary); } + Ok(()) } #[cfg(test)] @@ -426,6 +634,34 @@ mod tests { assert!(g.pool.subtree_eq(g.variables[1].root, e1)); } + #[test] + fn test_rejects_repeat_of_eof_helper_rule() { + // rule0: repeat(non_terminal(1)); rule1: eof() + let mut pool = RulePool::default(); + let r0 = { + let n = non_term(&mut pool, 1); + pool.repeat(n) + }; + let r1 = pool.push_node(Rule::Eof); + let (n0, n1) = (pool.intern("rule0"), pool.intern("rule1")); + let mut grammar = InputGrammar { + pool, + variables: vec![ + Variable { name: n0, root: r0 }, + Variable { name: n1, root: r1 }, + ], + ..Default::default() + }; + let mut meta = ExtractedGrammarMeta { + kinds: vec![VariableType::Named; 2], + ..Default::default() + }; + assert_eq!( + expand_repeats(&mut grammar, &mut meta).unwrap_err(), + ExpandRepeatsError("rule0".to_string()) + ); + } + fn term(p: &mut RulePool, i: u32) -> RuleId { p.push_node(Rule::Sym { kind: SymbolType::Terminal, @@ -453,7 +689,7 @@ mod tests { kinds, ..Default::default() }; - expand_repeats(&mut grammar, &mut meta); + expand_repeats(&mut grammar, &mut meta).unwrap(); (grammar, meta) } } diff --git a/crates/generate/src/prepare_grammar/expand_tokens.rs b/crates/generate/src/prepare_grammar/expand_tokens.rs index b01aa8c9b..32444a3c0 100644 --- a/crates/generate/src/prepare_grammar/expand_tokens.rs +++ b/crates/generate/src/prepare_grammar/expand_tokens.rs @@ -162,6 +162,12 @@ pub enum ExpandRuleError { UnexpectedSymbol(Symbol), #[error("unexpected reserved-word context {0}")] UnexpectedReserved(String), + #[error( + "`eof()` cannot be used inside a token. \ + A lexical rule cannot check for end of input, \ + so use `eof()` only at the end of a syntactic rule." + )] + UnexpectedEof, #[error("{0}")] Parse(String), #[error(transparent)] @@ -297,6 +303,7 @@ impl NfaBuilder { result } Rule::Blank => Ok(false), + Rule::Eof => Err(ExpandRuleError::UnexpectedEof)?, Rule::Sym { kind, index } => { Err(ExpandRuleError::UnexpectedSymbol(Symbol { kind, index }))? } diff --git a/crates/generate/src/prepare_grammar/extract_default_aliases.rs b/crates/generate/src/prepare_grammar/extract_default_aliases.rs index b9108c556..c76ee7003 100644 --- a/crates/generate/src/prepare_grammar/extract_default_aliases.rs +++ b/crates/generate/src/prepare_grammar/extract_default_aliases.rs @@ -72,7 +72,9 @@ pub(super) fn extract_default_aliases( SymbolType::External => &mut external_status_list[symbol_index], SymbolType::NonTerminal => &mut non_terminal_status_list[symbol_index], SymbolType::Terminal => &mut terminal_status_list[symbol_index], - SymbolType::End | SymbolType::EndOfNonTerminalExtra => panic!("Unexpected end token"), + SymbolType::End | SymbolType::EndOfNonTerminalExtra => { + panic!("Unexpected end token") + } }; status.appears_unaliased = true; } @@ -204,6 +206,7 @@ mod tests { steps_start, steps_len: steps.len() as u32, dynamic_precedence: 0, + requires_eof_lookahead: false, }); out.var_prods.push((ps, out.productions.len() as u32)); } diff --git a/crates/generate/src/prepare_grammar/flatten_grammar.rs b/crates/generate/src/prepare_grammar/flatten_grammar.rs index f4a8353e2..4193af6db 100644 --- a/crates/generate/src/prepare_grammar/flatten_grammar.rs +++ b/crates/generate/src/prepare_grammar/flatten_grammar.rs @@ -31,6 +31,8 @@ unless they are used only as the grammar's start rule. EmptyString(String), #[error("Rule `{0}` cannot be inlined because it contains a reference to itself")] RecursiveInline(String), + #[error("Rule `{0}` has no reachable productions.")] + NoReachableProductions(String), } #[derive(Clone, Copy, Default)] @@ -179,6 +181,11 @@ fn apply( let child = pool.child_slice(range)[selected as usize]; apply(pool, reserved_ids, child, f_ctx, at_end, st) } + Rule::Eof => { + let symbol = Symbol::end(); + st.push_step(symbol.kind, symbol.index, f_ctx); + Ok(true) + } Rule::Metadata { params, rule } => { let params = pool.params(params); let mut inner_ctx = f_ctx; @@ -227,11 +234,36 @@ fn apply( } /// Append the completed path as a production unless this variable already has an -/// identical one. -fn emit(st: &FlattenState, out: &mut ProductionStore, prod_start: u32) { +/// identical one. A path with `eof()` anywhere but the final step is discarded, +/// since such a production could never be completed. +fn emit(st: &mut FlattenState, out: &mut ProductionStore, prod_start: u32) -> bool { + let last = st.steps.len().saturating_sub(1); + let Some(eof_index) = st + .steps + .iter() + .position(|step| step.symbol() == Symbol::end()) + else { + return emit_ready(st, out, prod_start, false); + }; + if eof_index != last { + return false; + } + st.steps.pop(); + emit_ready(st, out, prod_start, true) +} + +fn emit_ready( + st: &FlattenState, + out: &mut ProductionStore, + prod_start: u32, + requires_eof_lookahead: bool, +) -> bool { for p in &out.productions[prod_start as usize..] { - if p.dynamic_precedence == st.dyn_prec && out.steps[p.step_range()] == st.steps[..] { - return; + if p.dynamic_precedence == st.dyn_prec + && p.requires_eof_lookahead == requires_eof_lookahead + && out.steps[p.step_range()] == st.steps[..] + { + return true; } } let steps_start = out.steps.len() as u32; @@ -240,7 +272,9 @@ fn emit(st: &FlattenState, out: &mut ProductionStore, prod_start: u32) { steps_start, steps_len: st.steps.len() as u32, dynamic_precedence: st.dyn_prec, + requires_eof_lookahead, }); + true } pub(super) fn flatten_grammar( @@ -263,6 +297,7 @@ pub(super) fn flatten_grammar( .collect(); for v in &g.variables { let prod_start = out.productions.len() as u32; + let mut dropped_for_eof = false; st.reset_variable(); loop { apply( @@ -274,13 +309,18 @@ pub(super) fn flatten_grammar( st, )?; if !st.dead { - emit(st, out, prod_start); + dropped_for_eof |= !emit(st, out, prod_start); } if !st.choices.advance() { break; } st.reset_path(); } + if dropped_for_eof && prod_start == out.productions.len() as u32 { + return Err(FlattenGrammarError::NoReachableProductions( + g.pool.resolve(v.name).to_string(), + )); + } out.var_prods .push((prod_start, out.productions.len() as u32)); } @@ -298,7 +338,7 @@ fn check( let used = out.steps.iter().any(|s| s.symbol() == symbol); let inlined = meta.inline.contains(&symbol); for p in &out.productions[p_start as usize..p_end as usize] { - if used && p.steps_len == 0 { + if used && p.steps_len == 0 && !p.requires_eof_lookahead { Err(FlattenGrammarError::EmptyString( g.pool.resolve(g.variables[i].name).to_string(), ))?; diff --git a/crates/generate/src/prepare_grammar/process_inlines.rs b/crates/generate/src/prepare_grammar/process_inlines.rs index e082cf0a7..32e6da970 100644 --- a/crates/generate/src/prepare_grammar/process_inlines.rs +++ b/crates/generate/src/prepare_grammar/process_inlines.rs @@ -1,6 +1,6 @@ use std::hash::{Hash as _, Hasher as _}; -use rustc_hash::{FxHashMap, FxHasher}; +use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -22,12 +22,17 @@ struct InlineBuilder<'a> { struct ScratchProd { steps: Vec, dynamic_precedence: i32, + requires_eof_lookahead: bool, } impl InlineBuilder<'_> { - fn build(mut self) -> InlinedProductionMap { + /// Returns the inline map along with the ids of productions whose every expansion + /// was dropped. + fn build(mut self) -> (InlinedProductionMap, FxHashSet) { + let mut dead = FxHashSet::default(); let mut worklist = Vec::new(); for prod_id in 0..self.first_inlined { + let mut survived = false; worklist.push((prod_id, 0u32)); while !worklist.is_empty() { let mut i = 0; @@ -43,12 +48,16 @@ impl InlineBuilder<'_> { } } else { worklist.remove(i); + survived = true; } } } + if !survived { + dead.insert(prod_id); + } } - InlinedProductionMap { map: self.map } + (InlinedProductionMap { map: self.map }, dead) } /// Expand inlining at one step of one production, dedup, and store the results @@ -62,6 +71,7 @@ impl InlineBuilder<'_> { let mut scratch = vec![ScratchProd { steps: self.out.steps[src.step_range()].to_vec(), dynamic_precedence: src.dynamic_precedence, + requires_eof_lookahead: src.requires_eof_lookahead, }]; let mut i = 0; while i < scratch.len() { @@ -77,8 +87,11 @@ impl InlineBuilder<'_> { let removed_step = removed_prod.steps[si]; let (v_start, v_end) = self.out.var_prods[symbol.index as usize]; let replacements = (v_start..v_end) - .map(|p_idx| { + .filter_map(|p_idx| { let p = self.out.productions[p_idx as usize]; + if p.requires_eof_lookahead && si + 1 < removed_prod.steps.len() { + return None; + } let mut production = removed_prod.clone(); production .steps @@ -105,7 +118,8 @@ impl InlineBuilder<'_> { if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() { production.dynamic_precedence = p.dynamic_precedence; } - production + production.requires_eof_lookahead |= p.requires_eof_lookahead; + Some(production) }) .collect::>(); scratch.splice(i..=i, replacements); @@ -115,11 +129,13 @@ impl InlineBuilder<'_> { for sp in scratch { let mut hasher = FxHasher::default(); sp.dynamic_precedence.hash(&mut hasher); + sp.requires_eof_lookahead.hash(&mut hasher); sp.steps.hash(&mut hasher); let candidates = self.memo.entry(hasher.finish()).or_default(); let existing = candidates.iter().copied().find(|&id| { let p = self.out.productions[id as usize]; p.dynamic_precedence == sp.dynamic_precedence + && p.requires_eof_lookahead == sp.requires_eof_lookahead && self.out.steps[p.step_range()] == sp.steps }); result.push(existing.unwrap_or_else(|| { @@ -129,6 +145,7 @@ impl InlineBuilder<'_> { steps_start, steps_len: sp.steps.len() as u32, dynamic_precedence: sp.dynamic_precedence, + requires_eof_lookahead: sp.requires_eof_lookahead, }); let id = (self.out.productions.len() - 1) as u32; candidates.push(id); @@ -156,6 +173,8 @@ pub enum ProcessInlinesError { Token(String), #[error("Rule `{0}` cannot be inlined because it is the first rule")] FirstRule(String), + #[error("Rule `{0}` has no reachable productions after inlining")] + NoReachableProductions(String), } pub(super) fn process_inlines( @@ -185,14 +204,30 @@ pub(super) fn process_inlines( } } - Ok(InlineBuilder { + let (map, dead) = InlineBuilder { first_inlined: out.productions.len() as u32, - out, + out: &mut *out, inline: &meta.inline, map: FxHashMap::default(), memo: FxHashMap::default(), } - .build()) + .build(); + + if !dead.is_empty() { + for (i, &(p_start, p_end)) in out.var_prods.iter().enumerate() { + if p_start == p_end || !(p_start..p_end).all(|p| dead.contains(&p)) { + continue; + } + let symbol = Symbol::non_terminal(i); + if i == 0 || out.steps.iter().any(|s| s.symbol() == symbol) { + Err(ProcessInlinesError::NoReachableProductions( + g.pool.resolve(g.variables[i].name).to_string(), + ))?; + } + } + } + + Ok(map) } #[cfg(test)] @@ -518,6 +553,7 @@ mod tests { steps_start, steps_len: steps.len() as u32, dynamic_precedence: *dynamic_prc, + requires_eof_lookahead: false, }); } out.var_prods.push((start, out.productions.len() as u32)); @@ -546,6 +582,53 @@ mod tests { }) } + #[test] + fn test_error_when_inlining_removes_all_productions() { + // var0: [nt1] + // var1: [nt2, t10] (nt2 is inlined) + // var2: one empty eof-gated production, like a flattened `_eof: _ => eof()` + // Inlining var2 into var1 drops var1's only production. + let mut out = ProductionStore::default(); + add_variable(&mut out, &[(vec![plain(Symbol::non_terminal(1))], 0)]); + add_variable( + &mut out, + &[( + vec![plain(Symbol::non_terminal(2)), plain(Symbol::terminal(10))], + 0, + )], + ); + let start = out.productions.len() as u32; + out.productions.push(Production { + steps_start: out.steps.len() as u32, + steps_len: 0, + dynamic_precedence: 0, + requires_eof_lookahead: true, + }); + out.var_prods.push((start, start + 1)); + + let mut pool = RulePool::default(); + let variables = ["rule0", "rule1", "rule2"] + .map(|name| { + let name = pool.intern(name); + let root = pool.push_node(crate::rules::Rule::Blank); + crate::grammars::Variable { name, root } + }) + .to_vec(); + let g = InputGrammar { + pool, + variables, + ..Default::default() + }; + let meta = ExtractedGrammarMeta { + inline: vec![Symbol::non_terminal(2)], + ..Default::default() + }; + assert_eq!( + process_inlines(&g, &meta, &mut out).unwrap_err(), + ProcessInlinesError::NoReachableProductions("rule1".to_string()) + ); + } + fn plain(symbol: Symbol) -> ProductionStep { ProductionStep::pack(symbol, Precedence::None, None, None, None, 0) } diff --git a/crates/generate/src/rules.rs b/crates/generate/src/rules.rs index ffdcb79a9..a83e8edf3 100644 --- a/crates/generate/src/rules.rs +++ b/crates/generate/src/rules.rs @@ -141,6 +141,7 @@ pub enum Rule { Seq(RuleIdRange), Choice(RuleIdRange), Repeat(RuleId), + Eof, Metadata { params: ParamsId, rule: RuleId }, Reserved { rule: RuleId, ctx: StrId }, } @@ -299,6 +300,10 @@ impl RulePool { self.push_node(Rule::Repeat(content)) } + pub fn eof(&mut self) -> RuleId { + self.push_node(Rule::Eof) + } + pub fn seq(&mut self, ids: &[RuleId]) -> RuleId { let range = self.push_children(ids); self.push_node(Rule::Seq(range)) @@ -353,7 +358,7 @@ impl RulePool { let node = self.node(id); std::mem::discriminant(&node).hash(&mut hasher); match node { - Rule::Blank => {} + Rule::Blank | Rule::Eof => {} Rule::String(s) | Rule::NamedSymbol(s) => s.hash(&mut hasher), Rule::Pattern(p, f) => { p.hash(&mut hasher); @@ -390,7 +395,7 @@ impl RulePool { let mut stack = vec![(a, b)]; while let Some((a, b)) = stack.pop() { match (self.node(a), self.node(b)) { - (Rule::Blank, Rule::Blank) => {} + (Rule::Blank, Rule::Blank) | (Rule::Eof, Rule::Eof) => {} (Rule::String(x), Rule::String(y)) | (Rule::NamedSymbol(x), Rule::NamedSymbol(y)) => { if x != y { @@ -476,7 +481,9 @@ impl RulePool { self.rule_is_referenced(rule, target, is_external) } Rule::Repeat(inner) => self.rule_is_referenced(inner, target, false), - Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => false, + Rule::Blank | Rule::Eof | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => { + false + } } } @@ -499,7 +506,7 @@ impl RulePool { self.collect_referenced_ids(rule, skip_top_level, out); } Rule::Repeat(inner) => self.collect_referenced_ids(inner, false, out), - Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {} + Rule::Blank | Rule::Eof | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {} } } } diff --git a/crates/generate/src/tables.rs b/crates/generate/src/tables.rs index bd171cf30..690905b2d 100644 --- a/crates/generate/src/tables.rs +++ b/crates/generate/src/tables.rs @@ -238,6 +238,7 @@ impl ActionListPool { lex_state_id: state.lex_state_id, external_lex_state_id: state.external_lex_state_id, core_id: state.core_id, + has_eof_gated_reduce: state.has_eof_gated_reduce, } }) .collect(); @@ -293,6 +294,7 @@ pub struct ParseState { pub lex_state_id: LexStateId, pub external_lex_state_id: LexStateId, pub core_id: u32, + pub has_eof_gated_reduce: bool, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/docs/src/assets/schemas/grammar.schema.json b/docs/src/assets/schemas/grammar.schema.json index e30c7ba0f..9ad1adf9f 100644 --- a/docs/src/assets/schemas/grammar.schema.json +++ b/docs/src/assets/schemas/grammar.schema.json @@ -129,6 +129,17 @@ "required": ["type"] }, + "eof-rule": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "EOF" + } + }, + "required": ["type"] + }, + "string-rule": { "type": "object", "properties": { @@ -321,6 +332,7 @@ "oneOf": [ { "$ref": "#/definitions/alias-rule" }, { "$ref": "#/definitions/blank-rule" }, + { "$ref": "#/definitions/eof-rule" }, { "$ref": "#/definitions/string-rule" }, { "$ref": "#/definitions/pattern-rule" }, { "$ref": "#/definitions/symbol-rule" }, diff --git a/docs/src/creating-parsers/2-the-grammar-dsl.md b/docs/src/creating-parsers/2-the-grammar-dsl.md index 90e041ac3..820de60cd 100644 --- a/docs/src/creating-parsers/2-the-grammar-dsl.md +++ b/docs/src/creating-parsers/2-the-grammar-dsl.md @@ -101,6 +101,11 @@ rule. In the resulting syntax tree, you can then use that field name to access s one passed into the `wordset` parameter. This is useful for contextual keywords, such as `if` in JavaScript, which cannot be used as a variable name in most contexts, but can be used as a property name. +- **End of Input : `eof()`** — This function creates a rule that matches the end of the input, without consuming any +characters. It may only appear as the final symbol of a rule, and is useful when a rule should match either an explicit +terminator (such as a newline) or the end of the file. Choice branches where other symbols follow `eof()` can never +match, so they are dropped, and `eof()` is not allowed inside `token()`. + In addition to the `name` and `rules` fields, grammars have a few other optional public fields that influence the behavior of the parser. Each of these fields is a function that accepts the grammar object (`$`) as its only parameter, like the grammar rules themselves. These fields are: diff --git a/test/fixtures/test_grammars/duplicate_reduction_precedence/corpus.txt b/test/fixtures/test_grammars/duplicate_reduction_precedence/corpus.txt new file mode 100644 index 000000000..fe33b8fca --- /dev/null +++ b/test/fixtures/test_grammars/duplicate_reduction_precedence/corpus.txt @@ -0,0 +1,10 @@ +====================== +higher precedence wins +====================== + +x +--- + +(start + (a + (x))) diff --git a/test/fixtures/test_grammars/duplicate_reduction_precedence/grammar.js b/test/fixtures/test_grammars/duplicate_reduction_precedence/grammar.js new file mode 100644 index 000000000..8ffc5cc0a --- /dev/null +++ b/test/fixtures/test_grammars/duplicate_reduction_precedence/grammar.js @@ -0,0 +1,13 @@ +// `a` has two productions that reduce identically and differ only in precedence. +// The higher one has to outrank `c`, which means the duplicate action still has +// to be compared against rather than skipped. +export default grammar({ + name: 'duplicate_reduction_precedence', + + rules: { + start: $ => choice($.a, $.c), + a: $ => choice($.x, prec(2, $.x)), + c: $ => $.x, + x: _ => 'x', + } +}) diff --git a/test/fixtures/test_grammars/eof_basic/corpus.txt b/test/fixtures/test_grammars/eof_basic/corpus.txt new file mode 100644 index 000000000..8caa2b62e --- /dev/null +++ b/test/fixtures/test_grammars/eof_basic/corpus.txt @@ -0,0 +1,34 @@ +============ +trailing newline +============ + +text +text + +--- + +(source_file + (line) + (line)) + +============ +no trailing newline +============ + +text +text +--- + +(source_file + (line) + (line)) + +============ +single line, no newline +============ + +text +--- + +(source_file + (line)) diff --git a/test/fixtures/test_grammars/eof_basic/grammar.js b/test/fixtures/test_grammars/eof_basic/grammar.js new file mode 100644 index 000000000..5e1e066bb --- /dev/null +++ b/test/fixtures/test_grammars/eof_basic/grammar.js @@ -0,0 +1,12 @@ +export default grammar({ + name: 'eof_basic', + + rules: { + source_file: $ => repeat($.line), + + line: $ => choice( + seq('text', '\n'), + seq('text', eof()), + ), + } +}); diff --git a/test/fixtures/test_grammars/eof_dropped_branch/corpus.txt b/test/fixtures/test_grammars/eof_dropped_branch/corpus.txt new file mode 100644 index 000000000..c09391dd0 --- /dev/null +++ b/test/fixtures/test_grammars/eof_dropped_branch/corpus.txt @@ -0,0 +1,46 @@ +============ +no items +============ + +() +--- + +(source_file) + +============ +items separated by newlines +============ + +(x +x +x) +--- + +(source_file (item) (item) (item)) + +============ +items separated by semicolons +============ + +(x;x;x) +--- + +(source_file (item) (item) (item)) + +============ +trailing separator +============ + +(x;) +--- + +(source_file (item)) + +============ +eof in an inline before a later symbol is unreachable +============ + +ok +--- + +(source_file (inline_test)) diff --git a/test/fixtures/test_grammars/eof_dropped_branch/grammar.js b/test/fixtures/test_grammars/eof_dropped_branch/grammar.js new file mode 100644 index 000000000..1489178e1 --- /dev/null +++ b/test/fixtures/test_grammars/eof_dropped_branch/grammar.js @@ -0,0 +1,30 @@ +// The `eof()` alternative of `terminator` is unreachable inside the parens +// (because `)` always has to come after) so the generator silently drops +// that production. The grammar still parses because the newline and `;` +// alternatives remain. + +const terminator = choice('\n', ';', eof()); + +export default grammar({ + name: 'eof_dropped_branch', + + inline: $ => [$._inline_eof], + + rules: { + source_file: $ => choice( + seq( + '(', + optional(seq($.item, repeat(seq(terminator, $.item)), optional(terminator))), + ')', + ), + $.inline_test, + ), + + item: _ => 'x', + + // Inlining the second alternative would put `bad` after EOF, so that + // alternative must be discarded while the reachable one remains. + inline_test: $ => choice('ok', seq($._inline_eof, 'bad')), + _inline_eof: _ => eof(), + } +}); diff --git a/test/fixtures/test_grammars/eof_duplicate_reduction/corpus.txt b/test/fixtures/test_grammars/eof_duplicate_reduction/corpus.txt new file mode 100644 index 000000000..edb705e44 --- /dev/null +++ b/test/fixtures/test_grammars/eof_duplicate_reduction/corpus.txt @@ -0,0 +1,20 @@ +=========================== +single line at end of input +=========================== + +text +--- + +(source_file + (line)) + +============= +several lines +============= + +texttext +--- + +(source_file + (line) + (line)) diff --git a/test/fixtures/test_grammars/eof_duplicate_reduction/grammar.js b/test/fixtures/test_grammars/eof_duplicate_reduction/grammar.js new file mode 100644 index 000000000..5617a92e5 --- /dev/null +++ b/test/fixtures/test_grammars/eof_duplicate_reduction/grammar.js @@ -0,0 +1,11 @@ +// Both branches reduce `line` to the same single `'text'` child, differing only in +// whether the reduce is gated on end of input. Identical actions are not a conflict, +// but they still have to be compared on precedence rather than skipped. +export default grammar({ + name: 'eof_duplicate_reduction', + + rules: { + source_file: $ => repeat($.line), + line: _ => choice('text', seq('text', eof())), + } +}); diff --git a/test/fixtures/test_grammars/eof_misplaced/expected_error.txt b/test/fixtures/test_grammars/eof_misplaced/expected_error.txt new file mode 100644 index 000000000..fdc0c39e1 --- /dev/null +++ b/test/fixtures/test_grammars/eof_misplaced/expected_error.txt @@ -0,0 +1 @@ +Rule `source_file` has no reachable productions. \ No newline at end of file diff --git a/test/fixtures/test_grammars/eof_misplaced/grammar.js b/test/fixtures/test_grammars/eof_misplaced/grammar.js new file mode 100644 index 000000000..dc9ac0f6a --- /dev/null +++ b/test/fixtures/test_grammars/eof_misplaced/grammar.js @@ -0,0 +1,7 @@ +export default grammar({ + name: 'eof_misplaced', + + rules: { + source_file: $ => seq(eof(), 'after'), + } +}); diff --git a/test/fixtures/test_grammars/eof_repeat_terminated/corpus.txt b/test/fixtures/test_grammars/eof_repeat_terminated/corpus.txt new file mode 100644 index 000000000..fb4547f63 --- /dev/null +++ b/test/fixtures/test_grammars/eof_repeat_terminated/corpus.txt @@ -0,0 +1,18 @@ +============ +single item at end of input +============ + +text +--- + +(source_file) + +============ +item not at end of input +============ + +texttext +--- + +(source_file + (ERROR)) diff --git a/test/fixtures/test_grammars/eof_repeat_terminated/grammar.js b/test/fixtures/test_grammars/eof_repeat_terminated/grammar.js new file mode 100644 index 000000000..fa0808549 --- /dev/null +++ b/test/fixtures/test_grammars/eof_repeat_terminated/grammar.js @@ -0,0 +1,10 @@ +// A repeat item ending in `eof()` can match at most once. The EOF-gated +// reduce must survive table minimization, so a second `text` is a parse +// error rather than being silently accepted. +export default grammar({ + name: 'eof_repeat_terminated', + + rules: { + source_file: $ => repeat(seq('text', eof())), + } +}); diff --git a/test/fixtures/test_grammars/eof_repeat_via_nullable_rule/expected_error.txt b/test/fixtures/test_grammars/eof_repeat_via_nullable_rule/expected_error.txt new file mode 100644 index 000000000..9565c560d --- /dev/null +++ b/test/fixtures/test_grammars/eof_repeat_via_nullable_rule/expected_error.txt @@ -0,0 +1 @@ +Rule `body` contains a repetition that can match the empty string at end of input \ No newline at end of file diff --git a/test/fixtures/test_grammars/eof_repeat_via_nullable_rule/grammar.js b/test/fixtures/test_grammars/eof_repeat_via_nullable_rule/grammar.js new file mode 100644 index 000000000..dc53b2b4c --- /dev/null +++ b/test/fixtures/test_grammars/eof_repeat_via_nullable_rule/grammar.js @@ -0,0 +1,11 @@ +// `n` matches the empty string, so `seq($.n, eof())` can match nothing at all at +// the end of input and the outer `repeat` would spin. +export default grammar({ + name: 'eof_repeat_via_nullable_rule', + + rules: { + start: $ => $.body, + body: $ => repeat1(seq($.n, eof())), + n: _ => optional('a'), + } +}) diff --git a/test/fixtures/test_grammars/eof_repeat_via_rule/corpus.txt b/test/fixtures/test_grammars/eof_repeat_via_rule/corpus.txt new file mode 100644 index 000000000..8c53666c5 --- /dev/null +++ b/test/fixtures/test_grammars/eof_repeat_via_rule/corpus.txt @@ -0,0 +1,23 @@ +======================== +one item at end of input +======================== + +aaa +--- + +(start + (body + (b))) + +======================== +item not at end of input +======================== + +aab +--- + +(start + (body + (b)) + (ERROR + (UNEXPECTED 'b'))) diff --git a/test/fixtures/test_grammars/eof_repeat_via_rule/grammar.js b/test/fixtures/test_grammars/eof_repeat_via_rule/grammar.js new file mode 100644 index 000000000..bc31819a2 --- /dev/null +++ b/test/fixtures/test_grammars/eof_repeat_via_rule/grammar.js @@ -0,0 +1,11 @@ +// The repeated content ends in `eof()`, so it can match at most once, but it +// still has to consume some input to get there. +export default grammar({ + name: 'eof_repeat_via_rule', + + rules: { + start: $ => $.body, + body: $ => repeat1(seq($.b, eof())), + b: _ => repeat1('a'), + } +})