feat!: add an eof function

This commit adds an `eof()` function for grammars, which is easier to
use than the NUL byte directly. It compiles down to a constraint that
the enclosing production can only reduce at end of input, not to a
shiftable token.

BREAKING CHANGE: Public error types for `tree-sitter-generate` were modified.

Co-authored-by: Will Lillis <will.lillis24@gmail.com>

crates/cli/npm/dsl.d.ts

@@ -15,6 +15,8 @@ type SeqRule = { type: 'SEQ'; members: Rule[] };
 type StringRule = { type: 'STRING'; value: string };
 type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
 type TokenRule = { type: 'TOKEN'; content: Rule };
+type EOFRule = { type: 'EOF' };
+
 
 type Rule =
   | AliasRule
@@ -33,7 +35,8 @@ type Rule =
   | SeqRule
   | StringRule
   | SymbolRule<string>
-  | TokenRule;
+  | TokenRule
+  | EOFRule;
 
 declare class RustRegex {
   value: string;
@@ -382,6 +385,19 @@ declare const token: {
   immediate(rule: RuleOrLiteral): ImmediateTokenRule;
 };
 
+/**
+ * Matches the end of input. May only appear as the final symbol of a
+ * (possibly nested) sequence; a production ending in `eof()` reduces only
+ * when the lookahead is end-of-input, rather than shifting a token.
+ *
+ * Choice branches that continue past `eof()` are dropped as unreachable,
+ * and `eof()` is not allowed inside `token()`.
+ *
+ * Useful when a rule should match either an explicit terminator (e.g. a
+ * newline) or the end of the file.
+ */
+declare function eof(): EOFRule;
+
 /**
  * Creates a new language grammar with the provided schema.
  *

crates/generate/src/build_tables/build_parse_table.rs

@@ -115,6 +115,7 @@ pub struct Interpretation {
     pub conflicting_lookahead: String,
     pub precedence: Option<String>,
     pub associativity: Option<String>,
+    pub requires_eof_lookahead: bool,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -142,16 +143,21 @@ impl std::fmt::Display for ConflictError {
             .iter()
             .map(|i| {
                 let line = i.to_string();
-                let prec_line = if let (Some(precedence), Some(associativity)) =
+                let mut annotations = Vec::new();
-                    (&i.precedence, &i.associativity)
+                if let (Some(precedence), Some(associativity)) = (&i.precedence, &i.associativity) {
-                {
+                    annotations.push(format!(
-                    Some(format!(
                         "(precedence: {precedence}, associativity: {associativity})",
-                    ))
+                    ));
+                } else if let Some(precedence) = &i.precedence {
+                    annotations.push(format!("(precedence: {precedence})"));
+                }
+                if i.requires_eof_lookahead {
+                    annotations.push("(reduces only at end of input)".to_string());
+                }
+                let prec_line = if annotations.is_empty() {
+                    None
                 } else {
-                    i.precedence
+                    Some(annotations.join("  "))
-                        .as_ref()
-                        .map(|precedence| format!("(precedence: {precedence})"))
                 };
 
                 (line, prec_line)
@@ -430,6 +436,7 @@ impl<'a> ParseTableBuilder<'a> {
                     nonterminal_entries: IndexMap::default(),
                     reserved_words: TokenSet::default(),
                     core_id,
+                    has_eof_gated_reduce: false,
                 });
                 self.parse_state_queue.push_back(ParseStateQueueEntry {
                     state_id,
@@ -549,7 +556,15 @@ impl<'a> ParseTableBuilder<'a> {
 
                 let precedence = item.precedence(self.syntax_grammar);
                 let associativity = item.associativity(self.syntax_grammar);
+                if item.production(self.syntax_grammar).requires_eof_lookahead {
+                    self.parse_table.states[state_id as usize].has_eof_gated_reduce = true;
+                }
                 for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() {
+                    if item.production(self.syntax_grammar).requires_eof_lookahead
+                        && lookahead != Symbol::end()
+                    {
+                        continue;
+                    }
                     let table_entry = self.parse_table.states[state_id as usize]
                         .terminal_entries
                         .entry(lookahead)
@@ -575,9 +590,15 @@ impl<'a> ParseTableBuilder<'a> {
                                 lookaheads_with_conflicts.remove(lookahead);
                                 *reduction_info = ReductionInfo::default();
                             }
+                            // Two items that reduce identically build the same tree, so
+                            // there is nothing for the user to resolve. Precedence is
+                            // still compared first, because an item that only repeats an
+                            // existing action can still outrank it and clear the entry.
                             Ordering::Equal => {
-                                table_entry.actions.push(action);
+                                if !table_entry.actions.contains(&action) {
-                                lookaheads_with_conflicts.insert(lookahead);
+                                    table_entry.actions.push(action);
+                                    lookaheads_with_conflicts.insert(lookahead);
+                                }
                             }
                             Ordering::Less => continue,
                         }
@@ -997,6 +1018,9 @@ impl<'a> ParseTableBuilder<'a> {
                     conflicting_lookahead: self.symbol_name(conflicting_lookahead),
                     precedence,
                     associativity,
+                    requires_eof_lookahead: item
+                        .production(self.syntax_grammar)
+                        .requires_eof_lookahead,
                 }
             })
             .collect::<Vec<_>>();

crates/generate/src/build_tables/item.rs

@@ -28,17 +28,19 @@ const fn start_production() -> ProdRef<'static> {
     ProdRef {
         steps: &START_STEPS,
         dynamic_precedence: 0,
+        requires_eof_lookahead: false,
     }
 }
 
 /// Precomputed identity keys for one `(production, dot)` pair.
 ///
 /// `cmp` is the rank of the content tuple `Ord` compared (dynamic precedence,
-/// length, precedence/associativity at the dot, then completed steps' aliases and fields
+/// `requires_eof_lookahead`, length, precedence/associativity at the dot, then
-/// and remaining steps in full). Equal ranks hold _exactly_ when the tuple is equal,
+/// completed steps' aliases and fields and remaining steps in full). Equal ranks
-/// so it doubles as the equality class for items without preceding inherited fields.
+/// hold _exactly_ when the tuple is equal, so it doubles as the equality class
-/// `eq_with_syms` subdivides `cmp` by the completed steps' symbols, which participate
+/// for items without preceding inherited fields. `eq_with_syms` subdivides `cmp`
-/// in equality only when `has_preceding_inherited_fields` is set.
+/// by the completed steps' symbols, which participate in equality only when
+/// `has_preceding_inherited_fields` is set.
 #[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
 pub struct DotKeys {
     pub cmp: u32,
@@ -198,6 +200,11 @@ impl Ord for ItemContent<'_> {
         self.production
             .dynamic_precedence
             .cmp(&other.production.dynamic_precedence)
+            .then_with(|| {
+                self.production
+                    .requires_eof_lookahead
+                    .cmp(&other.production.requires_eof_lookahead)
+            })
             .then_with(|| {
                 self.production
                     .steps

crates/generate/src/build_tables/item_set_builder.rs

@@ -193,6 +193,8 @@ impl<'a> ParseItemSetBuilder<'a> {
         //
         // Rather than computing these additions recursively, we use an explicit stack.
         let empty_lookaheads = TokenSet::new();
+        let mut eof_lookaheads = TokenSet::new();
+        eof_lookaheads.insert(Symbol::end());
         let mut stack = Vec::new();
         let mut follow_set_info_by_non_terminal = FxHashMap::<usize, FollowSetInfo>::default();
         for i in 0..syntax_grammar.variables.len() {
@@ -230,6 +232,13 @@ impl<'a> ParseItemSetBuilder<'a> {
                                 result.reserved_first_sets[&next_step.symbol()],
                                 false,
                             ));
+                        } else if production.requires_eof_lookahead {
+                            stack.push((
+                                symbol.index as usize,
+                                &eof_lookaheads,
+                                ReservedWordSetId::default(),
+                                false,
+                            ));
                         } else {
                             stack.push((
                                 symbol.index as usize,
@@ -269,18 +278,36 @@ impl<'a> ParseItemSetBuilder<'a> {
 
                     if let Some(ids) = inlines.inlined_prod_ids(item.prod_id, item.step_index) {
                         for &id in ids {
+                            let mut item_info = info;
+                            if syntax_grammar.production(id).requires_eof_lookahead {
+                                item_info.lookaheads =
+                                    result.lookaheads.intern_ref(&eof_lookaheads);
+                                item_info.reserved_lookaheads = ReservedWordSetId::default();
+                                item_info.propagates_lookaheads = false;
+                                item_info.contains_word = false;
+                            }
                             find_or_push(
                                 additions_for_non_terminal,
                                 TransitiveClosureAddition {
                                     item: item.substitute_production(id, key_map.keys_for(id)),
-                                    info,
+                                    info: item_info,
                                 },
                             );
                         }
                     } else {
+                        let mut item_info = info;
+                        if syntax_grammar.production(prod_id).requires_eof_lookahead {
+                            item_info.lookaheads = result.lookaheads.intern_ref(&eof_lookaheads);
+                            item_info.reserved_lookaheads = ReservedWordSetId::default();
+                            item_info.propagates_lookaheads = false;
+                            item_info.contains_word = false;
+                        }
                         find_or_push(
                             additions_for_non_terminal,
-                            TransitiveClosureAddition { item, info },
+                            TransitiveClosureAddition {
+                                item,
+                                info: item_info,
+                            },
                         );
                     }
                 }

crates/generate/src/build_tables/minimize_parse_table.rs

@@ -159,6 +159,9 @@ impl Minimizer<'_> {
 
         let mut unit_reduction_symbols_by_state = FxHashMap::default();
         for (i, state) in self.parse_table.states.iter().enumerate() {
+            if state.has_eof_gated_reduce {
+                continue;
+            }
             let mut only_unit_reductions = true;
             let mut unit_reduction_symbol = None;
             for (_, id) in &state.terminal_entries {
@@ -431,6 +434,7 @@ impl Minimizer<'_> {
             for state_id in &state_ids[1..] {
                 let other_parse_state = mem::take(&mut self.parse_table.states[*state_id as usize]);
 
+                parse_state.has_eof_gated_reduce |= other_parse_state.has_eof_gated_reduce;
                 parse_state
                     .terminal_entries
                     .extend(other_parse_state.terminal_entries);

crates/generate/src/dsl.js

@@ -33,6 +33,12 @@ function blank() {
   };
 }
 
+function eof() {
+  return {
+    type: "EOF"
+  };
+}
+
 function field(name, rule) {
   if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
     throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`);
@@ -528,6 +534,7 @@ function getEnv(name) {
 
 globalThis.alias = alias;
 globalThis.blank = blank;
+globalThis.eof = eof;
 globalThis.choice = choice;
 globalThis.optional = optional;
 globalThis.prec = prec;

crates/generate/src/grammars.rs

@@ -225,12 +225,17 @@ impl std::fmt::Display for ReservedWordSetId {
     }
 }
 
-/// A flattened production consisting of a step range and its dynamic precedence
+/// A flattened production consisting of a step range, its dynamic precedence, and
+/// whether its reduce action is gated on end-of-input lookahead.
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub struct Production {
     pub steps_start: u32,
     pub steps_len: u32,
     pub dynamic_precedence: i32,
+    /// True when the production was written ending in `eof()`. The reduce
+    /// action for this production is only emitted under the end-of-input
+    /// lookahead, never as a shift.
+    pub requires_eof_lookahead: bool,
 }
 
 impl Production {
@@ -291,6 +296,7 @@ impl SyntaxGrammar {
         ProdRef {
             steps: &self.steps[p.step_range()],
             dynamic_precedence: p.dynamic_precedence,
+            requires_eof_lookahead: p.requires_eof_lookahead,
         }
     }
 
@@ -307,6 +313,7 @@ impl SyntaxGrammar {
 pub struct ProdRef<'a> {
     pub steps: &'a [ProductionStep],
     pub dynamic_precedence: i32,
+    pub requires_eof_lookahead: bool,
 }
 
 impl ProdRef<'_> {

crates/generate/src/node_types.rs

@@ -2847,6 +2847,7 @@ mod tests {
                     steps_start,
                     steps_len: steps.len() as u32 - steps_start,
                     dynamic_precedence: 0,
+                    requires_eof_lookahead: false,
                 });
             }
             var_prods.push((prod_start, productions.len() as u32));

crates/generate/src/parse_grammar.rs

@@ -81,6 +81,7 @@ enum RuleJSON {
         context_name: String,
         content: Box<Self>,
     },
+    EOF,
 }
 
 #[derive(Deserialize)]
@@ -462,6 +463,7 @@ impl RulePool {
                 let sid = self.intern(&value);
                 Ok(self.string(sid))
             }
+            RuleJSON::EOF => Ok(self.eof()),
         }
     }
 }

crates/generate/src/prepare_grammar.rs

@@ -32,9 +32,12 @@ use crate::{
 
 pub use self::expand_tokens::expand_tokens;
 use self::{
-    expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
+    expand_repeats::{ExpandRepeatsError, expand_repeats},
-    extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
+    extract_default_aliases::extract_default_aliases,
-    intern_symbols::intern_symbols, process_inlines::process_inlines,
+    extract_tokens::extract_tokens,
+    flatten_grammar::flatten_grammar,
+    intern_symbols::intern_symbols,
+    process_inlines::process_inlines,
 };
 use super::{
     Diagnostic,
@@ -51,6 +54,7 @@ pub type PrepareGrammarResult<T> = Result<T, PrepareGrammarError>;
 pub enum PrepareGrammarError {
     ValidatePrecedences(#[from] ValidatePrecedenceError),
     ValidateIndirectRecursion(#[from] IndirectRecursionError),
+    ExpandRepeats(#[from] ExpandRepeatsError),
     InternSymbols(#[from] InternSymbolsError),
     ExtractTokens(#[from] ExtractTokensError),
     FlattenGrammar(#[from] FlattenGrammarError),
@@ -116,7 +120,7 @@ pub fn prepare_grammar(
 
     let interned_meta = intern_symbols(&mut g, diagnostics)?;
     let mut ext_meta = extract_tokens(&mut g, &interned_meta)?;
-    expand_repeats(&mut g, &mut ext_meta);
+    expand_repeats(&mut g, &mut ext_meta)?;
 
     let mut state = FlattenState::default();
     let mut out = ProductionStore::default();

crates/generate/src/prepare_grammar/expand_repeats.rs

@@ -1,10 +1,12 @@
 use rustc_hash::FxHashMap;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
 
 use crate::{
     grammars::{InputGrammar, Variable, VariableType},
     prepare_grammar::extract_tokens::ExtractedGrammarMeta,
-    rules::{Rule, RuleId, RulePool, Symbol},
+    rules::{Rule, RuleId, RulePool, Symbol, SymbolType},
-    strpool::StrId,
+    strpool::{StrId, StrPool},
 };
 
 #[derive(Default)]
@@ -13,6 +15,7 @@ struct Expander {
     aux: Vec<Variable>,
     memo: FxHashMap<u64, Vec<(RuleId, Symbol)>>,
     stack: Vec<Task>,
+    zero_width: ZeroWidth,
 }
 
 #[derive(Copy, Clone)]
@@ -21,6 +24,10 @@ enum Task {
     Expand { id: RuleId, content: RuleId },
 }
 
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
+#[error("Rule `{0}` contains a repetition that can match the empty string at end of input")]
+pub struct ExpandRepeatsError(pub String);
+
 impl Expander {
     /// Post-order repeat expansion over one root. Children expand first, and `Reserved`
     /// nodes are not descended.
@@ -30,7 +37,7 @@ impl Expander {
         root: RuleId,
         var_name: StrId,
         aux_repeat_counter: &mut u32,
-    ) {
+    ) -> Result<(), ExpandRepeatsError> {
         self.stack.clear();
         self.stack.push(Task::Visit(root));
         'walk: while let Some(task) = self.stack.pop() {
@@ -53,6 +60,10 @@ impl Expander {
                     _ => {} // For primitive rules, don't change anything.
                 },
                 Task::Expand { id, content } => {
+                    let width = self.zero_width.eval(pool, content);
+                    if width.eof_nullable {
+                        return Err(ExpandRepeatsError(pool.resolve(var_name).to_string()));
+                    }
                     // For repetitions, introduce an auxiliary rule that contains the
                     // repeated content, but can also contain a recursive binary tree structure.
                     let hash = pool.subtree_hash(content);
@@ -68,8 +79,10 @@ impl Expander {
                     let name = format!("{}_repeat{aux_repeat_counter}", pool.resolve(var_name));
                     let name = pool.intern(&name);
                     // Aux rules are appended after the original variables, so they occupy
-                    // non-terminal indices `preceding..`.
+                    // non-terminal indices `preceding..`. The aux stands for `Repeat(content)`,
+                    // which matches zero width exactly when `content` does.
                     let symbol = Symbol::non_terminal(self.preceding + self.aux.len());
+                    self.zero_width.push_variable(width);
                     self.memo.entry(hash).or_default().push((content, symbol));
                     let root = wrap_in_binary_tree(pool, symbol, content);
                     self.aux.push(Variable { name, root });
@@ -77,6 +90,189 @@ impl Expander {
                 }
             }
         }
+        Ok(())
+    }
+}
+
+/// Whether a rule can match zero characters, with and without crossing an `eof()`.
+#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
+struct Width {
+    nullable: bool,
+    eof_nullable: bool,
+}
+
+impl Width {
+    /// Fold `other` in, reporting whether that set anything new to `true`.
+    fn merge(&mut self, other: Self) -> bool {
+        let before = *self;
+        self.nullable |= other.nullable;
+        self.eof_nullable |= other.eof_nullable;
+        *self != before
+    }
+}
+
+#[derive(Clone, Copy)]
+enum Visit {
+    Enter(RuleId),
+    Exit(RuleId),
+}
+
+/// Zero width analysis over the rule pool.
+///
+/// `by_variable` is seeded by a fixpoint over the whole grammar before any expansion,
+/// which makes results indepedent of the order in which rules are declared.
+#[derive(Default)]
+struct ZeroWidth {
+    by_variable: Vec<Width>,
+    stack: Vec<Visit>,
+    values: Vec<Width>,
+}
+
+impl ZeroWidth {
+    /// Seed `by_variable` with every rule's width, as a least fixpoint.
+    ///
+    /// [`Self::eval`] reads a nonterminal's width from this table, but rules can
+    /// reference each other cyclically. Every rule starts at "matches nothing", and
+    /// all of them are re-evaluated against the current table until a full pass
+    /// changes nothing. Starting there prevents a cycle from marking itself. A rule
+    /// that reaches itself reads back  "matches nothing", so flags are only set by
+    /// a path that truly matches zero width.
+    fn new(pool: &RulePool, variables: &[Variable]) -> Self {
+        let mut this = Self {
+            by_variable: vec![Width::default(); variables.len()],
+            stack: Vec::new(),
+            values: Vec::new(),
+        };
+        loop {
+            let mut changed = false;
+            for (i, v) in variables.iter().enumerate() {
+                let width = this.eval(pool, v.root);
+                changed |= this.by_variable[i].merge(width);
+            }
+            if !changed {
+                break;
+            }
+        }
+        this
+    }
+
+    /// Record the width of a newly created auxiliary rule.
+    fn push_variable(&mut self, width: Width) {
+        self.by_variable.push(width);
+    }
+
+    /// Post-order walk of the subtree at `root`, combining children into their parent.
+    fn eval(&mut self, pool: &RulePool, root: RuleId) -> Width {
+        self.stack.clear();
+        self.values.clear();
+        self.stack.push(Visit::Enter(root));
+        while let Some(step) = self.stack.pop() {
+            match step {
+                Visit::Enter(id) => match pool.node(id) {
+                    Rule::Seq(range) | Rule::Choice(range) => {
+                        self.stack.push(Visit::Exit(id));
+                        for &c in pool.child_slice(range) {
+                            self.stack.push(Visit::Enter(c));
+                        }
+                    }
+                    Rule::Repeat(rule)
+                    | Rule::Metadata { rule, .. }
+                    | Rule::Reserved { rule, .. } => {
+                        self.stack.push(Visit::Exit(id));
+                        self.stack.push(Visit::Enter(rule));
+                    }
+                    Rule::Blank => self.values.push(Width {
+                        nullable: true,
+                        eof_nullable: false,
+                    }),
+                    Rule::String(s) => self.values.push(Width {
+                        nullable: s == StrPool::EMPTY_STR_ID,
+                        eof_nullable: false,
+                    }),
+                    Rule::Eof => self.values.push(Width {
+                        nullable: false,
+                        eof_nullable: true,
+                    }),
+                    Rule::Sym { kind, index } => {
+                        let width = match kind {
+                            SymbolType::End => Width {
+                                nullable: false,
+                                eof_nullable: true,
+                            },
+                            SymbolType::NonTerminal => self
+                                .by_variable
+                                .get(index as usize)
+                                .copied()
+                                .unwrap_or_default(),
+                            // External scanners decide at runtime how far to advance
+                            // and may return a zero width token, so this must count
+                            // as nullable.
+                            //
+                            // A scanner may gate itself on `lexer->eof`, but there's
+                            // no way to determine that here. Assuming so would reject
+                            // every grammar that `repeat`s an external.
+                            SymbolType::External => Width {
+                                nullable: true,
+                                eof_nullable: false,
+                            },
+                            // `expand_tokens` rejects tokens that match the empty string
+                            SymbolType::Terminal => Width::default(),
+                            // Lookahead marker that `build_parse_table` inserts for nonterminal
+                            // extras _after_ this pass runs.
+                            SymbolType::EndOfNonTerminalExtra => unreachable!(),
+                        };
+                        self.values.push(width);
+                    }
+                    // `extract_tokens` hoists every `Pattern` into the lexical grammar and
+                    // leaves a terminal symbol in its place. Only syntactic  (non
+                    // lexical grammar) rules are walked here.
+                    Rule::Pattern(..)
+                    // `intern_symbols` resolves every `NamedSymbol` to a `Sym`
+                    | Rule::NamedSymbol(_) => unreachable!(),
+                },
+                Visit::Exit(id) => match pool.node(id) {
+                    Rule::Choice(range) => {
+                        let base = self.values.len() - range.len as usize;
+                        let mut width = Width::default();
+                        for child in self.values.drain(base..) {
+                            width.nullable |= child.nullable;
+                            width.eof_nullable |= child.eof_nullable;
+                        }
+                        self.values.push(width);
+                    }
+                    Rule::Seq(range) => {
+                        let base = self.values.len() - range.len as usize;
+                        let mut nullable = true; // matches empty iff every element does
+                        let mut any_eof = false; // matches eof if any element does
+                        let mut all_zero_width = true; // every element matches empty or via `eof()`
+                        for child in self.values.drain(base..) {
+                            nullable &= child.nullable;
+                            any_eof |= child.eof_nullable;
+                            all_zero_width &= child.nullable || child.eof_nullable;
+                        }
+                        self.values.push(Width {
+                            nullable,
+                            eof_nullable: all_zero_width && any_eof,
+                        });
+                    }
+                    // Each of these wraps a single child whose width is both already on the
+                    // stack and exactly this (the wrapper) node's width:
+                    //
+                    // - `Repeat` is one or more, so it matches zero width exactly when its
+                    //   content does. Zero or more comes in  as `Choice(Repeat, Blank)` from
+                    //   `parse_grammar`, so that blank gives the nullable case instead of here.
+                    // - `Metadata` carries wrapping data that doesn't change how much input is
+                    //   matched. Its `token` forms are already replaced by terminals in
+                    //   `extrac_tokens`.
+                    // - `Reserved` only names the reserved word set for a its child.
+                    Rule::Repeat(_) | Rule::Metadata { .. } | Rule::Reserved { .. } => {}
+                    // Every other rule is a leaf. `Enter` pushed its width directly and
+                    // never queued an `Exit`.
+                    _ => unreachable!(),
+                },
+            }
+        }
+        self.values.pop().unwrap_or_default()
     }
 }
 
@@ -105,9 +301,13 @@ fn wrap_in_binary_tree(pool: &mut RulePool, symbol: Symbol, inner: RuleId) -> Ru
     }
 }
 
-pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGrammarMeta) {
+pub(super) fn expand_repeats(
+    grammar: &mut InputGrammar,
+    meta: &mut ExtractedGrammarMeta,
+) -> Result<(), ExpandRepeatsError> {
     let mut expander = Expander {
         preceding: grammar.variables.len(),
+        zero_width: ZeroWidth::new(&grammar.pool, &grammar.variables),
         ..Default::default()
     };
     for i in 0..grammar.variables.len() {
@@ -119,7 +319,14 @@ pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGra
         if meta.kinds[i] == VariableType::Hidden
             && let Rule::Repeat(content) = grammar.pool.node(root)
         {
-            expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count);
+            if expander
+                .zero_width
+                .eval(&grammar.pool, content)
+                .eof_nullable
+            {
+                return Err(ExpandRepeatsError(grammar.pool.resolve(name).to_string()));
+            }
+            expander.expand_root(&mut grammar.pool, content, name, &mut aux_repeat_count)?;
             grammar.variables[i].root =
                 wrap_in_binary_tree(&mut grammar.pool, Symbol::non_terminal(i), content);
             meta.kinds[i] = VariableType::Auxiliary;
@@ -127,12 +334,13 @@ pub(super) fn expand_repeats(grammar: &mut InputGrammar, meta: &mut ExtractedGra
             continue;
         }
 
-        expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count);
+        expander.expand_root(&mut grammar.pool, root, name, &mut aux_repeat_count)?;
     }
     for var in expander.aux {
         grammar.variables.push(var);
         meta.kinds.push(VariableType::Auxiliary);
     }
+    Ok(())
 }
 
 #[cfg(test)]
@@ -426,6 +634,34 @@ mod tests {
         assert!(g.pool.subtree_eq(g.variables[1].root, e1));
     }
 
+    #[test]
+    fn test_rejects_repeat_of_eof_helper_rule() {
+        // rule0: repeat(non_terminal(1)); rule1: eof()
+        let mut pool = RulePool::default();
+        let r0 = {
+            let n = non_term(&mut pool, 1);
+            pool.repeat(n)
+        };
+        let r1 = pool.push_node(Rule::Eof);
+        let (n0, n1) = (pool.intern("rule0"), pool.intern("rule1"));
+        let mut grammar = InputGrammar {
+            pool,
+            variables: vec![
+                Variable { name: n0, root: r0 },
+                Variable { name: n1, root: r1 },
+            ],
+            ..Default::default()
+        };
+        let mut meta = ExtractedGrammarMeta {
+            kinds: vec![VariableType::Named; 2],
+            ..Default::default()
+        };
+        assert_eq!(
+            expand_repeats(&mut grammar, &mut meta).unwrap_err(),
+            ExpandRepeatsError("rule0".to_string())
+        );
+    }
+
     fn term(p: &mut RulePool, i: u32) -> RuleId {
         p.push_node(Rule::Sym {
             kind: SymbolType::Terminal,
@@ -453,7 +689,7 @@ mod tests {
             kinds,
             ..Default::default()
         };
-        expand_repeats(&mut grammar, &mut meta);
+        expand_repeats(&mut grammar, &mut meta).unwrap();
         (grammar, meta)
     }
 }

crates/generate/src/prepare_grammar/expand_tokens.rs

@@ -162,6 +162,12 @@ pub enum ExpandRuleError {
     UnexpectedSymbol(Symbol),
     #[error("unexpected reserved-word context {0}")]
     UnexpectedReserved(String),
+    #[error(
+        "`eof()` cannot be used inside a token. \
+        A lexical rule cannot check for end of input, \
+        so use `eof()` only at the end of a syntactic rule."
+    )]
+    UnexpectedEof,
     #[error("{0}")]
     Parse(String),
     #[error(transparent)]
@@ -297,6 +303,7 @@ impl NfaBuilder {
                 result
             }
             Rule::Blank => Ok(false),
+            Rule::Eof => Err(ExpandRuleError::UnexpectedEof)?,
             Rule::Sym { kind, index } => {
                 Err(ExpandRuleError::UnexpectedSymbol(Symbol { kind, index }))?
             }

crates/generate/src/prepare_grammar/extract_default_aliases.rs

@@ -72,7 +72,9 @@ pub(super) fn extract_default_aliases(
             SymbolType::External => &mut external_status_list[symbol_index],
             SymbolType::NonTerminal => &mut non_terminal_status_list[symbol_index],
             SymbolType::Terminal => &mut terminal_status_list[symbol_index],
-            SymbolType::End | SymbolType::EndOfNonTerminalExtra => panic!("Unexpected end token"),
+            SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                panic!("Unexpected end token")
+            }
         };
         status.appears_unaliased = true;
     }
@@ -204,6 +206,7 @@ mod tests {
                 steps_start,
                 steps_len: steps.len() as u32,
                 dynamic_precedence: 0,
+                requires_eof_lookahead: false,
             });
             out.var_prods.push((ps, out.productions.len() as u32));
         }

crates/generate/src/prepare_grammar/flatten_grammar.rs

@@ -31,6 +31,8 @@ unless they are used only as the grammar's start rule.
     EmptyString(String),
     #[error("Rule `{0}` cannot be inlined because it contains a reference to itself")]
     RecursiveInline(String),
+    #[error("Rule `{0}` has no reachable productions.")]
+    NoReachableProductions(String),
 }
 
 #[derive(Clone, Copy, Default)]
@@ -179,6 +181,11 @@ fn apply(
             let child = pool.child_slice(range)[selected as usize];
             apply(pool, reserved_ids, child, f_ctx, at_end, st)
         }
+        Rule::Eof => {
+            let symbol = Symbol::end();
+            st.push_step(symbol.kind, symbol.index, f_ctx);
+            Ok(true)
+        }
         Rule::Metadata { params, rule } => {
             let params = pool.params(params);
             let mut inner_ctx = f_ctx;
@@ -227,11 +234,36 @@ fn apply(
 }
 
 /// Append the completed path as a production unless this variable already has an
-/// identical one.
+/// identical one. A path with `eof()` anywhere but the final step is discarded,
-fn emit(st: &FlattenState, out: &mut ProductionStore, prod_start: u32) {
+/// since such a production could never be completed.
+fn emit(st: &mut FlattenState, out: &mut ProductionStore, prod_start: u32) -> bool {
+    let last = st.steps.len().saturating_sub(1);
+    let Some(eof_index) = st
+        .steps
+        .iter()
+        .position(|step| step.symbol() == Symbol::end())
+    else {
+        return emit_ready(st, out, prod_start, false);
+    };
+    if eof_index != last {
+        return false;
+    }
+    st.steps.pop();
+    emit_ready(st, out, prod_start, true)
+}
+
+fn emit_ready(
+    st: &FlattenState,
+    out: &mut ProductionStore,
+    prod_start: u32,
+    requires_eof_lookahead: bool,
+) -> bool {
     for p in &out.productions[prod_start as usize..] {
-        if p.dynamic_precedence == st.dyn_prec && out.steps[p.step_range()] == st.steps[..] {
+        if p.dynamic_precedence == st.dyn_prec
-            return;
+            && p.requires_eof_lookahead == requires_eof_lookahead
+            && out.steps[p.step_range()] == st.steps[..]
+        {
+            return true;
         }
     }
     let steps_start = out.steps.len() as u32;
@@ -240,7 +272,9 @@ fn emit(st: &FlattenState, out: &mut ProductionStore, prod_start: u32) {
         steps_start,
         steps_len: st.steps.len() as u32,
         dynamic_precedence: st.dyn_prec,
+        requires_eof_lookahead,
     });
+    true
 }
 
 pub(super) fn flatten_grammar(
@@ -263,6 +297,7 @@ pub(super) fn flatten_grammar(
         .collect();
     for v in &g.variables {
         let prod_start = out.productions.len() as u32;
+        let mut dropped_for_eof = false;
         st.reset_variable();
         loop {
             apply(
@@ -274,13 +309,18 @@ pub(super) fn flatten_grammar(
                 st,
             )?;
             if !st.dead {
-                emit(st, out, prod_start);
+                dropped_for_eof |= !emit(st, out, prod_start);
             }
             if !st.choices.advance() {
                 break;
             }
             st.reset_path();
         }
+        if dropped_for_eof && prod_start == out.productions.len() as u32 {
+            return Err(FlattenGrammarError::NoReachableProductions(
+                g.pool.resolve(v.name).to_string(),
+            ));
+        }
         out.var_prods
             .push((prod_start, out.productions.len() as u32));
     }
@@ -298,7 +338,7 @@ fn check(
         let used = out.steps.iter().any(|s| s.symbol() == symbol);
         let inlined = meta.inline.contains(&symbol);
         for p in &out.productions[p_start as usize..p_end as usize] {
-            if used && p.steps_len == 0 {
+            if used && p.steps_len == 0 && !p.requires_eof_lookahead {
                 Err(FlattenGrammarError::EmptyString(
                     g.pool.resolve(g.variables[i].name).to_string(),
                 ))?;

crates/generate/src/prepare_grammar/process_inlines.rs

@@ -1,6 +1,6 @@
 use std::hash::{Hash as _, Hasher as _};
 
-use rustc_hash::{FxHashMap, FxHasher};
+use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
@@ -22,12 +22,17 @@ struct InlineBuilder<'a> {
 struct ScratchProd {
     steps: Vec<ProductionStep>,
     dynamic_precedence: i32,
+    requires_eof_lookahead: bool,
 }
 
 impl InlineBuilder<'_> {
-    fn build(mut self) -> InlinedProductionMap {
+    /// Returns the inline map along with the ids of productions whose every expansion
+    /// was dropped.
+    fn build(mut self) -> (InlinedProductionMap, FxHashSet<u32>) {
+        let mut dead = FxHashSet::default();
         let mut worklist = Vec::new();
         for prod_id in 0..self.first_inlined {
+            let mut survived = false;
             worklist.push((prod_id, 0u32));
             while !worklist.is_empty() {
                 let mut i = 0;
@@ -43,12 +48,16 @@ impl InlineBuilder<'_> {
                         }
                     } else {
                         worklist.remove(i);
+                        survived = true;
                     }
                 }
             }
+            if !survived {
+                dead.insert(prod_id);
+            }
         }
 
-        InlinedProductionMap { map: self.map }
+        (InlinedProductionMap { map: self.map }, dead)
     }
 
     /// Expand inlining at one step of one production, dedup, and store the results
@@ -62,6 +71,7 @@ impl InlineBuilder<'_> {
         let mut scratch = vec![ScratchProd {
             steps: self.out.steps[src.step_range()].to_vec(),
             dynamic_precedence: src.dynamic_precedence,
+            requires_eof_lookahead: src.requires_eof_lookahead,
         }];
         let mut i = 0;
         while i < scratch.len() {
@@ -77,8 +87,11 @@ impl InlineBuilder<'_> {
             let removed_step = removed_prod.steps[si];
             let (v_start, v_end) = self.out.var_prods[symbol.index as usize];
             let replacements = (v_start..v_end)
-                .map(|p_idx| {
+                .filter_map(|p_idx| {
                     let p = self.out.productions[p_idx as usize];
+                    if p.requires_eof_lookahead && si + 1 < removed_prod.steps.len() {
+                        return None;
+                    }
                     let mut production = removed_prod.clone();
                     production
                         .steps
@@ -105,7 +118,8 @@ impl InlineBuilder<'_> {
                     if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() {
                         production.dynamic_precedence = p.dynamic_precedence;
                     }
-                    production
+                    production.requires_eof_lookahead |= p.requires_eof_lookahead;
+                    Some(production)
                 })
                 .collect::<Vec<_>>();
             scratch.splice(i..=i, replacements);
@@ -115,11 +129,13 @@ impl InlineBuilder<'_> {
         for sp in scratch {
             let mut hasher = FxHasher::default();
             sp.dynamic_precedence.hash(&mut hasher);
+            sp.requires_eof_lookahead.hash(&mut hasher);
             sp.steps.hash(&mut hasher);
             let candidates = self.memo.entry(hasher.finish()).or_default();
             let existing = candidates.iter().copied().find(|&id| {
                 let p = self.out.productions[id as usize];
                 p.dynamic_precedence == sp.dynamic_precedence
+                    && p.requires_eof_lookahead == sp.requires_eof_lookahead
                     && self.out.steps[p.step_range()] == sp.steps
             });
             result.push(existing.unwrap_or_else(|| {
@@ -129,6 +145,7 @@ impl InlineBuilder<'_> {
                     steps_start,
                     steps_len: sp.steps.len() as u32,
                     dynamic_precedence: sp.dynamic_precedence,
+                    requires_eof_lookahead: sp.requires_eof_lookahead,
                 });
                 let id = (self.out.productions.len() - 1) as u32;
                 candidates.push(id);
@@ -156,6 +173,8 @@ pub enum ProcessInlinesError {
     Token(String),
     #[error("Rule `{0}` cannot be inlined because it is the first rule")]
     FirstRule(String),
+    #[error("Rule `{0}` has no reachable productions after inlining")]
+    NoReachableProductions(String),
 }
 
 pub(super) fn process_inlines(
@@ -185,14 +204,30 @@ pub(super) fn process_inlines(
         }
     }
 
-    Ok(InlineBuilder {
+    let (map, dead) = InlineBuilder {
         first_inlined: out.productions.len() as u32,
-        out,
+        out: &mut *out,
         inline: &meta.inline,
         map: FxHashMap::default(),
         memo: FxHashMap::default(),
     }
-    .build())
+    .build();
+
+    if !dead.is_empty() {
+        for (i, &(p_start, p_end)) in out.var_prods.iter().enumerate() {
+            if p_start == p_end || !(p_start..p_end).all(|p| dead.contains(&p)) {
+                continue;
+            }
+            let symbol = Symbol::non_terminal(i);
+            if i == 0 || out.steps.iter().any(|s| s.symbol() == symbol) {
+                Err(ProcessInlinesError::NoReachableProductions(
+                    g.pool.resolve(g.variables[i].name).to_string(),
+                ))?;
+            }
+        }
+    }
+
+    Ok(map)
 }
 
 #[cfg(test)]
@@ -518,6 +553,7 @@ mod tests {
                 steps_start,
                 steps_len: steps.len() as u32,
                 dynamic_precedence: *dynamic_prc,
+                requires_eof_lookahead: false,
             });
         }
         out.var_prods.push((start, out.productions.len() as u32));
@@ -546,6 +582,53 @@ mod tests {
         })
     }
 
+    #[test]
+    fn test_error_when_inlining_removes_all_productions() {
+        // var0: [nt1]
+        // var1: [nt2, t10] (nt2 is inlined)
+        // var2: one empty eof-gated production, like a flattened `_eof: _ => eof()`
+        // Inlining var2 into var1 drops var1's only production.
+        let mut out = ProductionStore::default();
+        add_variable(&mut out, &[(vec![plain(Symbol::non_terminal(1))], 0)]);
+        add_variable(
+            &mut out,
+            &[(
+                vec![plain(Symbol::non_terminal(2)), plain(Symbol::terminal(10))],
+                0,
+            )],
+        );
+        let start = out.productions.len() as u32;
+        out.productions.push(Production {
+            steps_start: out.steps.len() as u32,
+            steps_len: 0,
+            dynamic_precedence: 0,
+            requires_eof_lookahead: true,
+        });
+        out.var_prods.push((start, start + 1));
+
+        let mut pool = RulePool::default();
+        let variables = ["rule0", "rule1", "rule2"]
+            .map(|name| {
+                let name = pool.intern(name);
+                let root = pool.push_node(crate::rules::Rule::Blank);
+                crate::grammars::Variable { name, root }
+            })
+            .to_vec();
+        let g = InputGrammar {
+            pool,
+            variables,
+            ..Default::default()
+        };
+        let meta = ExtractedGrammarMeta {
+            inline: vec![Symbol::non_terminal(2)],
+            ..Default::default()
+        };
+        assert_eq!(
+            process_inlines(&g, &meta, &mut out).unwrap_err(),
+            ProcessInlinesError::NoReachableProductions("rule1".to_string())
+        );
+    }
+
     fn plain(symbol: Symbol) -> ProductionStep {
         ProductionStep::pack(symbol, Precedence::None, None, None, None, 0)
     }

crates/generate/src/rules.rs

@@ -141,6 +141,7 @@ pub enum Rule {
     Seq(RuleIdRange),
     Choice(RuleIdRange),
     Repeat(RuleId),
+    Eof,
     Metadata { params: ParamsId, rule: RuleId },
     Reserved { rule: RuleId, ctx: StrId },
 }
@@ -299,6 +300,10 @@ impl RulePool {
         self.push_node(Rule::Repeat(content))
     }
 
+    pub fn eof(&mut self) -> RuleId {
+        self.push_node(Rule::Eof)
+    }
+
     pub fn seq(&mut self, ids: &[RuleId]) -> RuleId {
         let range = self.push_children(ids);
         self.push_node(Rule::Seq(range))
@@ -353,7 +358,7 @@ impl RulePool {
             let node = self.node(id);
             std::mem::discriminant(&node).hash(&mut hasher);
             match node {
-                Rule::Blank => {}
+                Rule::Blank | Rule::Eof => {}
                 Rule::String(s) | Rule::NamedSymbol(s) => s.hash(&mut hasher),
                 Rule::Pattern(p, f) => {
                     p.hash(&mut hasher);
@@ -390,7 +395,7 @@ impl RulePool {
         let mut stack = vec![(a, b)];
         while let Some((a, b)) = stack.pop() {
             match (self.node(a), self.node(b)) {
-                (Rule::Blank, Rule::Blank) => {}
+                (Rule::Blank, Rule::Blank) | (Rule::Eof, Rule::Eof) => {}
                 (Rule::String(x), Rule::String(y))
                 | (Rule::NamedSymbol(x), Rule::NamedSymbol(y)) => {
                     if x != y {
@@ -476,7 +481,9 @@ impl RulePool {
                 self.rule_is_referenced(rule, target, is_external)
             }
             Rule::Repeat(inner) => self.rule_is_referenced(inner, target, false),
-            Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => false,
+            Rule::Blank | Rule::Eof | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {
+                false
+            }
         }
     }
 
@@ -499,7 +506,7 @@ impl RulePool {
                 self.collect_referenced_ids(rule, skip_top_level, out);
             }
             Rule::Repeat(inner) => self.collect_referenced_ids(inner, false, out),
-            Rule::Blank | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {}
+            Rule::Blank | Rule::Eof | Rule::String(_) | Rule::Pattern(..) | Rule::Sym { .. } => {}
         }
     }
 }

crates/generate/src/tables.rs

@@ -238,6 +238,7 @@ impl ActionListPool {
                     lex_state_id: state.lex_state_id,
                     external_lex_state_id: state.external_lex_state_id,
                     core_id: state.core_id,
+                    has_eof_gated_reduce: state.has_eof_gated_reduce,
                 }
             })
             .collect();
@@ -293,6 +294,7 @@ pub struct ParseState<T = ActionListId> {
     pub lex_state_id: LexStateId,
     pub external_lex_state_id: LexStateId,
     pub core_id: u32,
+    pub has_eof_gated_reduce: bool,
 }
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]

docs/src/assets/schemas/grammar.schema.json

@@ -129,6 +129,17 @@
       "required": ["type"]
     },
 
+    "eof-rule": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string",
+          "const": "EOF"
+        }
+      },
+      "required": ["type"]
+    },
+
     "string-rule": {
       "type": "object",
       "properties": {
@@ -321,6 +332,7 @@
       "oneOf": [
         { "$ref": "#/definitions/alias-rule" },
         { "$ref": "#/definitions/blank-rule" },
+        { "$ref": "#/definitions/eof-rule" },
         { "$ref": "#/definitions/string-rule" },
         { "$ref": "#/definitions/pattern-rule" },
         { "$ref": "#/definitions/symbol-rule" },

docs/src/creating-parsers/2-the-grammar-dsl.md

@@ -101,6 +101,11 @@ rule. In the resulting syntax tree, you can then use that field name to access s
 one passed into the `wordset` parameter. This is useful for contextual keywords, such as `if` in JavaScript, which cannot
 be used as a variable name in most contexts, but can be used as a property name.
 
+- **End of Input : `eof()`** — This function creates a rule that matches the end of the input, without consuming any
+characters. It may only appear as the final symbol of a rule, and is useful when a rule should match either an explicit
+terminator (such as a newline) or the end of the file. Choice branches where other symbols follow `eof()` can never
+match, so they are dropped, and `eof()` is not allowed inside `token()`.
+
 In addition to the `name` and `rules` fields, grammars have a few other optional public fields that influence the behavior
 of the parser. Each of these fields is a function that accepts the grammar object (`$`) as its only parameter, like the
 grammar rules themselves. These fields are:

test/fixtures/test_grammars/duplicate_reduction_precedence/corpus.txt

@@ -0,0 +1,10 @@
+======================
+higher precedence wins
+======================
+
+x
+---
+
+(start
+  (a
+    (x)))

test/fixtures/test_grammars/duplicate_reduction_precedence/grammar.js

@@ -0,0 +1,13 @@
+// `a` has two productions that reduce identically and differ only in precedence.
+// The higher one has to outrank `c`, which means the duplicate action still has
+// to be compared against rather than skipped.
+export default grammar({
+    name: 'duplicate_reduction_precedence',
+
+    rules: {
+        start: $ => choice($.a, $.c),
+        a: $ => choice($.x, prec(2, $.x)),
+        c: $ => $.x,
+        x: _ => 'x',
+    }
+})

test/fixtures/test_grammars/eof_basic/corpus.txt

@@ -0,0 +1,34 @@
+============
+trailing newline
+============
+
+text
+text
+
+---
+
+(source_file
+  (line)
+  (line))
+
+============
+no trailing newline
+============
+
+text
+text
+---
+
+(source_file
+  (line)
+  (line))
+
+============
+single line, no newline
+============
+
+text
+---
+
+(source_file
+  (line))

test/fixtures/test_grammars/eof_basic/grammar.js

@@ -0,0 +1,12 @@
+export default grammar({
+    name: 'eof_basic',
+
+    rules: {
+        source_file: $ => repeat($.line),
+
+        line: $ => choice(
+            seq('text', '\n'),
+            seq('text', eof()),
+        ),
+    }
+});

test/fixtures/test_grammars/eof_dropped_branch/corpus.txt

@@ -0,0 +1,46 @@
+============
+no items
+============
+
+()
+---
+
+(source_file)
+
+============
+items separated by newlines
+============
+
+(x
+x
+x)
+---
+
+(source_file (item) (item) (item))
+
+============
+items separated by semicolons
+============
+
+(x;x;x)
+---
+
+(source_file (item) (item) (item))
+
+============
+trailing separator
+============
+
+(x;)
+---
+
+(source_file (item))
+
+============
+eof in an inline before a later symbol is unreachable
+============
+
+ok
+---
+
+(source_file (inline_test))

test/fixtures/test_grammars/eof_dropped_branch/grammar.js

@@ -0,0 +1,30 @@
+// The `eof()` alternative of `terminator` is unreachable inside the parens
+// (because `)` always has to come after) so the generator silently drops
+// that production. The grammar still parses because the newline and `;`
+// alternatives remain.
+
+const terminator = choice('\n', ';', eof());
+
+export default grammar({
+    name: 'eof_dropped_branch',
+
+    inline: $ => [$._inline_eof],
+
+    rules: {
+        source_file: $ => choice(
+            seq(
+                '(',
+                optional(seq($.item, repeat(seq(terminator, $.item)), optional(terminator))),
+                ')',
+            ),
+            $.inline_test,
+        ),
+
+        item: _ => 'x',
+
+        // Inlining the second alternative would put `bad` after EOF, so that
+        // alternative must be discarded while the reachable one remains.
+        inline_test: $ => choice('ok', seq($._inline_eof, 'bad')),
+        _inline_eof: _ => eof(),
+    }
+});

test/fixtures/test_grammars/eof_duplicate_reduction/corpus.txt

@@ -0,0 +1,20 @@
+===========================
+single line at end of input
+===========================
+
+text
+---
+
+(source_file
+  (line))
+
+=============
+several lines
+=============
+
+texttext
+---
+
+(source_file
+  (line)
+  (line))

test/fixtures/test_grammars/eof_duplicate_reduction/grammar.js

@@ -0,0 +1,11 @@
+// Both branches reduce `line` to the same single `'text'` child, differing only in
+// whether the reduce is gated on end of input. Identical actions are not a conflict,
+// but they still have to be compared on precedence rather than skipped.
+export default grammar({
+    name: 'eof_duplicate_reduction',
+
+    rules: {
+        source_file: $ => repeat($.line),
+        line: _ => choice('text', seq('text', eof())),
+    }
+});

test/fixtures/test_grammars/eof_misplaced/expected_error.txt

@@ -0,0 +1 @@
+Rule `source_file` has no reachable productions.

test/fixtures/test_grammars/eof_misplaced/grammar.js

@@ -0,0 +1,7 @@
+export default grammar({
+    name: 'eof_misplaced',
+
+    rules: {
+        source_file: $ => seq(eof(), 'after'),
+    }
+});

test/fixtures/test_grammars/eof_repeat_terminated/corpus.txt

@@ -0,0 +1,18 @@
+============
+single item at end of input
+============
+
+text
+---
+
+(source_file)
+
+============
+item not at end of input
+============
+
+texttext
+---
+
+(source_file
+  (ERROR))

test/fixtures/test_grammars/eof_repeat_terminated/grammar.js

@@ -0,0 +1,10 @@
+// A repeat item ending in `eof()` can match at most once. The EOF-gated
+// reduce must survive table minimization, so a second `text` is a parse
+// error rather than being silently accepted.
+export default grammar({
+    name: 'eof_repeat_terminated',
+
+    rules: {
+        source_file: $ => repeat(seq('text', eof())),
+    }
+});

test/fixtures/test_grammars/eof_repeat_via_nullable_rule/expected_error.txt

@@ -0,0 +1 @@
+Rule `body` contains a repetition that can match the empty string at end of input

test/fixtures/test_grammars/eof_repeat_via_nullable_rule/grammar.js

@@ -0,0 +1,11 @@
+// `n` matches the empty string, so `seq($.n, eof())` can match nothing at all at
+// the end of input and the outer `repeat` would spin.
+export default grammar({
+    name: 'eof_repeat_via_nullable_rule',
+
+    rules: {
+        start: $ => $.body,
+        body: $ => repeat1(seq($.n, eof())),
+        n: _ => optional('a'),
+    }
+})

test/fixtures/test_grammars/eof_repeat_via_rule/corpus.txt

@@ -0,0 +1,23 @@
+========================
+one item at end of input
+========================
+
+aaa
+---
+
+(start
+  (body
+    (b)))
+
+========================
+item not at end of input
+========================
+
+aab
+---
+
+(start
+  (body
+    (b))
+  (ERROR
+    (UNEXPECTED 'b')))

test/fixtures/test_grammars/eof_repeat_via_rule/grammar.js

@@ -0,0 +1,11 @@
+// The repeated content ends in `eof()`, so it can match at most once, but it
+// still has to consume some input to get there.
+export default grammar({
+    name: 'eof_repeat_via_rule',
+
+    rules: {
+      start: $ => $.body,
+      body: $ => repeat1(seq($.b, eof())),
+      b: _ => repeat1('a'),
+    }
+})