commit a4c4b85a16ce0ecbb550d6de47801d2e387e629b Author: Max Brunsfeld Date: Wed Dec 5 12:50:12 2018 -0800 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..53eaa2196 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +**/*.rs.bk diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..209086810 Binary files /dev/null and b/Cargo.lock differ diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..965cc81e6 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "rust-tree-sitter-cli" +version = "0.1.0" +authors = ["Max Brunsfeld "] +edition = "2018" + +[dependencies] +bitvec = "0.8" +clap = "2.32" +dirs = "1.0.2" +ignore = "0.4.4" +libloading = "0.5" +rusqlite = "0.14.0" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +tree-sitter = "0.3.1" diff --git a/src/build_tables/item.rs b/src/build_tables/item.rs new file mode 100644 index 000000000..c8d30997c --- /dev/null +++ b/src/build_tables/item.rs @@ -0,0 +1,22 @@ +use crate::grammars::Production; +use std::collections::HashMap; +use bitvec::BitVec; + +#[derive(Debug, PartialEq, Eq)] +pub(super) struct LookaheadSet { + terminal_bits: BitVec, + external_bits: BitVec, + eof: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(super) struct ParseItem { + variable_index: u32, + production_index: u32, + step_index: u32, +} + +#[derive(Debug, PartialEq, Eq)] +pub(super) struct ParseItemSet { + entries: HashMap +} diff --git a/src/build_tables/mod.rs b/src/build_tables/mod.rs new file mode 100644 index 000000000..c5dd5b544 --- /dev/null +++ b/src/build_tables/mod.rs @@ -0,0 +1,34 @@ +mod item; + +use std::collections::{HashMap, VecDeque}; +use crate::grammars::{SyntaxGrammar, LexicalGrammar}; +use crate::tables::{ParseTable, LexTable, ParseStateId}; +use crate::rules::{AliasMap, Symbol}; +use crate::error::Result; +use self::item::ParseItemSet; + +type SymbolSequence = Vec; + +struct ParseStateQueueEntry { + preceding_symbols: SymbolSequence, + item_set: ParseItemSet, + state_id: ParseStateId, +} + +struct ParseTableBuilder<'a> { + syntax_grammar: &'a SyntaxGrammar, + lexical_grammar: &'a LexicalGrammar, + simple_aliases: &'a AliasMap, + state_ids_by_item_set: HashMap, + item_sets_by_state_id: Vec<&'a ParseItemSet>, + parse_state_queue: VecDeque, + parse_table: ParseTable, +} + +pub fn build_tables( + syntax_grammar: &SyntaxGrammar, + lexical_grammar: &LexicalGrammar, + simple_aliases: &AliasMap +) -> Result<(ParseTable, LexTable, LexTable, Option)> { + unimplemented!(); +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 000000000..90e7b8f95 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,13 @@ +#[derive(Debug)] +pub enum Error { + GrammarError(String), + SymbolError(String), +} + +pub type Result = std::result::Result; + +impl From for Error { + fn from(error: serde_json::Error) -> Self { + Error::GrammarError(error.to_string()) + } +} diff --git a/src/generate.rs b/src/generate.rs new file mode 100644 index 000000000..4507fb6f6 --- /dev/null +++ b/src/generate.rs @@ -0,0 +1,26 @@ +use crate::error::Result; +use crate::parse_grammar::parse_grammar; +use crate::prepare_grammar::prepare_grammar; +use crate::build_tables::build_tables; +use crate::render::render_c_code; + +pub fn generate_parser_for_grammar(input: String) -> Result { + let input_grammar = parse_grammar(&input)?; + let (syntax_grammar, lexical_grammar, simple_aliases) = prepare_grammar(&input_grammar)?; + let (parse_table, main_lex_table, keyword_lex_table, keyword_capture_token) = build_tables( + &syntax_grammar, + &lexical_grammar, + &simple_aliases + )?; + let c_code = render_c_code( + &input_grammar.name, + parse_table, + main_lex_table, + keyword_lex_table, + keyword_capture_token, + syntax_grammar, + lexical_grammar, + simple_aliases + ); + Ok(c_code) +} diff --git a/src/grammars.rs b/src/grammars.rs new file mode 100644 index 000000000..6f5b772e9 --- /dev/null +++ b/src/grammars.rs @@ -0,0 +1,98 @@ +use crate::rules::{Associativity, Alias, Rule, Symbol}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VariableType { + Hidden, + Auxiliary, + Anonymous, + Named +} + +// Input grammar + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InputVariable { + pub name: String, + pub kind: VariableType, + pub rule: Rule, +} + +#[derive(PartialEq, Eq)] +pub struct InputGrammar { + pub name: String, + pub variables: Vec, + pub extra_tokens: Vec, + pub expected_conflicts: Vec>, + pub external_tokens: Vec, + pub variables_to_inline: Vec, + pub word_token: Option, +} + +// Extracted lexical grammar + +#[derive(PartialEq, Eq)] +pub struct LexicalVariable { + name: String, + kind: VariableType, + rule: Rule, + is_string: bool, +} + +pub struct LexicalGrammar { + variables: Vec, + separators: Vec, +} + +// Extracted syntax grammar + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProductionStep { + symbol: Symbol, + precedence: i32, + associativity: Option, + alias: Option, + is_excluded: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Production { + steps: Vec, + dynamic_precedence: i32, +} + +#[derive(Clone, PartialEq, Eq)] +pub struct SyntaxVariable { + name: String, + kind: VariableType, +} + +#[derive(Clone, PartialEq, Eq)] +pub struct ExternalToken { + name: String, + kind: VariableType, + corresponding_internal_token: Symbol, +} + +pub struct SyntaxGrammar { + variables: Vec, + extra_tokens: Vec, + expected_conflicts: Vec>, + external_tokens: Vec, + variables_to_inline: Vec, + word_token: Symbol, +} + +#[cfg(test)] +impl InputVariable { + pub fn named(name: &str, rule: Rule) -> Self { + Self { name: name.to_string(), kind: VariableType::Named, rule } + } + + pub fn auxiliary(name: &str, rule: Rule) -> Self { + Self { name: name.to_string(), kind: VariableType::Auxiliary, rule } + } + + pub fn hidden(name: &str, rule: Rule) -> Self { + Self { name: name.to_string(), kind: VariableType::Hidden, rule } + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 000000000..3eeb306aa --- /dev/null +++ b/src/main.rs @@ -0,0 +1,35 @@ +use clap::{App, Arg, SubCommand}; + +#[macro_use] extern crate serde_derive; +#[macro_use] extern crate serde_json; + +mod build_tables; +mod error; +mod generate; +mod grammars; +mod parse_grammar; +mod prepare_grammar; +mod render; +mod rules; +mod tables; + +fn main() { + let matches = App::new("tree-sitter") + .version("0.1") + .author("Max Brunsfeld ") + .about("Generates and tests parsers") + .subcommand( + SubCommand::with_name("generate") + .about("Generate a parser") + ).subcommand( + SubCommand::with_name("parse") + .about("Parse a file") + .arg(Arg::with_name("path").index(1)) + ).subcommand( + SubCommand::with_name("test") + .about("Run a parser's tests") + .arg(Arg::with_name("path").index(1).required(true)) + .arg(Arg::with_name("line").index(2).required(true)) + .arg(Arg::with_name("column").index(3).required(true)) + ); +} diff --git a/src/parse_grammar.rs b/src/parse_grammar.rs new file mode 100644 index 000000000..4c21e5ba3 --- /dev/null +++ b/src/parse_grammar.rs @@ -0,0 +1,153 @@ +use serde_json::{Map, Value}; +use crate::error::Result; +use crate::grammars::{InputGrammar, InputVariable, VariableType}; +use crate::rules::Rule; +use std::collections::HashMap; + +#[derive(Deserialize)] +#[serde(tag = "type")] +#[allow(non_camel_case_types)] +pub enum RuleJSON { + BLANK, + STRING { + value: String, + }, + PATTERN { + value: String, + }, + SYMBOL { + name: String, + }, + CHOICE { + members: Vec, + }, + SEQ { + members: Vec, + }, + REPEAT { + content: Box, + }, + PREC_LEFT { + value: i32, + content: Box, + }, + PREC_RIGHT { + value: i32, + content: Box, + }, + PREC { + value: i32, + content: Box, + }, + TOKEN { + content: Box, + }, + TOKEN_IMMEDIATE { + content: Box, + }, +} + +#[derive(Deserialize)] +struct GrammarJSON { + name: String, + rules: Map, + conflicts: Option>>, + externals: Option>, + extras: Option>, + inline: Option>, + word: Option, +} + +pub fn parse_grammar(input: &str) -> Result { + let grammar_json: GrammarJSON = serde_json::from_str(&input)?; + + let mut variables = Vec::with_capacity(grammar_json.rules.len()); + for (name, value) in grammar_json.rules { + variables.push(InputVariable { + name: name.to_owned(), + kind: VariableType::Named, + rule: parse_rule(serde_json::from_value(value)?), + }) + } + + let extra_tokens = grammar_json.extras + .unwrap_or(Vec::new()) + .into_iter() + .map(parse_rule) + .collect(); + let external_tokens = grammar_json.externals + .unwrap_or(Vec::new()) + .into_iter() + .map(parse_rule) + .collect(); + let expected_conflicts = grammar_json.conflicts + .unwrap_or(Vec::new()); + let variables_to_inline = grammar_json.inline + .unwrap_or(Vec::new()); + + Ok(InputGrammar { + name: grammar_json.name, + word_token: grammar_json.word, + variables, + extra_tokens, + expected_conflicts, + external_tokens, + variables_to_inline, + }) +} + +fn parse_rule(json: RuleJSON) -> Rule { + match json { + RuleJSON::BLANK => Rule::Blank, + RuleJSON::STRING { value } => Rule::String(value), + RuleJSON::PATTERN { value } => Rule::Pattern(value), + RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name), + RuleJSON::CHOICE { members } => Rule::choice(members.into_iter().map(parse_rule).collect()), + RuleJSON::SEQ { members } => Rule::seq(members.into_iter().map(parse_rule).collect()), + RuleJSON::REPEAT { content } => Rule::repeat(parse_rule(*content)), + RuleJSON::PREC { value, content } => Rule::prec(value, parse_rule(*content)), + RuleJSON::PREC_LEFT { value, content } => Rule::prec_left(value, parse_rule(*content)), + RuleJSON::PREC_RIGHT { value, content } => Rule::prec_right(value, parse_rule(*content)), + RuleJSON::TOKEN { content } => Rule::token(parse_rule(*content)), + RuleJSON::TOKEN_IMMEDIATE { content } => Rule::immediate_token(parse_rule(*content)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_grammar() { + let grammar = parse_grammar(&json!({ + "name": "my_lang", + "rules": { + "file": { + "type": "REPEAT", + "content": { + "type": "SYMBOL", + "name": "statement" + } + }, + "statement": { + "type": "STRING", + "value": "foo" + } + } + }).to_string()).unwrap(); + + assert_eq!(grammar.name, "my_lang"); + assert_eq!(grammar.variables, vec![ + InputVariable { + name: "file".to_string(), + kind: VariableType::Named, + rule: Rule::repeat(Rule::NamedSymbol("statement".to_string())) + }, + InputVariable { + name: "statement".to_string(), + kind: VariableType::Named, + rule: Rule::String("foo".to_string()) + }, + ]); + } +} diff --git a/src/prepare_grammar/expand_repeats.rs b/src/prepare_grammar/expand_repeats.rs new file mode 100644 index 000000000..69db150cd --- /dev/null +++ b/src/prepare_grammar/expand_repeats.rs @@ -0,0 +1,220 @@ +use crate::rules::{Rule, Symbol}; +use crate::grammars::{InputVariable, VariableType}; +use std::collections::HashMap; +use std::mem; +use std::rc::Rc; +use super::ExtractedGrammar; + +struct Expander { + variable_name: String, + repeat_count_in_variable: usize, + preceding_symbol_count: usize, + auxiliary_variables: Vec, + existing_repeats: HashMap +} + +impl Expander { + fn expand_variable(&mut self, variable: &mut InputVariable) { + self.variable_name.clear(); + self.variable_name.push_str(&variable.name); + self.repeat_count_in_variable = 0; + let mut rule = Rule::Blank; + mem::swap(&mut rule, &mut variable.rule); + variable.rule = self.expand_rule(&rule); + } + + fn expand_rule(&mut self, rule: &Rule) -> Rule { + match rule { + Rule::Choice { elements } => + Rule::Choice { + elements: elements.iter().map(|element| self.expand_rule(element)).collect() + }, + + Rule::Seq { left, right } => + Rule::Seq { + left: Rc::new(self.expand_rule(left)), + right: Rc::new(self.expand_rule(right)), + }, + + Rule::Repeat(content) => { + let inner_rule = self.expand_rule(content); + + if let Some(existing_symbol) = self.existing_repeats.get(&inner_rule) { + return Rule::Symbol(*existing_symbol); + } + + self.repeat_count_in_variable += 1; + let rule_name = format!("{}_repeat{}", self.variable_name, self.repeat_count_in_variable); + let repeat_symbol = Symbol::non_terminal(self.preceding_symbol_count + self.auxiliary_variables.len()); + let rc_symbol = Rc::new(Rule::Symbol(repeat_symbol)); + self.existing_repeats.insert(inner_rule.clone(), repeat_symbol); + self.auxiliary_variables.push(InputVariable { + name: rule_name, + kind: VariableType::Auxiliary, + rule: Rule::Choice { + elements: vec![ + Rule::Seq { + left: rc_symbol.clone(), + right: rc_symbol + }, + inner_rule + ], + }, + }); + + Rule::Symbol(repeat_symbol) + } + + Rule::Metadata { rule, params } => Rule::Metadata { + rule: Rc::new(self.expand_rule(rule)), + params: params.clone() + }, + + _ => rule.clone() + } + } +} + +pub(super) fn expand_repeats(mut grammar: ExtractedGrammar) -> ExtractedGrammar { + let mut expander = Expander { + variable_name: String::new(), + repeat_count_in_variable: 0, + preceding_symbol_count: grammar.variables.len(), + auxiliary_variables: Vec::new(), + existing_repeats: HashMap::new(), + }; + + for mut variable in grammar.variables.iter_mut() { + expander.expand_variable(&mut variable); + } + + grammar.variables.extend(expander.auxiliary_variables.into_iter()); + grammar +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basic_repeat_expansion() { + // Repeats nested inside of sequences and choices are expanded. + let grammar = expand_repeats(build_grammar(vec![ + InputVariable::named("rule0", Rule::seq(vec![ + Rule::terminal(10), + Rule::choice(vec![ + Rule::repeat(Rule::terminal(11)), + Rule::repeat(Rule::terminal(12)), + ]), + Rule::terminal(13), + ])), + ])); + + assert_eq!(grammar.variables, vec![ + InputVariable::named("rule0", Rule::seq(vec![ + Rule::terminal(10), + Rule::choice(vec![ + Rule::non_terminal(1), + Rule::non_terminal(2), + ]), + Rule::terminal(13), + ])), + InputVariable::auxiliary("rule0_repeat1", Rule::choice(vec![ + Rule::seq(vec![ + Rule::non_terminal(1), + Rule::non_terminal(1), + ]), + Rule::terminal(11), + ])), + InputVariable::auxiliary("rule0_repeat2", Rule::choice(vec![ + Rule::seq(vec![ + Rule::non_terminal(2), + Rule::non_terminal(2), + ]), + Rule::terminal(12), + ])), + ]); + } + + #[test] + fn test_repeat_deduplication() { + // Terminal 4 appears inside of a repeat in three different places. + let grammar = expand_repeats(build_grammar(vec![ + InputVariable::named("rule0", Rule::choice(vec![ + Rule::seq(vec![ Rule::terminal(1), Rule::repeat(Rule::terminal(4)) ]), + Rule::seq(vec![ Rule::terminal(2), Rule::repeat(Rule::terminal(4)) ]), + ])), + InputVariable::named("rule1", Rule::seq(vec![ + Rule::terminal(3), + Rule::repeat(Rule::terminal(4)), + ])), + ])); + + // Only one auxiliary rule is created for repeating terminal 4. + assert_eq!(grammar.variables, vec![ + InputVariable::named("rule0", Rule::choice(vec![ + Rule::seq(vec![ Rule::terminal(1), Rule::non_terminal(2) ]), + Rule::seq(vec![ Rule::terminal(2), Rule::non_terminal(2) ]), + ])), + InputVariable::named("rule1", Rule::seq(vec![ + Rule::terminal(3), + Rule::non_terminal(2), + ])), + InputVariable::auxiliary("rule0_repeat1", Rule::choice(vec![ + Rule::seq(vec![ + Rule::non_terminal(2), + Rule::non_terminal(2), + ]), + Rule::terminal(4), + ])) + ]); + } + + #[test] + fn test_expansion_of_nested_repeats() { + let grammar = expand_repeats(build_grammar(vec![ + InputVariable::named("rule0", Rule::seq(vec![ + Rule::terminal(10), + Rule::repeat(Rule::seq(vec![ + Rule::terminal(11), + Rule::repeat(Rule::terminal(12)) + ])), + ])), + ])); + + assert_eq!(grammar.variables, vec![ + InputVariable::named("rule0", Rule::seq(vec![ + Rule::terminal(10), + Rule::non_terminal(2), + ])), + InputVariable::auxiliary("rule0_repeat1", Rule::choice(vec![ + Rule::seq(vec![ + Rule::non_terminal(1), + Rule::non_terminal(1), + ]), + Rule::terminal(12), + ])), + InputVariable::auxiliary("rule0_repeat2", Rule::choice(vec![ + Rule::seq(vec![ + Rule::non_terminal(2), + Rule::non_terminal(2), + ]), + Rule::seq(vec![ + Rule::terminal(11), + Rule::non_terminal(1), + ]), + ])), + ]); + } + + fn build_grammar(variables: Vec) -> ExtractedGrammar { + ExtractedGrammar { + variables, + extra_tokens: Vec::new(), + external_tokens: Vec::new(), + expected_conflicts: Vec::new(), + variables_to_inline: Vec::new(), + word_token: None, + } + } +} diff --git a/src/prepare_grammar/extract_simple_aliases.rs b/src/prepare_grammar/extract_simple_aliases.rs new file mode 100644 index 000000000..250246f39 --- /dev/null +++ b/src/prepare_grammar/extract_simple_aliases.rs @@ -0,0 +1,10 @@ +use crate::rules::AliasMap; +use crate::grammars::{LexicalGrammar, SyntaxGrammar}; +use super::ExtractedGrammar; + +pub(super) fn extract_simple_aliases( + syntax_grammar: &mut SyntaxGrammar, + lexical_grammar: &mut LexicalGrammar +) -> AliasMap { + unimplemented!(); +} diff --git a/src/prepare_grammar/extract_tokens.rs b/src/prepare_grammar/extract_tokens.rs new file mode 100644 index 000000000..660d38195 --- /dev/null +++ b/src/prepare_grammar/extract_tokens.rs @@ -0,0 +1,7 @@ +use crate::error::Result; +use crate::grammars::LexicalGrammar; +use super::{InternedGrammar, ExtractedGrammar}; + +pub(super) fn extract_tokens(grammar: InternedGrammar) -> Result<(ExtractedGrammar, LexicalGrammar)> { + unimplemented!(); +} diff --git a/src/prepare_grammar/flatten_grammar.rs b/src/prepare_grammar/flatten_grammar.rs new file mode 100644 index 000000000..36fe76c9c --- /dev/null +++ b/src/prepare_grammar/flatten_grammar.rs @@ -0,0 +1,7 @@ +use crate::error::Result; +use crate::grammars::SyntaxGrammar; +use super::ExtractedGrammar; + +pub(super) fn flatten_grammar(grammar: ExtractedGrammar) -> Result { + unimplemented!(); +} diff --git a/src/prepare_grammar/intern_symbols.rs b/src/prepare_grammar/intern_symbols.rs new file mode 100644 index 000000000..00a5c3304 --- /dev/null +++ b/src/prepare_grammar/intern_symbols.rs @@ -0,0 +1,237 @@ +use crate::error::{Error, Result}; +use crate::rules::{Rule, Symbol}; +use crate::grammars::{InputGrammar, InputVariable, VariableType}; +use std::rc::Rc; +use super::InternedGrammar; + +pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result { + let interner = Interner { grammar }; + + if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden { + return Err(Error::GrammarError("Grammar's start rule must be visible".to_string())); + } + + let mut variables = Vec::with_capacity(grammar.variables.len()); + for variable in grammar.variables.iter() { + variables.push(InputVariable { + name: variable.name.clone(), + kind: variable_type_for_name(&variable.name), + rule: interner.intern_rule(&variable.rule)?, + }); + } + + let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len()); + for external_token in grammar.external_tokens.iter() { + let rule = interner.intern_rule(&external_token)?; + let (name, kind) = if let Rule::NamedSymbol(name) = external_token { + (name.clone(), variable_type_for_name(&name)) + } else { + (String::new(), VariableType::Anonymous) + }; + external_tokens.push(InputVariable { name, kind, rule }); + } + + let mut extra_tokens = Vec::with_capacity(grammar.extra_tokens.len()); + for extra_token in grammar.extra_tokens.iter() { + extra_tokens.push(interner.intern_rule(extra_token)?); + } + + let mut expected_conflicts = Vec::new(); + for conflict in grammar.expected_conflicts.iter() { + let mut interned_conflict = Vec::with_capacity(conflict.len()); + for name in conflict { + interned_conflict.push(interner + .intern_name(&name) + .ok_or_else(|| symbol_error(name))? + ); + } + expected_conflicts.push(interned_conflict); + } + + let mut variables_to_inline = Vec::new(); + for name in grammar.variables_to_inline.iter() { + if let Some(symbol) = interner.intern_name(&name) { + variables_to_inline.push(symbol); + } + } + + let mut word_token = None; + if let Some(name) = grammar.word_token.as_ref() { + word_token = Some(interner + .intern_name(&name) + .ok_or_else(|| symbol_error(&name))? + ); + } + + Ok(InternedGrammar { + variables, + external_tokens, + extra_tokens, + expected_conflicts, + variables_to_inline, + word_token, + }) +} + +struct Interner<'a> { + grammar: &'a InputGrammar +} + +impl<'a> Interner<'a> { + fn intern_rule(&self, rule: &Rule) -> Result { + match rule { + Rule::Choice { elements } => { + let mut result = Vec::with_capacity(elements.len()); + for element in elements { + result.push(self.intern_rule(element)?); + } + Ok(Rule::Choice { elements: result }) + }, + + Rule::Seq { left, right } => + Ok(Rule::Seq { + left: Rc::new(self.intern_rule(left)?), + right: Rc::new(self.intern_rule(right)?), + }), + + Rule::Repeat(content) => + Ok(Rule::Repeat(Rc::new(self.intern_rule(content)?))), + + Rule::Metadata { rule, params } => + Ok(Rule::Metadata { + rule: Rc::new(self.intern_rule(rule)?), + params: params.clone() + }), + + Rule::NamedSymbol(name) => { + if let Some(symbol) = self.intern_name(&name) { + Ok(Rule::Symbol(symbol)) + } else { + Err(symbol_error(name)) + } + }, + + _ => Ok(rule.clone()) + + } + } + + fn intern_name(&self, symbol: &str) -> Option { + for (i, variable) in self.grammar.variables.iter().enumerate() { + if variable.name == symbol { + return Some(Symbol::non_terminal(i)) + } + } + + for (i, external_token) in self.grammar.external_tokens.iter().enumerate() { + if let Rule::NamedSymbol(name) = external_token { + if name == symbol { + return Some(Symbol::external(i)) + } + } + } + + return None + } +} + +fn symbol_error(name: &str) -> Error { + Error::SymbolError(format!("Undefined symbol '{}'", name)) +} + +fn variable_type_for_name(name: &str) -> VariableType { + if name.starts_with("_") { + VariableType::Hidden + } else { + VariableType::Named + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basic_repeat_expansion() { + let grammar = intern_symbols(&build_grammar(vec![ + InputVariable::named("x", Rule::choice(vec![ + Rule::named("y"), + Rule::named("_z"), + ])), + InputVariable::named("y", Rule::named("_z")), + InputVariable::named("_z", Rule::string("a")), + ])).unwrap(); + + assert_eq!(grammar.variables, vec![ + InputVariable::named("x", Rule::choice(vec![ + Rule::non_terminal(1), + Rule::non_terminal(2), + ])), + InputVariable::named("y", Rule::non_terminal(2)), + InputVariable::hidden("_z", Rule::string("a")), + ]); + } + + #[test] + fn test_interning_external_token_names() { + // Variable `y` is both an internal and an external token. + // Variable `z` is just an external token. + let mut input_grammar = build_grammar(vec![ + InputVariable::named("w", Rule::choice(vec![ + Rule::named("x"), + Rule::named("y"), + Rule::named("z"), + ])), + InputVariable::named("x", Rule::string("a")), + InputVariable::named("y", Rule::string("b")), + ]); + input_grammar.external_tokens.extend(vec![ + Rule::named("y"), + Rule::named("z"), + ]); + + let grammar = intern_symbols(&input_grammar).unwrap(); + + // Variable `y` is referred to by its internal index. + // Variable `z` is referred to by its external index. + assert_eq!(grammar.variables, vec![ + InputVariable::named("w", Rule::choice(vec![ + Rule::non_terminal(1), + Rule::non_terminal(2), + Rule::external(1), + ])), + InputVariable::named("x", Rule::string("a")), + InputVariable::named("y", Rule::string("b")), + ]); + + // The external token for `y` refers back to its internal index. + assert_eq!(grammar.external_tokens, vec![ + InputVariable::named("y", Rule::non_terminal(2)), + InputVariable::named("z", Rule::external(1)), + ]); + } + + #[test] + fn test_grammar_with_undefined_symbols() { + let result = intern_symbols(&build_grammar(vec![ + InputVariable::named("x", Rule::named("y")), + ])); + + match result { + Err(Error::SymbolError(message)) => assert_eq!(message, "Undefined symbol 'y'"), + _ => panic!("Expected an error but got none"), + } + } + + fn build_grammar(variables: Vec) -> InputGrammar { + InputGrammar { + variables, + name: "the_language".to_string(), + extra_tokens: Vec::new(), + external_tokens: Vec::new(), + expected_conflicts: Vec::new(), + variables_to_inline: Vec::new(), + word_token: None, + } + } +} diff --git a/src/prepare_grammar/mod.rs b/src/prepare_grammar/mod.rs new file mode 100644 index 000000000..0788edcaf --- /dev/null +++ b/src/prepare_grammar/mod.rs @@ -0,0 +1,40 @@ +mod intern_symbols; +mod extract_tokens; +mod expand_repeats; +mod flatten_grammar; +mod normalize_rules; +mod extract_simple_aliases; + +use crate::rules::{AliasMap, Rule, Symbol}; +use crate::grammars::{InputGrammar, SyntaxGrammar, LexicalGrammar, InputVariable, ExternalToken}; +use crate::error::Result; +use self::intern_symbols::intern_symbols; +use self::extract_tokens::extract_tokens; +use self::expand_repeats::expand_repeats; +use self::flatten_grammar::flatten_grammar; +use self::normalize_rules::normalize_rules; +use self::extract_simple_aliases::extract_simple_aliases; + +pub(self) struct IntermediateGrammar { + variables: Vec, + extra_tokens: Vec, + expected_conflicts: Vec>, + external_tokens: Vec, + variables_to_inline: Vec, + word_token: Option, +} + +pub(self) type InternedGrammar = IntermediateGrammar; +pub(self) type ExtractedGrammar = IntermediateGrammar; + +pub fn prepare_grammar( + input_grammar: &InputGrammar +) -> Result<(SyntaxGrammar, LexicalGrammar, AliasMap)> { + let interned_grammar = intern_symbols(input_grammar)?; + let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?; + let syntax_grammar = expand_repeats(syntax_grammar); + let mut syntax_grammar = flatten_grammar(syntax_grammar)?; + let mut lexical_grammar = normalize_rules(lexical_grammar); + let simple_aliases = extract_simple_aliases(&mut syntax_grammar, &mut lexical_grammar); + Ok((syntax_grammar, lexical_grammar, simple_aliases)) +} diff --git a/src/prepare_grammar/normalize_rules.rs b/src/prepare_grammar/normalize_rules.rs new file mode 100644 index 000000000..9e625ef58 --- /dev/null +++ b/src/prepare_grammar/normalize_rules.rs @@ -0,0 +1,5 @@ +use crate::grammars::LexicalGrammar; + +pub(super) fn normalize_rules(grammar: LexicalGrammar) -> LexicalGrammar { + unimplemented!(); +} diff --git a/src/render/mod.rs b/src/render/mod.rs new file mode 100644 index 000000000..85ce1f327 --- /dev/null +++ b/src/render/mod.rs @@ -0,0 +1,16 @@ +use crate::rules::{Symbol, AliasMap}; +use crate::grammars::{SyntaxGrammar, LexicalGrammar}; +use crate::tables::{ParseTable, LexTable}; + +pub fn render_c_code( + name: &str, + parse_table: ParseTable, + main_lex_table: LexTable, + keyword_lex_table: LexTable, + keyword_capture_token: Option, + syntax_grammar: SyntaxGrammar, + lexical_grammar: LexicalGrammar, + simple_aliases: AliasMap, +) -> String { + unimplemented!(); +} diff --git a/src/rules.rs b/src/rules.rs new file mode 100644 index 000000000..3cccca0d2 --- /dev/null +++ b/src/rules.rs @@ -0,0 +1,205 @@ +use std::rc::Rc; +use std::collections::HashMap; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum SymbolType { + External, + Terminal, + NonTerminal, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Associativity { + Left, + Right +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Alias { + value: String, + is_named: bool, +} + +pub type AliasMap = HashMap; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub struct MetadataParams { + precedence: Option, + dynamic_precedence: i32, + associativity: Option, + is_token: bool, + is_string: bool, + is_active: bool, + is_main_token: bool, + is_excluded: bool, + alias: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Symbol { + kind: SymbolType, + index: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Rule { + Blank, + CharacterSet(Vec), + String(String), + Pattern(String), + NamedSymbol(String), + Symbol(Symbol), + Choice { + elements: Vec, + }, + Metadata { + params: MetadataParams, + rule: Rc, + }, + Repeat(Rc), + Seq { + left: Rc, + right: Rc, + } +} + +impl Rule { + pub fn token(content: Rule) -> Self { + add_metadata(content, |params| { + params.is_token = true; + }) + } + + pub fn immediate_token(content: Rule) -> Self { + add_metadata(content, |params| { + params.is_token = true; + params.is_main_token = true; + }) + } + + pub fn prec(value: i32, content: Rule) -> Self { + add_metadata(content, |params| { + params.precedence = Some(value); + }) + } + + pub fn prec_left(value: i32, content: Rule) -> Self { + add_metadata(content, |params| { + params.associativity = Some(Associativity::Left); + params.precedence = Some(value); + }) + } + + pub fn prec_right(value: i32, content: Rule) -> Self { + add_metadata(content, |params| { + params.associativity = Some(Associativity::Right); + params.precedence = Some(value); + }) + } + + pub fn repeat(rule: Rule) -> Self { + Rule::Repeat(Rc::new(rule)) + } + + pub fn choice(rules: Vec) -> Self { + let mut elements = Vec::with_capacity(rules.len()); + for rule in rules { + choice_helper(&mut elements, rule); + } + Rule::Choice { elements } + } + + pub fn seq(rules: Vec) -> Self { + let mut result = Rule::Blank; + for rule in rules { + match rule { + Rule::Blank => continue, + Rule::Metadata { rule, params: _ } => { + if *rule == Rule::Blank { + continue; + } + }, + _ => { + if result == Rule::Blank { + result = rule; + } else { + result = Rule::Seq { + left: Rc::new(result), + right: Rc::new(rule), + } + } + } + } + } + result + } + + pub fn terminal(index: usize) -> Self { + Rule::Symbol(Symbol::terminal(index)) + } + + pub fn non_terminal(index: usize) -> Self { + Rule::Symbol(Symbol::non_terminal(index)) + } + + pub fn external(index: usize) -> Self { + Rule::Symbol(Symbol::external(index)) + } + + pub fn named(name: &'static str) -> Self { + Rule::NamedSymbol(name.to_string()) + } + + pub fn string(value: &'static str) -> Self { + Rule::String(value.to_string()) + } +} + +impl Symbol { + pub fn non_terminal(index: usize) -> Self { + Symbol { kind: SymbolType::NonTerminal, index } + } + + pub fn terminal(index: usize) -> Self { + Symbol { kind: SymbolType::Terminal, index } + } + + pub fn external(index: usize) -> Self { + Symbol { kind: SymbolType::External, index } + } +} + +impl From for Rule { + fn from(symbol: Symbol) -> Self { + Rule::Symbol(symbol) + } +} + +fn add_metadata(input: Rule, f: T) -> Rule { + match input { + Rule::Metadata { rule, mut params } => { + f(&mut params); + Rule::Metadata { rule, params } + }, + _ => { + let mut params = MetadataParams::default(); + f(&mut params); + Rule::Metadata { rule: Rc::new(input), params } + } + } +} + +fn choice_helper(result: &mut Vec, rule: Rule) { + match rule { + Rule::Choice {elements} => { + for element in elements { + choice_helper(result, element); + } + }, + _ => { + if !result.contains(&rule) { + result.push(rule); + } + } + } +} diff --git a/src/tables.rs b/src/tables.rs new file mode 100644 index 000000000..10b1e41d0 --- /dev/null +++ b/src/tables.rs @@ -0,0 +1,77 @@ +use std::collections::HashMap; +use std::ops::Range; +use crate::rules::{Associativity, Symbol, Alias}; + +pub type AliasSequenceId = usize; +pub type ParseStateId = usize; +pub type LexStateId = usize; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParseActionType { + Error, + Shift, + Reduce, + Accept, + Recover, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParseAction { + Accept, + Error, + Shift(ParseStateId), + ShiftExtra, + Recover, + Reduce { + symbol: Symbol, + child_count: usize, + precedence: i32, + dynamic_precedence: i32, + associativity: Option, + alias_sequence_id: Option, + is_repetition: bool, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParseTableEntry { + actions: Vec, + reusable: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParseState { + terminal_entries: HashMap, + nonterminal_entries: HashMap +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ParseTable { + states: Vec, + alias_sequences: Vec>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdvanceAction { + state: LexStateId, + precedence: Range, + in_main_token: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AcceptTokenAction { + symbol: Symbol, + precedence: i32, + implicit_precedence: i32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LexState { + advance_actions: HashMap, + accept_action: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct LexTable { + states: Vec, +}