0.25.8

(cherry picked from commit 854f527f6e)

.github/workflows/build.yml

@@ -195,6 +195,13 @@ jobs:
         npm run build
         npm run build:debug
 
+    - name: Check no_std builds
+      if: ${{ !matrix.no-run && inputs.run-test }}
+      shell: bash
+      run: |
+        cd lib
+        $BUILD_CMD check --no-default-features
+
     - name: Build target
       run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }}
 

.github/workflows/ci.yml

@@ -32,11 +32,6 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: stable
-
-      - name: Set up nightly Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: nightly
           components: clippy, rustfmt
 
       - name: Lint files

Cargo.toml

@@ -3,6 +3,7 @@ default-members = ["cli"]
 members = [
   "cli",
   "cli/config",
+  "cli/generate",
   "cli/loader",
   "lib",
   "lib/language",
@@ -13,7 +14,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.25.1"
+version = "0.25.8"
 authors = [
   "Max Brunsfeld <maxbrunsfeld@gmail.com>",
   "Amaan Qureshi <amaanq12@gmail.com>",
@@ -59,6 +60,8 @@ missing_errors_doc = "allow"
 missing_panics_doc = "allow"
 module_name_repetitions = "allow"
 multiple_crate_versions = "allow"
+needless_for_each = "allow"
+obfuscated_if_else = "allow"
 option_if_let_else = "allow"
 or_fun_call = "allow"
 range_plus_one = "allow"
@@ -143,6 +146,7 @@ tempfile = "3.15.0"
 thiserror = "2.0.11"
 tiny_http = "0.12.0"
 toml = "0.8.19"
+topological-sort = "0.2.2"
 unindent = "0.2.3"
 url = { version = "2.5.4", features = ["serde"] }
 walkdir = "2.5.0"

Makefile

@@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT)
 $(error Windows is not supported)
 endif
 
-VERSION := 0.25.1
+VERSION := 0.25.8
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -106,15 +106,15 @@ test-wasm:
 lint:
 	cargo update --workspace --locked --quiet
 	cargo check --workspace --all-targets
-	cargo +nightly fmt --all --check
-	cargo +nightly clippy --workspace --all-targets -- -D warnings
+	cargo fmt --all --check
+	cargo clippy --workspace --all-targets -- -D warnings
 
 lint-web:
 	npm --prefix lib/binding_web ci
 	npm --prefix lib/binding_web run lint
 
 format:
-	cargo +nightly fmt --all
+	cargo fmt --all
 
 changelog:
 	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)

build.zig.zon

@@ -1,6 +1,6 @@
 .{
   .name = "tree-sitter",
-  .version = "0.25.1",
+  .version = "0.25.8",
   .paths = .{
     "build.zig",
     "build.zig.zon",

cli/Cargo.toml

@@ -59,6 +59,7 @@ similar.workspace = true
 smallbitvec.workspace = true
 streaming-iterator.workspace = true
 tiny_http.workspace = true
+topological-sort.workspace = true
 url.workspace = true
 walkdir.workspace = true
 wasmparser.workspace = true

cli/benches/benchmark.rs

@@ -112,7 +112,7 @@ fn main() {
 
             parse(path, max_path_length, |source| {
                 Query::new(&language, str::from_utf8(source).unwrap())
-                    .with_context(|| format!("Query file path: {path:?}"))
+                    .with_context(|| format!("Query file path: {}", path.display()))
                     .expect("Failed to parse query");
             });
         }
@@ -201,7 +201,7 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
     );
 
     let source_code = fs::read(path)
-        .with_context(|| format!("Failed to read {path:?}"))
+        .with_context(|| format!("Failed to read {}", path.display()))
         .unwrap();
     let time = Instant::now();
     for _ in 0..*REPETITION_COUNT {
@@ -221,6 +221,6 @@ fn get_language(path: &Path) -> Language {
     let src_path = GRAMMARS_DIR.join(path).join("src");
     TEST_LOADER
         .load_language_at_path(CompileConfig::new(&src_path, None, None))
-        .with_context(|| format!("Failed to load language at path {src_path:?}"))
+        .with_context(|| format!("Failed to load language at path {}", src_path.display()))
         .unwrap()
 }

cli/build.rs

@@ -60,8 +60,6 @@ fn web_playground_files_present() -> bool {
     paths.iter().all(|p| Path::new(p).exists())
 }
 
-// When updating this function, don't forget to also update generate/build.rs which has a
-// near-identical function.
 fn read_git_sha() -> Option<String> {
     let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
 

cli/generate/Cargo.toml

@@ -29,6 +29,9 @@ serde.workspace = true
 serde_json.workspace = true
 smallbitvec.workspace = true
 thiserror.workspace = true
-url.workspace = true
+topological-sort.workspace = true
 
 tree-sitter.workspace = true
+
+[target.'cfg(windows)'.dependencies]
+url.workspace = true

cli/generate/build.rs

@@ -1,32 +0,0 @@
-use std::{env, path::PathBuf, process::Command};
-
-fn main() {
-    if let Some(git_sha) = read_git_sha() {
-        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
-    }
-}
-
-// This is copied from the build.rs in parent directory. This should be updated if the
-// parent build.rs gets fixes.
-fn read_git_sha() -> Option<String> {
-    let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
-
-    if !crate_path
-        .parent()?
-        .parent()
-        .is_some_and(|p| p.join(".git").exists())
-    {
-        return None;
-    }
-
-    Command::new("git")
-        .args(["rev-parse", "HEAD"])
-        .current_dir(crate_path)
-        .output()
-        .map_or(None, |output| {
-            if !output.status.success() {
-                return None;
-            }
-            Some(String::from_utf8_lossy(&output.stdout).to_string())
-        })
-}

cli/generate/src/build_tables/build_parse_table.rs

@@ -908,7 +908,7 @@ impl<'a> ParseTableBuilder<'a> {
 
         let get_rule_names = |items: &[&ParseItem]| -> Vec<String> {
             let mut last_rule_id = None;
-            let mut result = Vec::new();
+            let mut result = Vec::with_capacity(items.len());
             for item in items {
                 if last_rule_id == Some(item.variable_index) {
                     continue;

cli/generate/src/lib.rs

@@ -27,7 +27,7 @@ mod tables;
 use build_tables::build_tables;
 pub use build_tables::ParseTableBuilderError;
 use grammars::InputGrammar;
-pub use node_types::VariableInfoError;
+pub use node_types::{SuperTypeCycleError, VariableInfoError};
 use parse_grammar::parse_grammar;
 pub use parse_grammar::ParseGrammarError;
 use prepare_grammar::prepare_grammar;
@@ -70,6 +70,8 @@ pub enum GenerateError {
     BuildTables(#[from] ParseTableBuilderError),
     #[error(transparent)]
     ParseVersion(#[from] ParseVersionError),
+    #[error(transparent)]
+    SuperTypeCycle(#[from] SuperTypeCycleError),
 }
 
 impl From<std::io::Error> for GenerateError {
@@ -183,7 +185,8 @@ pub fn generate_parser_in_directory(
     if grammar_path.file_name().unwrap() != "grammar.json" {
         fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| {
             GenerateError::IO(format!(
-                "Failed to write grammar.json to {src_path:?} -- {e}"
+                "Failed to write grammar.json to {} -- {e}",
+                src_path.display()
             ))
         })?;
     }
@@ -249,7 +252,7 @@ fn generate_parser_for_grammar_with_opts(
         &lexical_grammar,
         &simple_aliases,
         &variable_info,
-    );
+    )?;
     let supertype_symbol_map =
         node_types::get_supertype_symbol_map(&syntax_grammar, &simple_aliases, &variable_info);
     let tables = build_tables(

cli/generate/src/node_types.rs

@@ -1,7 +1,4 @@
-use std::{
-    cmp::Ordering,
-    collections::{BTreeMap, HashMap, HashSet},
-};
+use std::collections::{BTreeMap, HashMap, HashSet};
 
 use anyhow::Result;
 use serde::Serialize;
@@ -444,12 +441,33 @@ pub fn get_supertype_symbol_map(
     supertype_symbol_map
 }
 
+pub type SuperTypeCycleResult<T> = Result<T, SuperTypeCycleError>;
+
+#[derive(Debug, Error, Serialize)]
+pub struct SuperTypeCycleError {
+    items: Vec<String>,
+}
+
+impl std::fmt::Display for SuperTypeCycleError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Dependency cycle detected in node types:")?;
+        for (i, item) in self.items.iter().enumerate() {
+            write!(f, " {item}")?;
+            if i < self.items.len() - 1 {
+                write!(f, ",")?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
 pub fn generate_node_types_json(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     default_aliases: &AliasMap,
     variable_info: &[VariableInfo],
-) -> Vec<NodeInfoJSON> {
+) -> SuperTypeCycleResult<Vec<NodeInfoJSON>> {
     let mut node_types_json = BTreeMap::new();
 
     let child_type_to_node_type = |child_type: &ChildType| match child_type {
@@ -507,6 +525,31 @@ pub fn generate_node_types_json(
 
     let aliases_by_symbol = get_aliases_by_symbol(syntax_grammar, default_aliases);
 
+    let empty = HashSet::new();
+    let extra_names = syntax_grammar
+        .extra_symbols
+        .iter()
+        .flat_map(|symbol| {
+            aliases_by_symbol
+                .get(symbol)
+                .unwrap_or(&empty)
+                .iter()
+                .map(|alias| {
+                    alias.as_ref().map_or(
+                        match symbol.kind {
+                            SymbolType::NonTerminal => &syntax_grammar.variables[symbol.index].name,
+                            SymbolType::Terminal => &lexical_grammar.variables[symbol.index].name,
+                            SymbolType::External => {
+                                &syntax_grammar.external_tokens[symbol.index].name
+                            }
+                            _ => unreachable!(),
+                        },
+                        |alias| &alias.value,
+                    )
+                })
+        })
+        .collect::<HashSet<_>>();
+
     let mut subtype_map = Vec::new();
     for (i, info) in variable_info.iter().enumerate() {
         let symbol = Symbol::non_terminal(i);
@@ -519,7 +562,7 @@ pub fn generate_node_types_json(
                         kind: variable.name.clone(),
                         named: true,
                         root: false,
-                        extra: false,
+                        extra: extra_names.contains(&variable.name),
                         fields: None,
                         children: None,
                         subtypes: None,
@@ -563,7 +606,7 @@ pub fn generate_node_types_json(
                         kind: kind.clone(),
                         named: is_named,
                         root: i == 0,
-                        extra: false,
+                        extra: extra_names.contains(&kind),
                         fields: Some(BTreeMap::new()),
                         children: None,
                         subtypes: None,
@@ -602,15 +645,33 @@ pub fn generate_node_types_json(
         }
     }
 
-    // Sort the subtype map so that subtypes are listed before their supertypes.
-    subtype_map.sort_by(|a, b| {
-        if b.1.contains(&a.0) {
-            Ordering::Less
-        } else if a.1.contains(&b.0) {
-            Ordering::Greater
-        } else {
-            Ordering::Equal
+    // Sort the subtype map topologically so that subtypes are listed before their supertypes.
+    let mut sorted_kinds = Vec::with_capacity(subtype_map.len());
+    let mut top_sort = topological_sort::TopologicalSort::<String>::new();
+    for (supertype, subtypes) in &subtype_map {
+        for subtype in subtypes {
+            top_sort.add_dependency(subtype.kind.clone(), supertype.kind.clone());
         }
+    }
+    loop {
+        let mut next_kinds = top_sort.pop_all();
+        match (next_kinds.is_empty(), top_sort.is_empty()) {
+            (true, true) => break,
+            (true, false) => {
+                let mut items = top_sort.collect::<Vec<String>>();
+                items.sort();
+                return Err(SuperTypeCycleError { items });
+            }
+            (false, _) => {
+                next_kinds.sort();
+                sorted_kinds.extend(next_kinds);
+            }
+        }
+    }
+    subtype_map.sort_by(|a, b| {
+        let a_idx = sorted_kinds.iter().position(|n| n.eq(&a.0.kind)).unwrap();
+        let b_idx = sorted_kinds.iter().position(|n| n.eq(&b.0.kind)).unwrap();
+        a_idx.cmp(&b_idx)
     });
 
     for node_type_json in node_types_json.values_mut() {
@@ -634,7 +695,6 @@ pub fn generate_node_types_json(
 
     let mut anonymous_node_types = Vec::new();
 
-    let empty = HashSet::new();
     let regular_tokens = lexical_grammar
         .variables
         .iter()
@@ -668,29 +728,6 @@ pub fn generate_node_types_json(
                         })
                     })
             });
-    let extra_names = syntax_grammar
-        .extra_symbols
-        .iter()
-        .flat_map(|symbol| {
-            aliases_by_symbol
-                .get(symbol)
-                .unwrap_or(&empty)
-                .iter()
-                .map(|alias| {
-                    alias.as_ref().map_or(
-                        match symbol.kind {
-                            SymbolType::NonTerminal => &syntax_grammar.variables[symbol.index].name,
-                            SymbolType::Terminal => &lexical_grammar.variables[symbol.index].name,
-                            SymbolType::External => {
-                                &syntax_grammar.external_tokens[symbol.index].name
-                            }
-                            _ => unreachable!(),
-                        },
-                        |alias| &alias.value,
-                    )
-                })
-        })
-        .collect::<HashSet<_>>();
 
     for (name, kind) in regular_tokens.chain(external_tokens) {
         match kind {
@@ -743,7 +780,7 @@ pub fn generate_node_types_json(
             .then_with(|| a.kind.cmp(&b.kind))
     });
     result.dedup();
-    result
+    Ok(result)
 }
 
 fn process_supertypes(info: &mut FieldInfoJSON, subtype_map: &[(NodeTypeJSON, Vec<NodeTypeJSON>)]) {
@@ -829,7 +866,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(node_types.len(), 3);
 
@@ -918,7 +956,9 @@ mod tests {
                 },
                 // This rule is not reachable from the start symbol, but
                 // it is reachable from the 'extra_symbols' so it
-                // should be present in the node_types
+                // should be present in the node_types.
+                // But because it's only a literal, it will get replaced by
+                // a lexical variable.
                 Variable {
                     name: "v3".to_string(),
                     kind: VariableType::Named,
@@ -926,7 +966,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(node_types.len(), 4);
 
@@ -1007,6 +1048,118 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_node_types_deeper_extras() {
+        let node_types = get_node_types(&InputGrammar {
+            extra_symbols: vec![Rule::named("v3")],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::seq(vec![
+                        Rule::field("f1".to_string(), Rule::named("v2")),
+                        Rule::field("f2".to_string(), Rule::string(";")),
+                    ]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                // This rule is not reachable from the start symbol, but
+                // it is reachable from the 'extra_symbols' so it
+                // should be present in the node_types.
+                // Because it is not just a literal, it won't get replaced
+                // by a lexical variable.
+                Variable {
+                    name: "v3".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::seq(vec![Rule::string("y"), Rule::repeat(Rule::string("z"))]),
+                },
+            ],
+            ..Default::default()
+        })
+        .unwrap();
+
+        assert_eq!(node_types.len(), 6);
+
+        assert_eq!(
+            node_types[0],
+            NodeInfoJSON {
+                kind: "v1".to_string(),
+                named: true,
+                root: true,
+                extra: false,
+                subtypes: None,
+                children: None,
+                fields: Some(
+                    vec![
+                        (
+                            "f1".to_string(),
+                            FieldInfoJSON {
+                                multiple: false,
+                                required: true,
+                                types: vec![NodeTypeJSON {
+                                    kind: "v2".to_string(),
+                                    named: true,
+                                }]
+                            }
+                        ),
+                        (
+                            "f2".to_string(),
+                            FieldInfoJSON {
+                                multiple: false,
+                                required: true,
+                                types: vec![NodeTypeJSON {
+                                    kind: ";".to_string(),
+                                    named: false,
+                                }]
+                            }
+                        ),
+                    ]
+                    .into_iter()
+                    .collect()
+                )
+            }
+        );
+        assert_eq!(
+            node_types[1],
+            NodeInfoJSON {
+                kind: "v3".to_string(),
+                named: true,
+                root: false,
+                extra: true,
+                subtypes: None,
+                children: None,
+                fields: Some(BTreeMap::default())
+            }
+        );
+        assert_eq!(
+            node_types[2],
+            NodeInfoJSON {
+                kind: ";".to_string(),
+                named: false,
+                root: false,
+                extra: false,
+                subtypes: None,
+                children: None,
+                fields: None
+            }
+        );
+        assert_eq!(
+            node_types[3],
+            NodeInfoJSON {
+                kind: "v2".to_string(),
+                named: true,
+                root: false,
+                extra: false,
+                subtypes: None,
+                children: None,
+                fields: None
+            }
+        );
+    }
+
     #[test]
     fn test_node_types_with_supertypes() {
         let node_types = get_node_types(&InputGrammar {
@@ -1038,7 +1191,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             node_types[0],
@@ -1127,7 +1281,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             node_types[0],
@@ -1212,7 +1367,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             node_types[0],
@@ -1286,7 +1442,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(node_types.iter().find(|t| t.kind == "foo_identifier"), None);
         assert_eq!(
@@ -1342,7 +1499,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             node_types[0],
@@ -1391,7 +1549,8 @@ mod tests {
                 ]),
             }],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             node_types,
@@ -1439,7 +1598,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             &node_types
@@ -1558,7 +1718,8 @@ mod tests {
                 },
             ],
             ..Default::default()
-        });
+        })
+        .unwrap();
 
         assert_eq!(
             node_types.iter().map(|n| &n.kind).collect::<Vec<_>>(),
@@ -1885,7 +2046,7 @@ mod tests {
         );
     }
 
-    fn get_node_types(grammar: &InputGrammar) -> Vec<NodeInfoJSON> {
+    fn get_node_types(grammar: &InputGrammar) -> SuperTypeCycleResult<Vec<NodeInfoJSON>> {
         let (syntax_grammar, lexical_grammar, _, default_aliases) =
             prepare_grammar(grammar).unwrap();
         let variable_info =

cli/generate/src/parse_grammar.rs

@@ -238,13 +238,14 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
     let mut in_progress = HashSet::new();
 
     for (name, rule) in &rules {
-        if !variable_is_used(
-            &rules,
-            &extra_symbols,
-            &external_tokens,
-            name,
-            &mut in_progress,
-        ) && grammar_json.word.as_ref().is_none_or(|w| w != name)
+        if grammar_json.word.as_ref().is_none_or(|w| w != name)
+            && !variable_is_used(
+                &rules,
+                &extra_symbols,
+                &external_tokens,
+                name,
+                &mut in_progress,
+            )
         {
             grammar_json.conflicts.retain(|r| !r.contains(name));
             grammar_json.supertypes.retain(|r| r != name);
@@ -272,12 +273,11 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         .reserved
         .into_iter()
         .map(|(name, rule_values)| {
-            let mut reserved_words = Vec::new();
-
             let Value::Array(rule_values) = rule_values else {
                 Err(ParseGrammarError::InvalidReservedWordSet)?
             };
 
+            let mut reserved_words = Vec::with_capacity(rule_values.len());
             for value in rule_values {
                 reserved_words.push(parse_rule(serde_json::from_value(value)?, false)?);
             }

cli/generate/src/prepare_grammar/expand_tokens.rs

@@ -90,7 +90,7 @@ pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> ExpandTokensResult
         Rule::repeat(Rule::choice(grammar.separators))
     };
 
-    let mut variables = Vec::new();
+    let mut variables = Vec::with_capacity(grammar.variables.len());
     for (i, variable) in grammar.variables.into_iter().enumerate() {
         if variable.rule.is_empty() {
             Err(ExpandTokensError::EmptyString(variable.name.clone()))?;
@@ -195,7 +195,7 @@ impl NfaBuilder {
                 Ok(!s.is_empty())
             }
             Rule::Choice(elements) => {
-                let mut alternative_state_ids = Vec::new();
+                let mut alternative_state_ids = Vec::with_capacity(elements.len());
                 for element in elements {
                     if self.expand_rule(element, next_state_id)? {
                         alternative_state_ids.push(self.nfa.last_state_id());
@@ -338,7 +338,7 @@ impl NfaBuilder {
                 Ok(result)
             }
             HirKind::Alternation(alternations) => {
-                let mut alternative_state_ids = Vec::new();
+                let mut alternative_state_ids = Vec::with_capacity(alternations.len());
                 for hir in alternations {
                     if self.expand_regex(hir, next_state_id)? {
                         alternative_state_ids.push(self.nfa.last_state_id());

cli/generate/src/prepare_grammar/extract_tokens.rs

@@ -26,10 +26,34 @@ unless they are used only as the grammar's start rule.
     ExternalTokenNonTerminal(String),
     #[error("Non-symbol rules cannot be used as external tokens")]
     NonSymbolExternalToken,
-    #[error("Non-terminal symbol '{0}' cannot be used as the word token, because its rule is duplicated in '{1}'")]
-    NonTerminalWordToken(String, String),
-    #[error("Reserved words must be tokens")]
-    NonTokenReservedWord,
+    #[error(transparent)]
+    WordToken(NonTerminalWordTokenError),
+    #[error("Reserved word '{0}' must be a token")]
+    NonTokenReservedWord(String),
+}
+
+#[derive(Debug, Error, Serialize)]
+pub struct NonTerminalWordTokenError {
+    pub symbol_name: String,
+    pub conflicting_symbol_name: Option<String>,
+}
+
+impl std::fmt::Display for NonTerminalWordTokenError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Non-terminal symbol '{}' cannot be used as the word token",
+            self.symbol_name
+        )?;
+        if let Some(conflicting_name) = &self.conflicting_symbol_name {
+            writeln!(
+                f,
+                ", because its rule is duplicated in '{conflicting_name}'",
+            )
+        } else {
+            writeln!(f)
+        }
+    }
 }
 
 pub(super) fn extract_tokens(
@@ -62,7 +86,7 @@ pub(super) fn extract_tokens(
     // that pointed to that variable will need to be updated to point to the
     // variable in the lexical grammar. Symbols that pointed to later variables
     // will need to have their indices decremented.
-    let mut variables = Vec::new();
+    let mut variables = Vec::with_capacity(grammar.variables.len());
     let mut symbol_replacer = SymbolReplacer {
         replacements: HashMap::new(),
     };
@@ -162,23 +186,23 @@ pub(super) fn extract_tokens(
         let token = symbol_replacer.replace_symbol(token);
         if token.is_non_terminal() {
             let word_token_variable = &variables[token.index];
-            let conflicting_variable = variables
+            let conflicting_symbol_name = variables
                 .iter()
                 .enumerate()
                 .find(|(i, v)| *i != token.index && v.rule == word_token_variable.rule)
-                .expect("Failed to find a variable with the same rule as the word token");
+                .map(|(_, v)| v.name.clone());
 
-            Err(ExtractTokensError::NonTerminalWordToken(
-                word_token_variable.name.clone(),
-                conflicting_variable.1.name.clone(),
-            ))?;
+            Err(ExtractTokensError::WordToken(NonTerminalWordTokenError {
+                symbol_name: word_token_variable.name.clone(),
+                conflicting_symbol_name,
+            }))?;
         }
         word_token = Some(token);
     }
 
-    let mut reserved_word_contexts = Vec::new();
+    let mut reserved_word_contexts = Vec::with_capacity(grammar.reserved_word_sets.len());
     for reserved_word_context in grammar.reserved_word_sets {
-        let mut reserved_words = Vec::new();
+        let mut reserved_words = Vec::with_capacity(reserved_word_contexts.len());
         for reserved_rule in reserved_word_context.reserved_words {
             if let Rule::Symbol(symbol) = reserved_rule {
                 reserved_words.push(symbol_replacer.replace_symbol(symbol));
@@ -188,7 +212,12 @@ pub(super) fn extract_tokens(
             {
                 reserved_words.push(Symbol::terminal(index));
             } else {
-                Err(ExtractTokensError::NonTokenReservedWord)?;
+                let token_name = match &reserved_rule {
+                    Rule::String(s) => s.clone(),
+                    Rule::Pattern(p, _) => p.clone(),
+                    _ => "unknown".to_string(),
+                };
+                Err(ExtractTokensError::NonTokenReservedWord(token_name))?;
             }
         }
         reserved_word_contexts.push(ReservedWordContext {

cli/generate/src/prepare_grammar/flatten_grammar.rs

@@ -57,8 +57,9 @@ impl RuleFlattener {
     }
 
     fn flatten_variable(&mut self, variable: Variable) -> FlattenGrammarResult<SyntaxVariable> {
-        let mut productions = Vec::new();
-        for rule in extract_choices(variable.rule) {
+        let choices = extract_choices(variable.rule);
+        let mut productions = Vec::with_capacity(choices.len());
+        for rule in choices {
             let production = self.flatten_rule(rule)?;
             if !productions.contains(&production) {
                 productions.push(production);
@@ -195,7 +196,7 @@ fn extract_choices(rule: Rule) -> Vec<Rule> {
             let mut result = vec![Rule::Blank];
             for element in elements {
                 let extraction = extract_choices(element);
-                let mut next_result = Vec::new();
+                let mut next_result = Vec::with_capacity(result.len());
                 for entry in result {
                     for extraction_entry in &extraction {
                         next_result.push(Rule::Seq(vec![entry.clone(), extraction_entry.clone()]));
@@ -206,7 +207,7 @@ fn extract_choices(rule: Rule) -> Vec<Rule> {
             result
         }
         Rule::Choice(elements) => {
-            let mut result = Vec::new();
+            let mut result = Vec::with_capacity(elements.len());
             for element in elements {
                 for rule in extract_choices(element) {
                     result.push(rule);
@@ -262,9 +263,10 @@ pub(super) fn flatten_grammar(
 
     for (i, variable) in variables.iter().enumerate() {
         let symbol = Symbol::non_terminal(i);
+        let used = symbol_is_used(&variables, symbol);
 
         for production in &variable.productions {
-            if production.steps.is_empty() && symbol_is_used(&variables, symbol) {
+            if used && production.steps.is_empty() {
                 Err(FlattenGrammarError::EmptyString(variable.name.clone()))?;
             }
 
@@ -533,7 +535,7 @@ mod tests {
 
         assert_eq!(
             result.unwrap_err().to_string(),
-            "Rule `test` cannot be inlined because it contains a reference to itself.",
+            "Rule `test` cannot be inlined because it contains a reference to itself",
         );
     }
 }

cli/generate/src/prepare_grammar/intern_symbols.rs

@@ -65,7 +65,7 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> InternSymbolsResult<Inte
 
     let mut reserved_words = Vec::with_capacity(grammar.reserved_words.len());
     for reserved_word_set in &grammar.reserved_words {
-        let mut interned_set = Vec::new();
+        let mut interned_set = Vec::with_capacity(reserved_word_set.reserved_words.len());
         for rule in &reserved_word_set.reserved_words {
             interned_set.push(interner.intern_rule(rule, None)?);
         }
@@ -75,7 +75,7 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> InternSymbolsResult<Inte
         });
     }
 
-    let mut expected_conflicts = Vec::new();
+    let mut expected_conflicts = Vec::with_capacity(grammar.expected_conflicts.len());
     for conflict in &grammar.expected_conflicts {
         let mut interned_conflict = Vec::with_capacity(conflict.len());
         for name in conflict {

cli/generate/src/render.rs

@@ -24,7 +24,6 @@ pub const ABI_VERSION_MIN: usize = 14;
 pub const ABI_VERSION_MAX: usize = tree_sitter::LANGUAGE_VERSION;
 const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
 const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION");
-const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA");
 
 macro_rules! add {
     ($this: tt, $($arg: tt)*) => {{
@@ -322,13 +321,9 @@ impl Generator {
     }
 
     fn add_header(&mut self) {
-        let version = BUILD_SHA.map_or_else(
-            || BUILD_VERSION.to_string(),
-            |build_sha| format!("{BUILD_VERSION} ({build_sha})"),
-        );
         add_line!(
             self,
-            "/* Automatically generated by tree-sitter v{version} */",
+            "/* Automatically @generated by tree-sitter v{BUILD_VERSION} */",
         );
         add_line!(self, "");
     }
@@ -683,12 +678,12 @@ impl Generator {
             &mut next_flat_field_map_index,
         );
 
-        let mut field_map_ids = Vec::new();
+        let mut field_map_ids = Vec::with_capacity(self.parse_table.production_infos.len());
         for production_info in &self.parse_table.production_infos {
             if production_info.field_map.is_empty() {
                 field_map_ids.push((0, 0));
             } else {
-                let mut flat_field_map = Vec::new();
+                let mut flat_field_map = Vec::with_capacity(production_info.field_map.len());
                 for (field_name, locations) in &production_info.field_map {
                     for location in locations {
                         flat_field_map.push((field_name.clone(), *location));
@@ -1111,7 +1106,11 @@ impl Generator {
             return;
         }
 
-        add_line!(self, "const TSCharacterRange {}[] = {{", info.constant_name);
+        add_line!(
+            self,
+            "static const TSCharacterRange {}[] = {{",
+            info.constant_name
+        );
 
         indent!(self);
         for (ix, range) in characters.ranges().enumerate() {
@@ -1351,7 +1350,12 @@ impl Generator {
             indent!(self);
 
             let mut next_table_index = 0;
-            let mut small_state_indices = Vec::new();
+            let mut small_state_indices = Vec::with_capacity(
+                self.parse_table
+                    .states
+                    .len()
+                    .saturating_sub(self.large_state_count),
+            );
             let mut symbols_by_value = HashMap::<(usize, SymbolType), Vec<Symbol>>::new();
             for state in self.parse_table.states.iter().skip(self.large_state_count) {
                 small_state_indices.push(next_table_index);
@@ -1847,11 +1851,11 @@ impl Generator {
                         '\u{007F}' => "DEL",
                         '\u{FEFF}' => "BOM",
                         '\u{0080}'..='\u{FFFF}' => {
-                            result.push_str(&format!("u{:04x}", c as u32));
+                            write!(result, "u{:04x}", c as u32).unwrap();
                             break 'special_chars;
                         }
                         '\u{10000}'..='\u{10FFFF}' => {
-                            result.push_str(&format!("U{:08x}", c as u32));
+                            write!(result, "U{:08x}", c as u32).unwrap();
                             break 'special_chars;
                         }
                         '0'..='9' | 'a'..='z' | 'A'..='Z' | '_' => unreachable!(),
@@ -1882,11 +1886,9 @@ impl Generator {
                 '\r' => result += "\\r",
                 '\t' => result += "\\t",
                 '\0' => result += "\\0",
-                '\u{0001}'..='\u{001f}' => result += &format!("\\x{:02x}", c as u32),
-                '\u{007F}'..='\u{FFFF}' => result += &format!("\\u{:04x}", c as u32),
-                '\u{10000}'..='\u{10FFFF}' => {
-                    result.push_str(&format!("\\U{:08x}", c as u32));
-                }
+                '\u{0001}'..='\u{001f}' => write!(result, "\\x{:02x}", c as u32).unwrap(),
+                '\u{007F}'..='\u{FFFF}' => write!(result, "\\u{:04x}", c as u32).unwrap(),
+                '\u{10000}'..='\u{10FFFF}' => write!(result, "\\U{:08x}", c as u32).unwrap(),
                 _ => result.push(c),
             }
         }

cli/generate/src/rules.rs

@@ -306,7 +306,6 @@ impl Symbol {
 }
 
 impl From<Symbol> for Rule {
-    #[must_use]
     fn from(symbol: Symbol) -> Self {
         Self::Symbol(symbol)
     }

cli/loader/emscripten-version

@@ -1 +1 @@
-4.0.1
+4.0.4

cli/loader/src/lib.rs

@@ -11,6 +11,7 @@ use std::{
     ffi::{OsStr, OsString},
     fs,
     io::{BufRead, BufReader},
+    marker::PhantomData,
     mem,
     path::{Path, PathBuf},
     process::Command,
@@ -18,7 +19,6 @@ use std::{
     time::SystemTime,
 };
 
-#[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))]
 use anyhow::Error;
 use anyhow::{anyhow, Context, Result};
 use etcetera::BaseStrategy as _;
@@ -327,6 +327,7 @@ pub struct LanguageConfiguration<'a> {
     highlight_names: &'a Mutex<Vec<String>>,
     #[cfg(feature = "tree-sitter-highlight")]
     use_all_highlight_names: bool,
+    _phantom: PhantomData<&'a ()>,
 }
 
 pub struct Loader {
@@ -561,8 +562,8 @@ impl Loader {
                 // If multiple language configurations match, then determine which
                 // one to use by applying the configurations' content regexes.
                 else {
-                    let file_contents =
-                        fs::read(path).with_context(|| format!("Failed to read path {path:?}"))?;
+                    let file_contents = fs::read(path)
+                        .with_context(|| format!("Failed to read path {}", path.display()))?;
                     let file_contents = String::from_utf8_lossy(&file_contents);
                     let mut best_score = -2isize;
                     let mut best_configuration_id = None;
@@ -780,8 +781,8 @@ impl Loader {
         if recompile {
             fs::create_dir_all(lock_path.parent().unwrap()).with_context(|| {
                 format!(
-                    "Failed to create directory {:?}",
-                    lock_path.parent().unwrap()
+                    "Failed to create directory {}",
+                    lock_path.parent().unwrap().display()
                 )
             })?;
             let lock_file = fs::OpenOptions::new()
@@ -799,7 +800,7 @@ impl Loader {
         }
 
         let library = unsafe { Library::new(&output_path) }
-            .with_context(|| format!("Error opening dynamic library {output_path:?}"))?;
+            .with_context(|| format!("Error opening dynamic library {}", output_path.display()))?;
         let language = unsafe {
             let language_fn = library
                 .get::<Symbol<unsafe extern "C" fn() -> Language>>(language_fn_name.as_bytes())
@@ -1214,6 +1215,7 @@ impl Loader {
                     highlight_names: &self.highlight_names,
                     #[cfg(feature = "tree-sitter-highlight")]
                     use_all_highlight_names: self.use_all_highlight_names,
+                    _phantom: PhantomData,
                 };
 
                 for file_type in &configuration.file_types {
@@ -1283,6 +1285,7 @@ impl Loader {
                 highlight_names: &self.highlight_names,
                 #[cfg(feature = "tree-sitter-highlight")]
                 use_all_highlight_names: self.use_all_highlight_names,
+                _phantom: PhantomData,
             };
             self.language_configurations.push(unsafe {
                 mem::transmute::<LanguageConfiguration<'_>, LanguageConfiguration<'static>>(
@@ -1564,7 +1567,7 @@ impl LanguageConfiguration<'_> {
         error.row = source[range.start..offset_within_section]
             .matches('\n')
             .count();
-        Error::from(error).context(format!("Error in query file {path:?}"))
+        Error::from(error).context(format!("Error in query file {}", path.display()))
     }
 
     #[allow(clippy::type_complexity)]
@@ -1581,7 +1584,7 @@ impl LanguageConfiguration<'_> {
                 let abs_path = self.root_path.join(path);
                 let prev_query_len = query.len();
                 query += &fs::read_to_string(&abs_path)
-                    .with_context(|| format!("Failed to read query file {path:?}"))?;
+                    .with_context(|| format!("Failed to read query file {}", path.display()))?;
                 path_ranges.push((path.clone(), prev_query_len..query.len()));
             }
         } else {
@@ -1599,7 +1602,7 @@ impl LanguageConfiguration<'_> {
             let path = queries_path.join(default_path);
             if path.exists() {
                 query = fs::read_to_string(&path)
-                    .with_context(|| format!("Failed to read query file {path:?}"))?;
+                    .with_context(|| format!("Failed to read query file {}", path.display()))?;
                 path_ranges.push((PathBuf::from(default_path), 0..query.len()));
             }
         }
@@ -1612,8 +1615,8 @@ fn needs_recompile(lib_path: &Path, paths_to_check: &[PathBuf]) -> Result<bool>
     if !lib_path.exists() {
         return Ok(true);
     }
-    let lib_mtime =
-        mtime(lib_path).with_context(|| format!("Failed to read mtime of {lib_path:?}"))?;
+    let lib_mtime = mtime(lib_path)
+        .with_context(|| format!("Failed to read mtime of {}", lib_path.display()))?;
     for path in paths_to_check {
         if mtime(path)? > lib_mtime {
             return Ok(true);

cli/npm/install.js

@@ -6,7 +6,8 @@ const http = require('http');
 const https = require('https');
 const packageJSON = require('./package.json');
 
-// Look to a results table in https://github.com/tree-sitter/tree-sitter/issues/2196
+https.globalAgent.keepAlive = false;
+
 const matrix = {
   platform: {
     'darwin': {

cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.25.1",
+  "version": "0.25.8",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

cli/src/fuzz/allocations.rs

@@ -109,7 +109,7 @@ unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_
     let result = realloc(ptr, size);
     if ptr.is_null() {
         record_alloc(result);
-    } else if ptr != result {
+    } else if !core::ptr::eq(ptr, result) {
         record_dealloc(ptr);
         record_alloc(result);
     }

cli/src/fuzz/mod.rs

@@ -56,7 +56,9 @@ fn regex_env_var(name: &'static str) -> Option<Regex> {
 pub fn new_seed() -> usize {
     int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
         let mut rng = rand::thread_rng();
-        rng.gen::<usize>()
+        let seed = rng.gen::<usize>();
+        eprintln!("Seed: {seed}");
+        seed
     })
 }
 
@@ -213,8 +215,9 @@ pub fn fuzz_language_corpus(
                 }
 
                 // Perform a random series of edits and reparse.
-                let mut undo_stack = Vec::new();
-                for _ in 0..=rand.unsigned(*EDIT_COUNT) {
+                let edit_count = rand.unsigned(*EDIT_COUNT);
+                let mut undo_stack = Vec::with_capacity(edit_count);
+                for _ in 0..=edit_count {
                     let edit = get_random_edit(&mut rand, &input);
                     undo_stack.push(invert_edit(&input, &edit));
                     perform_edit(&mut tree, &mut input, &edit).unwrap();

cli/src/fuzz/random.rs

@@ -20,8 +20,8 @@ impl Rand {
     }
 
     pub fn words(&mut self, max_count: usize) -> Vec<u8> {
-        let mut result = Vec::new();
         let word_count = self.unsigned(max_count);
+        let mut result = Vec::with_capacity(2 * word_count);
         for i in 0..word_count {
             if i > 0 {
                 if self.unsigned(5) == 0 {

cli/src/highlight.rs

@@ -1,5 +1,5 @@
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeMap, HashSet},
     fmt::Write,
     fs,
     io::{self, Write as _},
@@ -82,9 +82,9 @@ impl<'de> Deserialize<'de> for Theme {
     {
         let mut styles = Vec::new();
         let mut highlight_names = Vec::new();
-        if let Ok(colors) = HashMap::<String, Value>::deserialize(deserializer) {
-            highlight_names.reserve(colors.len());
+        if let Ok(colors) = BTreeMap::<String, Value>::deserialize(deserializer) {
             styles.reserve(colors.len());
+            highlight_names.reserve(colors.len());
             for (name, style_value) in colors {
                 let mut style = Style::default();
                 parse_style(&mut style, style_value);
@@ -127,7 +127,7 @@ impl Serialize for Theme {
                 || effects.contains(Effects::ITALIC)
                 || effects.contains(Effects::UNDERLINE)
             {
-                let mut style_json = HashMap::new();
+                let mut style_json = BTreeMap::new();
                 if let Some(color) = color {
                     style_json.insert("color", color);
                 }

cli/src/init.rs

@@ -679,10 +679,13 @@ pub fn generate_grammar_files(
                 |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
-                    contents = contents.replace(
-                        "https://github.com/ChimeHQ/SwiftTreeSitter",
-                        "https://github.com/tree-sitter/swift-tree-sitter",
-                    );
+                    contents = contents
+                        .replace(
+                            "https://github.com/ChimeHQ/SwiftTreeSitter",
+                            "https://github.com/tree-sitter/swift-tree-sitter",
+                        )
+                        .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
+                        .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
                     write_file(path, contents)?;
                     Ok(())
                 },

cli/src/input.rs

@@ -89,8 +89,8 @@ pub fn get_input(
                 let Some(path_str) = path.to_str() else {
                     bail!("Invalid path: {}", path.display());
                 };
-                let paths =
-                    glob(path_str).with_context(|| format!("Invalid glob pattern {path:?}"))?;
+                let paths = glob(path_str)
+                    .with_context(|| format!("Invalid glob pattern {}", path.display()))?;
                 for path in paths {
                     incorporate_path(path?, positive);
                 }

cli/src/main.rs

@@ -206,7 +206,8 @@ struct Parse {
     #[arg(long, short)]
     pub quiet: bool,
     #[allow(clippy::doc_markdown)]
-    /// Apply edits in the format: \"row, col delcount insert_text\"
+    /// Apply edits in the format: \"row,col|position delcount insert_text\", can be supplied
+    /// multiple times
     #[arg(
         long,
         num_args = 1..,

cli/src/parse.rs

@@ -29,18 +29,28 @@ pub struct Stats {
 impl fmt::Display for Stats {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let duration_us = self.total_duration.as_micros();
+        let success_rate = if self.total_parses > 0 {
+            format!(
+                "{:.2}%",
+                ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0,
+            )
+        } else {
+            "N/A".to_string()
+        };
+        let duration_str = match (self.total_parses, duration_us) {
+            (0, _) => "N/A".to_string(),
+            (_, 0) => "0 bytes/ms".to_string(),
+            (_, _) => format!(
+                "{} bytes/ms",
+                ((self.total_bytes as u128) * 1_000) / duration_us
+            ),
+        };
         writeln!(
             f,
-            "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {:.2}%; average speed: {} bytes/ms",
+            "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {success_rate}; average speed: {duration_str}",
             self.total_parses,
             self.successful_parses,
             self.total_parses - self.successful_parses,
-            ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0,
-            if duration_us != 0 {
-                ((self.total_bytes as u128) * 1_000) / duration_us
-            } else {
-                0
-            }
         )
     }
 }
@@ -225,7 +235,7 @@ pub struct ParseStats {
     pub cumulative_stats: Stats,
 }
 
-#[derive(Serialize, ValueEnum, Debug, Clone, Default, Eq, PartialEq)]
+#[derive(Serialize, ValueEnum, Debug, Copy, Clone, Default, Eq, PartialEq)]
 pub enum ParseDebugType {
     #[default]
     Quiet,
@@ -273,10 +283,11 @@ pub fn parse_file_at_path(
     }
     // Log to stderr if `--debug` was passed
     else if opts.debug != ParseDebugType::Quiet {
-        let mut curr_version: usize = 0usize;
+        let mut curr_version: usize = 0;
         let use_color = std::env::var("NO_COLOR").map_or(true, |v| v != "1");
-        parser.set_logger(Some(Box::new(|log_type, message| {
-            if opts.debug == ParseDebugType::Normal {
+        let debug = opts.debug;
+        parser.set_logger(Some(Box::new(move |log_type, message| {
+            if debug == ParseDebugType::Normal {
                 if log_type == LogType::Lex {
                     write!(&mut io::stderr(), "  ").unwrap();
                 }
@@ -686,19 +697,23 @@ pub fn parse_file_at_path(
             if let Some(node) = first_error {
                 let start = node.start_position();
                 let end = node.end_position();
+                let mut node_text = String::new();
+                for c in node.kind().chars() {
+                    if let Some(escaped) = escape_invisible(c) {
+                        node_text += escaped;
+                    } else {
+                        node_text.push(c);
+                    }
+                }
                 write!(&mut stdout, "\t(")?;
                 if node.is_missing() {
                     if node.is_named() {
-                        write!(&mut stdout, "MISSING {}", node.kind())?;
+                        write!(&mut stdout, "MISSING {node_text}")?;
                     } else {
-                        write!(
-                            &mut stdout,
-                            "MISSING \"{}\"",
-                            node.kind().replace('\n', "\\n")
-                        )?;
+                        write!(&mut stdout, "MISSING \"{node_text}\"")?;
                     }
                 } else {
-                    write!(&mut stdout, "{}", node.kind())?;
+                    write!(&mut stdout, "{node_text}")?;
                 }
                 write!(
                     &mut stdout,

cli/src/query.rs

@@ -34,7 +34,7 @@ pub fn query_file_at_path(
     let mut stdout = stdout.lock();
 
     let query_source = fs::read_to_string(query_path)
-        .with_context(|| format!("Error reading query file {query_path:?}"))?;
+        .with_context(|| format!("Error reading query file {}", query_path.display()))?;
     let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?;
 
     let mut query_cursor = QueryCursor::new();
@@ -55,7 +55,7 @@ pub fn query_file_at_path(
     }
 
     let source_code =
-        fs::read(path).with_context(|| format!("Error reading source file {path:?}"))?;
+        fs::read(path).with_context(|| format!("Error reading source file {}", path.display()))?;
     let tree = parser.parse(&source_code, None).unwrap();
 
     let start = Instant::now();

cli/src/templates/cmakelists.cmake

@@ -15,6 +15,8 @@ if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$")
     message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer")
 endif()
 
+include(GNUInstallDirs)
+
 find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI")
 
 add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c"
@@ -47,13 +49,11 @@ set_target_properties(tree-sitter-KEBAB_PARSER_NAME
 configure_file(bindings/c/tree-sitter-KEBAB_PARSER_NAME.pc.in
                "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-KEBAB_PARSER_NAME.pc" @ONLY)
 
-include(GNUInstallDirs)
-
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree_sitter"
         DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
         FILES_MATCHING PATTERN "*.h")
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-KEBAB_PARSER_NAME.pc"
-        DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig")
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 install(TARGETS tree-sitter-KEBAB_PARSER_NAME
         LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")
 

cli/src/templates/gitattributes

@@ -37,5 +37,6 @@ Package.swift linguist-generated
 Package.resolved linguist-generated
 
 # Zig bindings
+bindings/zig/* linguist-generated
 build.zig linguist-generated
 build.zig.zon linguist-generated

cli/src/templates/gitignore

@@ -1,13 +1,16 @@
 # Rust artifacts
 target/
+Cargo.lock
 
 # Node artifacts
 build/
 prebuilds/
 node_modules/
+package-lock.json
 
 # Swift artifacts
 .build/
+Package.resolved
 
 # Go artifacts
 _obj/

cli/src/templates/lib.rs

@@ -1,7 +1,7 @@
-//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library.
+//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter] parsing library.
 //!
-//! Typically, you will use the [LANGUAGE][] constant to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
+//! Typically, you will use the [`LANGUAGE`] constant to add this language to a
+//! tree-sitter [`Parser`], and then use the parser to parse some code:
 //!
 //! ```
 //! let code = r#"
@@ -15,7 +15,7 @@
 //! assert!(!tree.root_node().has_error());
 //! ```
 //!
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [`Parser`]: https://docs.rs/tree-sitter/RUST_BINDING_VERSION/tree_sitter/struct.Parser.html
 //! [tree-sitter]: https://tree-sitter.github.io/
 
 use tree_sitter_language::LanguageFn;
@@ -24,12 +24,10 @@ extern "C" {
     fn tree_sitter_PARSER_NAME() -> *const ();
 }
 
-/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar.
-///
-/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
+/// The tree-sitter [`LanguageFn`] for this grammar.
 pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) };
 
-/// The content of the [`node-types.json`][] file for this grammar.
+/// The content of the [`node-types.json`] file for this grammar.
 ///
 /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
 pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");

cli/src/templates/makefile

@@ -77,7 +77,9 @@ install: all
 	install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER)
 	ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR)
 	ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT)
+ifneq ($(wildcard queries/*.scm),)
 	install -m644 queries/*.scm '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME
+endif
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \

cli/src/templates/package.json

@@ -29,15 +29,15 @@
     "*.wasm"
   ],
   "dependencies": {
-    "node-addon-api": "^8.2.1",
-    "node-gyp-build": "^4.8.2"
+    "node-addon-api": "^8.3.1",
+    "node-gyp-build": "^4.8.4"
   },
   "devDependencies": {
     "prebuildify": "^6.0.1",
     "tree-sitter-cli": "^CLI_VERSION"
   },
   "peerDependencies": {
-    "tree-sitter": "^0.21.1"
+    "tree-sitter": "^0.22.4"
   },
   "peerDependenciesMeta": {
     "tree-sitter": {

cli/src/templates/package.swift

@@ -14,7 +14,7 @@ let package = Package(
         .library(name: "PARSER_CLASS_NAME", targets: ["PARSER_CLASS_NAME"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.8.0"),
+        .package(name: "SwiftTreeSitter", url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.9.0"),
     ],
     targets: [
         .target(

cli/src/templates/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=42", "wheel"]
+requires = ["setuptools>=62.4.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]

cli/src/test_highlight.rs

@@ -172,7 +172,7 @@ pub fn iterate_assertions(
             let mut j = i;
             while let (false, Some(highlight)) = (passed, highlights.get(j)) {
                 end_column = position.column + length - 1;
-                if highlight.0.column > end_column {
+                if highlight.0.row >= position.row && highlight.0.column > end_column {
                     break 'highlight_loop;
                 }
 

cli/src/tests/async_context_test.rs

@@ -238,7 +238,7 @@ async fn yield_now() {
     SimpleYieldNow { yielded: false }.await;
 }
 
-pub fn noop_waker() -> Waker {
+pub const fn noop_waker() -> Waker {
     const VTABLE: RawWakerVTable = RawWakerVTable::new(
         // Cloning just returns a new no-op raw waker
         |_| RAW,

cli/src/tests/corpus_test.rs

@@ -23,7 +23,7 @@ use crate::{
 };
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_bash(seed: usize) {
+fn test_corpus_for_bash_language(seed: usize) {
     test_language_corpus(
         "bash",
         seed,
@@ -39,73 +39,77 @@ fn test_corpus_for_bash(seed: usize) {
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_c(seed: usize) {
+fn test_corpus_for_c_language(seed: usize) {
     test_language_corpus("c", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_cpp(seed: usize) {
+fn test_corpus_for_cpp_language(seed: usize) {
     test_language_corpus("cpp", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_embedded_template(seed: usize) {
+fn test_corpus_for_embedded_template_language(seed: usize) {
     test_language_corpus("embedded-template", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_go(seed: usize) {
+fn test_corpus_for_go_language(seed: usize) {
     test_language_corpus("go", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_html(seed: usize) {
+fn test_corpus_for_html_language(seed: usize) {
     test_language_corpus("html", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_java(seed: usize) {
-    test_language_corpus("java", seed, None, None);
+fn test_corpus_for_java_language(seed: usize) {
+    test_language_corpus(
+        "java",
+        seed,
+        Some(&["java - corpus - expressions - switch with unnamed pattern variable"]),
+        None,
+    );
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_javascript(seed: usize) {
+fn test_corpus_for_javascript_language(seed: usize) {
     test_language_corpus("javascript", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_json(seed: usize) {
+fn test_corpus_for_json_language(seed: usize) {
     test_language_corpus("json", seed, None, None);
 }
 
-#[ignore]
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_php(seed: usize) {
-    test_language_corpus("php", seed, None, None);
+fn test_corpus_for_php_language(seed: usize) {
+    test_language_corpus("php", seed, None, Some("php"));
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_python(seed: usize) {
+fn test_corpus_for_python_language(seed: usize) {
     test_language_corpus("python", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_ruby(seed: usize) {
+fn test_corpus_for_ruby_language(seed: usize) {
     test_language_corpus("ruby", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_rust(seed: usize) {
+fn test_corpus_for_rust_language(seed: usize) {
     test_language_corpus("rust", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_typescript(seed: usize) {
+fn test_corpus_for_typescript_language(seed: usize) {
     test_language_corpus("typescript", seed, None, Some("typescript"));
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_tsx(seed: usize) {
+fn test_corpus_for_tsx_language(seed: usize) {
     test_language_corpus("typescript", seed, None, Some("tsx"));
 }
 
@@ -239,8 +243,9 @@ pub fn test_language_corpus(
                 }
 
                 // Perform a random series of edits and reparse.
-                let mut undo_stack = Vec::new();
-                for _ in 0..=rand.unsigned(*EDIT_COUNT) {
+                let edit_count = rand.unsigned(*EDIT_COUNT);
+                let mut undo_stack = Vec::with_capacity(edit_count);
+                for _ in 0..=edit_count {
                     let edit = get_random_edit(&mut rand, &input);
                     undo_stack.push(invert_edit(&input, &edit));
                     perform_edit(&mut tree, &mut input, &edit).unwrap();

cli/src/tests/helpers/allocations.rs

@@ -108,7 +108,7 @@ unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_
     let result = realloc(ptr, size);
     if ptr.is_null() {
         record_alloc(result);
-    } else if ptr != result {
+    } else if !core::ptr::eq(ptr, result) {
         record_dealloc(ptr);
         record_alloc(result);
     }

cli/src/tests/helpers/fixtures.rs

@@ -6,11 +6,13 @@ use std::{
 
 use anyhow::Context;
 use tree_sitter::Language;
-use tree_sitter_generate::{ALLOC_HEADER, ARRAY_HEADER};
+use tree_sitter_generate::{load_grammar_file, ALLOC_HEADER, ARRAY_HEADER};
 use tree_sitter_highlight::HighlightConfiguration;
 use tree_sitter_loader::{CompileConfig, Loader};
 use tree_sitter_tags::TagsConfiguration;
 
+use crate::tests::generate_parser;
+
 include!("./dirs.rs");
 
 static TEST_LOADER: LazyLock<Loader> = LazyLock::new(|| {
@@ -40,6 +42,13 @@ pub fn get_language(name: &str) -> Language {
     TEST_LOADER.load_language_at_path(config).unwrap()
 }
 
+pub fn get_test_fixture_language(name: &str) -> Language {
+    let grammar_dir_path = fixtures_dir().join("test_grammars").join(name);
+    let grammar_json = load_grammar_file(&grammar_dir_path.join("grammar.js"), None).unwrap();
+    let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap();
+    get_test_language(&parser_name, &parser_code, Some(&grammar_dir_path))
+}
+
 pub fn get_language_queries_path(language_name: &str) -> PathBuf {
     GRAMMARS_DIR.join(language_name).join("queries")
 }

cli/src/tests/highlight_test.rs

@@ -350,12 +350,11 @@ fn test_highlighting_empty_lines() {
 fn test_highlighting_carriage_returns() {
     let source = "a = \"a\rb\"\r\nb\r";
 
-    // FIXME(amaanq): figure why this changed w/ JS's grammar changes
     assert_eq!(
         &to_html(source, &JS_HIGHLIGHT).unwrap(),
         &[
-            "<span class=variable>a</span> <span class=operator>=</span> <span class=string>&quot;a<span class=variable>b</span>&quot;</span>\n",
-            "<span class=variable>b</span>\n",
+            "<span class=variable>a</span> <span class=operator>=</span> <span class=string>&quot;a<span class=carriage-return></span><span class=variable>b</span>&quot;</span>\n",
+            "<span class=variable>b</span><span class=carriage-return></span>\n",
         ],
     );
 }
@@ -598,7 +597,7 @@ fn test_highlighting_via_c_api() {
     let output_line_offsets =
         unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };
 
-    let mut lines = Vec::new();
+    let mut lines = Vec::with_capacity(output_line_count as usize);
     for i in 0..(output_line_count as usize) {
         let line_start = output_line_offsets[i] as usize;
         let line_end = output_line_offsets

cli/src/tests/language_test.rs

@@ -152,6 +152,7 @@ fn test_supertypes() {
                         "_literal_pattern",
                         "captured_pattern",
                         "const_block",
+                        "generic_pattern",
                         "identifier",
                         "macro_invocation",
                         "mut_pattern",

cli/src/tests/node_test.rs

@@ -6,7 +6,10 @@ use super::{
     helpers::fixtures::{fixtures_dir, get_language, get_test_language},
     Rand,
 };
-use crate::{parse::perform_edit, tests::generate_parser};
+use crate::{
+    parse::perform_edit,
+    tests::{generate_parser, helpers::fixtures::get_test_fixture_language},
+};
 
 const JSON_EXAMPLE: &str = r#"
 
@@ -308,19 +311,8 @@ fn test_parent_of_zero_width_node() {
 
 #[test]
 fn test_next_sibling_of_zero_width_node() {
-    let grammar_json = load_grammar_file(
-        &fixtures_dir()
-            .join("test_grammars")
-            .join("next_sibling_from_zwt")
-            .join("grammar.js"),
-        None,
-    )
-    .unwrap();
-
-    let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap();
-
     let mut parser = Parser::new();
-    let language = get_test_language(&parser_name, &parser_code, None);
+    let language = get_test_fixture_language("next_sibling_from_zwt");
     parser.set_language(&language).unwrap();
 
     let tree = parser.parse("abdef", None).unwrap();

cli/src/tests/parser_test.rs

@@ -6,7 +6,6 @@ use std::{
 use tree_sitter::{
     Decode, IncludedRangesError, InputEdit, LogType, ParseOptions, ParseState, Parser, Point, Range,
 };
-use tree_sitter_generate::load_grammar_file;
 use tree_sitter_proc_macro::retry;
 
 use super::helpers::{
@@ -17,7 +16,7 @@ use super::helpers::{
 use crate::{
     fuzz::edits::Edit,
     parse::perform_edit,
-    tests::{generate_parser, helpers::fixtures::fixtures_dir, invert_edit},
+    tests::{generate_parser, helpers::fixtures::get_test_fixture_language, invert_edit},
 };
 
 #[test]
@@ -482,15 +481,9 @@ fn test_parsing_empty_file_with_reused_tree() {
 
 #[test]
 fn test_parsing_after_editing_tree_that_depends_on_column_values() {
-    let dir = fixtures_dir()
-        .join("test_grammars")
-        .join("uses_current_column");
-    let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap();
-    let (grammar_name, parser_code) = generate_parser(&grammar_json).unwrap();
-
     let mut parser = Parser::new();
     parser
-        .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir)))
+        .set_language(&get_test_fixture_language("uses_current_column"))
         .unwrap();
 
     let mut code = b"
@@ -559,16 +552,9 @@ h + i
 
 #[test]
 fn test_parsing_after_editing_tree_that_depends_on_column_position() {
-    let dir = fixtures_dir()
-        .join("test_grammars")
-        .join("depends_on_column");
-
-    let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap();
-    let (grammar_name, parser_code) = generate_parser(grammar_json.as_str()).unwrap();
-
     let mut parser = Parser::new();
     parser
-        .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir)))
+        .set_language(&get_test_fixture_language("depends_on_column"))
         .unwrap();
 
     let mut code = b"\n x".to_vec();
@@ -1702,13 +1688,9 @@ if foo && bar || baz {}
 
 #[test]
 fn test_parsing_with_scanner_logging() {
-    let dir = fixtures_dir().join("test_grammars").join("external_tokens");
-    let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap();
-    let (grammar_name, parser_code) = generate_parser(&grammar_json).unwrap();
-
     let mut parser = Parser::new();
     parser
-        .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir)))
+        .set_language(&get_test_fixture_language("external_tokens"))
         .unwrap();
 
     let mut found = false;
@@ -1726,13 +1708,9 @@ fn test_parsing_with_scanner_logging() {
 
 #[test]
 fn test_parsing_get_column_at_eof() {
-    let dir = fixtures_dir().join("test_grammars").join("get_col_eof");
-    let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap();
-    let (grammar_name, parser_code) = generate_parser(&grammar_json).unwrap();
-
     let mut parser = Parser::new();
     parser
-        .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir)))
+        .set_language(&get_test_fixture_language("get_col_eof"))
         .unwrap();
 
     parser.parse("a", None).unwrap();

cli/src/tests/query_test.rs

@@ -5621,3 +5621,14 @@ const foo = [
     assert_eq!(matches.len(), 1);
     assert_eq!(matches[0].1, captures);
 }
+
+#[test]
+fn test_query_with_predicate_causing_oob_access() {
+    let language = get_language("rust");
+
+    let query = "(call_expression
+     function: (scoped_identifier
+       path: (scoped_identifier (identifier) @_regex (#any-of? @_regex \"Regex\" \"RegexBuilder\") .))
+     (#set! injection.language \"regex\"))";
+    Query::new(&language, query).unwrap();
+}

cli/src/tests/tags_test.rs

@@ -401,8 +401,11 @@ fn test_tags_via_c_api() {
 
         let syntax_types = unsafe {
             let mut len = 0;
-            let ptr =
-                c::ts_tagger_syntax_kinds_for_scope_name(tagger, c_scope_name.as_ptr(), &mut len);
+            let ptr = c::ts_tagger_syntax_kinds_for_scope_name(
+                tagger,
+                c_scope_name.as_ptr(),
+                &raw mut len,
+            );
             slice::from_raw_parts(ptr, len as usize)
                 .iter()
                 .map(|i| CStr::from_ptr(*i).to_str().unwrap())

cli/src/tests/tree_test.rs

@@ -3,7 +3,11 @@ use std::str;
 use tree_sitter::{InputEdit, Parser, Point, Range, Tree};
 
 use super::helpers::fixtures::get_language;
-use crate::{fuzz::edits::Edit, parse::perform_edit, tests::invert_edit};
+use crate::{
+    fuzz::edits::Edit,
+    parse::perform_edit,
+    tests::{helpers::fixtures::get_test_fixture_language, invert_edit},
+};
 
 #[test]
 fn test_tree_edit() {
@@ -377,6 +381,40 @@ fn test_tree_cursor() {
     assert_eq!(copy.node().kind(), "struct_item");
 }
 
+#[test]
+fn test_tree_cursor_previous_sibling_with_aliases() {
+    let mut parser = Parser::new();
+    parser
+        .set_language(&get_test_fixture_language("aliases_in_root"))
+        .unwrap();
+
+    let text = "# comment\n# \nfoo foo";
+    let tree = parser.parse(text, None).unwrap();
+    let mut cursor = tree.walk();
+    assert_eq!(cursor.node().kind(), "document");
+
+    cursor.goto_first_child();
+    assert_eq!(cursor.node().kind(), "comment");
+
+    assert!(cursor.goto_next_sibling());
+    assert_eq!(cursor.node().kind(), "comment");
+
+    assert!(cursor.goto_next_sibling());
+    assert_eq!(cursor.node().kind(), "bar");
+
+    assert!(cursor.goto_previous_sibling());
+    assert_eq!(cursor.node().kind(), "comment");
+
+    assert!(cursor.goto_previous_sibling());
+    assert_eq!(cursor.node().kind(), "comment");
+
+    assert!(cursor.goto_next_sibling());
+    assert_eq!(cursor.node().kind(), "comment");
+
+    assert!(cursor.goto_next_sibling());
+    assert_eq!(cursor.node().kind(), "bar");
+}
+
 #[test]
 fn test_tree_cursor_previous_sibling() {
     let mut parser = Parser::new();

cli/src/wasm.rs

@@ -23,10 +23,18 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)>
 pub fn get_grammar_name(language_dir: &Path) -> Result<String> {
     let src_dir = language_dir.join("src");
     let grammar_json_path = src_dir.join("grammar.json");
-    let grammar_json = fs::read_to_string(&grammar_json_path)
-        .with_context(|| format!("Failed to read grammar file {grammar_json_path:?}"))?;
-    let grammar: GrammarJSON = serde_json::from_str(&grammar_json)
-        .with_context(|| format!("Failed to parse grammar file {grammar_json_path:?}"))?;
+    let grammar_json = fs::read_to_string(&grammar_json_path).with_context(|| {
+        format!(
+            "Failed to read grammar file {}",
+            grammar_json_path.display()
+        )
+    })?;
+    let grammar: GrammarJSON = serde_json::from_str(&grammar_json).with_context(|| {
+        format!(
+            "Failed to parse grammar file {}",
+            grammar_json_path.display()
+        )
+    })?;
     Ok(grammar.name)
 }
 

docs/src/assets/schemas/config.schema.json

@@ -22,8 +22,15 @@
             "examples": [
               "Rust",
               "HTML"
-            ],
-            "$comment": "This is used in the description and the class names."
+            ]
+          },
+          "title": {
+            "type": "string",
+            "description": "The title of the language.",
+            "examples": [
+              "Rust",
+              "HTML"
+            ]
           },
           "scope": {
             "type": "string",
@@ -237,9 +244,7 @@
       "properties": {
         "c": {
           "type": "boolean",
-          "default": true,
-          "const": true,
-          "$comment": "Always generated"
+          "default": true
         },
         "go": {
           "type": "boolean",
@@ -255,9 +260,7 @@
         },
         "node": {
           "type": "boolean",
-          "default": true,
-          "const": true,
-          "$comment": "Always generated (for now)"
+          "default": true
         },
         "python": {
           "type": "boolean",
@@ -265,9 +268,7 @@
         },
         "rust": {
           "type": "boolean",
-          "default": true,
-          "const": true,
-          "$comment": "Always generated"
+          "default": true
         },
         "swift": {
           "type": "boolean",

docs/src/cli/parse.md

@@ -66,7 +66,7 @@ Suppress main output.
 
 ### `--edits <EDITS>...`
 
-Apply edits after parsing the file. Edits are in the form of `row, col delcount insert_text` where row and col are 0-indexed.
+Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, or position are 0-indexed.
 
 ### `--encoding <ENCODING>`
 

highlight/src/lib.rs

@@ -143,6 +143,8 @@ pub struct HtmlRenderer {
     pub html: Vec<u8>,
     pub line_offsets: Vec<u32>,
     carriage_return_highlight: Option<Highlight>,
+    // The offset in `self.html` of the last carriage return.
+    last_carriage_return: Option<usize>,
 }
 
 #[derive(Debug)]
@@ -1090,6 +1092,7 @@ impl HtmlRenderer {
             html: Vec::with_capacity(BUFFER_HTML_RESERVE_CAPACITY),
             line_offsets: Vec::with_capacity(BUFFER_LINES_RESERVE_CAPACITY),
             carriage_return_highlight: None,
+            last_carriage_return: None,
         };
         result.line_offsets.push(0);
         result
@@ -1131,6 +1134,9 @@ impl HtmlRenderer {
                 Err(a) => return Err(a),
             }
         }
+        if let Some(offset) = self.last_carriage_return.take() {
+            self.add_carriage_return(offset, attribute_callback);
+        }
         if self.html.last() != Some(&b'\n') {
             self.html.push(b'\n');
         }
@@ -1155,14 +1161,21 @@ impl HtmlRenderer {
             })
     }
 
-    fn add_carriage_return<F>(&mut self, attribute_callback: &F)
+    fn add_carriage_return<F>(&mut self, offset: usize, attribute_callback: &F)
     where
         F: Fn(Highlight, &mut Vec<u8>),
     {
         if let Some(highlight) = self.carriage_return_highlight {
+            // If a CR is the last character in a `HighlightEvent::Source`
+            // region, then we don't know until the next `Source` event or EOF
+            // whether it is part of CRLF or on its own. To avoid unbounded
+            // lookahead, save the offset of the CR and insert there now that we
+            // know.
+            let rest = self.html.split_off(offset);
             self.html.extend(b"<span ");
             (attribute_callback)(highlight, &mut self.html);
             self.html.extend(b"></span>");
+            self.html.extend(rest);
         }
     }
 
@@ -1194,19 +1207,17 @@ impl HtmlRenderer {
             }
         }
 
-        let mut last_char_was_cr = false;
         for c in LossyUtf8::new(src).flat_map(|p| p.bytes()) {
             // Don't render carriage return characters, but allow lone carriage returns (not
             // followed by line feeds) to be styled via the attribute callback.
             if c == b'\r' {
-                last_char_was_cr = true;
+                self.last_carriage_return = Some(self.html.len());
                 continue;
             }
-            if last_char_was_cr {
+            if let Some(offset) = self.last_carriage_return.take() {
                 if c != b'\n' {
-                    self.add_carriage_return(attribute_callback);
+                    self.add_carriage_return(offset, attribute_callback);
                 }
-                last_char_was_cr = false;
             }
 
             // At line boundaries, close and re-open all of the open tags.

lib/CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.25.1"
+        VERSION "0.25.8"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
@@ -83,13 +83,13 @@ set_target_properties(tree-sitter
 
 target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE)
 
-configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
-
 include(GNUInstallDirs)
 
+configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
+
 install(FILES include/tree_sitter/api.h
         DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter")
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc"
-        DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig")
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 install(TARGETS tree-sitter
         LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")

lib/binding_rust/build.rs

@@ -112,7 +112,10 @@ fn generate_bindings(out_dir: &std::path::Path) {
         .expect("Failed to generate bindings");
 
     let bindings_rs = out_dir.join("bindings.rs");
-    bindings
-        .write_to_file(&bindings_rs)
-        .unwrap_or_else(|_| panic!("Failed to write bindings into path: {bindings_rs:?}"));
+    bindings.write_to_file(&bindings_rs).unwrap_or_else(|_| {
+        panic!(
+            "Failed to write bindings into path: {}",
+            bindings_rs.display()
+        )
+    });
 }

lib/binding_rust/ffi.rs

@@ -16,6 +16,7 @@ extern "C" {
 }
 
 #[cfg(windows)]
+#[cfg(feature = "std")]
 extern "C" {
     pub(crate) fn _ts_dup(handle: *mut std::os::raw::c_void) -> std::os::raw::c_int;
 }

lib/binding_rust/lib.rs

@@ -1222,7 +1222,7 @@ impl Parser {
             len: u32,
             code_point: *mut i32,
         ) -> u32 {
-            let (c, len) = D::decode(std::slice::from_raw_parts(data, len as usize));
+            let (c, len) = D::decode(core::slice::from_raw_parts(data, len as usize));
             if let Some(code_point) = code_point.as_mut() {
                 *code_point = c;
             }
@@ -1422,7 +1422,7 @@ impl Parser {
         if let Some(flag) = flag {
             ffi::ts_parser_set_cancellation_flag(
                 self.0.as_ptr(),
-                std::ptr::from_ref::<AtomicUsize>(flag).cast::<usize>(),
+                core::ptr::from_ref::<AtomicUsize>(flag).cast::<usize>(),
             );
         } else {
             ffi::ts_parser_set_cancellation_flag(self.0.as_ptr(), ptr::null());
@@ -1432,7 +1432,11 @@ impl Parser {
 
 impl Drop for Parser {
     fn drop(&mut self) {
-        self.stop_printing_dot_graphs();
+        #[cfg(feature = "std")]
+        #[cfg(not(target_os = "wasi"))]
+        {
+            self.stop_printing_dot_graphs();
+        }
         self.set_logger(None);
         unsafe { ffi::ts_parser_delete(self.0.as_ptr()) }
     }
@@ -2087,7 +2091,7 @@ impl<'tree> Node<'tree> {
 
 impl PartialEq for Node<'_> {
     fn eq(&self, other: &Self) -> bool {
-        self.0.id == other.0.id
+        core::ptr::eq(self.0.id, other.0.id)
     }
 }
 

lib/binding_web/lib/tree-sitter.c

@@ -44,17 +44,22 @@ static inline void marshal_node(const void **buffer, TSNode node) {
   buffer[4] = (const void *)node.context[3];
 }
 
-static inline TSNode unmarshal_node(const TSTree *tree) {
+static inline TSNode unmarshal_node_at(const TSTree *tree, uint32_t index) {
   TSNode node;
-  node.id = TRANSFER_BUFFER[0];
-  node.context[0] = code_unit_to_byte((uint32_t)TRANSFER_BUFFER[1]);
-  node.context[1] = (uint32_t)TRANSFER_BUFFER[2];
-  node.context[2] = code_unit_to_byte((uint32_t)TRANSFER_BUFFER[3]);
-  node.context[3] = (uint32_t)TRANSFER_BUFFER[4];
+  const void **buffer = TRANSFER_BUFFER + index * SIZE_OF_NODE;
+  node.id = buffer[0];
+  node.context[0] = code_unit_to_byte((uint32_t)buffer[1]);
+  node.context[1] = (uint32_t)buffer[2];
+  node.context[2] = code_unit_to_byte((uint32_t)buffer[3]);
+  node.context[3] = (uint32_t)buffer[4];
   node.tree = tree;
   return node;
 }
 
+static inline TSNode unmarshal_node(const TSTree *tree) {
+  return unmarshal_node_at(tree, 0);
+}
+
 static inline void marshal_cursor(const TSTreeCursor *cursor) {
   TRANSFER_BUFFER[0] = cursor->id;
   TRANSFER_BUFFER[1] = (const void *)cursor->context[0];
@@ -616,7 +621,7 @@ void ts_node_parent_wasm(const TSTree *tree) {
 
 void ts_node_child_with_descendant_wasm(const TSTree *tree) {
   TSNode node = unmarshal_node(tree);
-  TSNode descendant = unmarshal_node(tree);
+  TSNode descendant = unmarshal_node_at(tree, 1);
   marshal_node(TRANSFER_BUFFER, ts_node_child_with_descendant(node, descendant));
 }
 

lib/binding_web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.25.0",
+  "version": "0.25.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "web-tree-sitter",
-      "version": "0.25.0",
+      "version": "0.25.1",
       "license": "MIT",
       "devDependencies": {
         "@eslint/js": "^9.19.0",

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.25.1",
+  "version": "0.25.8",
   "description": "Tree-sitter bindings for the web",
   "repository": "https://github.com/tree-sitter/tree-sitter",
   "homepage": "https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web",
@@ -19,12 +19,16 @@
   "exports": {
     ".": {
       "import": "./tree-sitter.js",
-      "require": "./tree-sitter.cjs"
+      "require": "./tree-sitter.cjs",
+      "types": "./web-tree-sitter.d.ts"
     },
+    "./tree-sitter.wasm": "./tree-sitter.wasm",
     "./debug": {
       "import": "./debug/tree-sitter.js",
-      "require": "./debug/tree-sitter.cjs"
-    }
+      "require": "./debug/tree-sitter.cjs",
+      "types": "./web-tree-sitter.d.ts"
+    },
+    "./debug/tree-sitter.wasm": "./debug/tree-sitter.wasm"
   },
   "types": "web-tree-sitter.d.ts",
   "keywords": [

lib/binding_web/src/language.ts

@@ -261,8 +261,7 @@ export class Language {
     } else {
       // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
       if (globalThis.process?.versions.node) {
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
-        const fs: typeof import('fs/promises') = require('fs/promises');
+        const fs: typeof import('fs/promises') = await import('fs/promises');
         bytes = fs.readFile(input);
       } else {
         bytes = fetch(input)

lib/binding_web/src/marshal.ts

@@ -34,8 +34,8 @@ export function unmarshalCaptures(
  *
  * Marshals a {@link Node} to the transfer buffer.
  */
-export function marshalNode(node: Node) {
-  let address = TRANSFER_BUFFER;
+export function marshalNode(node: Node, index = 0) {
+  let address = TRANSFER_BUFFER + index * SIZE_OF_NODE;
   C.setValue(address, node.id, 'i32');
   address += SIZE_OF_INT;
   C.setValue(address, node.startIndex, 'i32');

lib/binding_web/src/node.ts

@@ -416,6 +416,11 @@ export class Node {
     // Convert the type strings to numeric type symbols
     const symbols: number[] = [];
     const typesBySymbol = this.tree.language.types;
+    for (const node_type of types) {
+      if (node_type == "ERROR") {
+        symbols.push(65535); // Internally, ts_builtin_sym_error is -1, which is UINT_16MAX
+      }
+    }
     for (let i = 0, n = typesBySymbol.length; i < n; i++) {
       if (types.includes(typesBySymbol[i])) {
         symbols.push(i);
@@ -517,7 +522,7 @@ export class Node {
    */
   childWithDescendant(descendant: Node): Node | null {
     marshalNode(this);
-    marshalNode(descendant);
+    marshalNode(descendant, 1);
     C._ts_node_child_with_descendant_wasm(this.tree[0]);
     return unmarshalNode(this.tree);
   }

lib/binding_web/test/language.test.ts

@@ -89,6 +89,7 @@ describe('Language', () => {
               '_literal_pattern',
               'captured_pattern',
               'const_block',
+              'generic_pattern',
               'identifier',
               'macro_invocation',
               'mut_pattern',

lib/binding_web/test/node.test.ts

@@ -63,7 +63,7 @@ describe('Node', () => {
       tree = parser.parse('x10 + 1000')!;
       expect(tree.rootNode.children).toHaveLength(1);
       const sumNode = tree.rootNode.firstChild!.firstChild!;
-      expect(sumNode.children.map(child => child!.type)).toEqual(['identifier', '+', 'number' ]);
+      expect(sumNode.children.map(child => child!.type)).toEqual(['identifier', '+', 'number']);
     });
   });
 
@@ -189,6 +189,21 @@ describe('Node', () => {
     });
   });
 
+  describe('.childWithDescendant()', () => {
+    it('correctly retrieves immediate children', () => {
+      const sourceCode = 'let x = 1; console.log(x);';
+      tree = parser.parse(sourceCode)!;
+      const root = tree.rootNode
+      const child = root.children[0].children[0]
+      const a = root.childWithDescendant(child)
+      expect(a!.startIndex).toBe(0)
+      const b = a!.childWithDescendant(child)
+      expect(b).toEqual(child)
+      const c = b!.childWithDescendant(child)
+      expect(c).toBeNull()
+    });
+  });
+
   describe('.nextSibling and .previousSibling', () => {
     it('returns the node\'s next and previous sibling', () => {
       tree = parser.parse('x10 + 1000')!;
@@ -449,6 +464,24 @@ describe('Node', () => {
     });
   });
 
+  describe('.descendantsOfType("ERROR")', () => {
+    it('finds all of the descendants of an ERROR node', () => {
+      tree = parser.parse(
+        `if ({a: 'b'} {c: 'd'}) {
+          // ^ ERROR
+          x = function(a) { b; } function(c) { d; }
+        }`
+      )!;
+      const errorNode = tree.rootNode;
+      const descendants = errorNode.descendantsOfType('ERROR');
+      expect(
+        descendants.map((node) => node!.startIndex)
+      ).toEqual(
+        [4]
+      );
+    });
+  });
+
   describe('.descendantsOfType', () => {
     it('finds all descendants of a given type in the given range', () => {
       tree = parser.parse('a + 1 * b * 2 + c + 3')!;

lib/binding_web/test/parser.test.ts

@@ -256,7 +256,7 @@ describe('Parser', () => {
       expect(() => parser.parse({})).toThrow('Argument must be a string or a function');
     });
 
-    it('handles long input strings', { timeout: 5000 }, () => {
+    it('handles long input strings', { timeout: 10000 }, () => {
       const repeatCount = 10000;
       const inputString = `[${Array(repeatCount).fill('0').join(',')}]`;
 

lib/binding_web/test/query.test.ts

@@ -64,7 +64,7 @@ describe('Query', () => {
   });
 
   describe('.matches', () => {
-    it('returns all of the matches for the given query', () => {
+    it('returns all of the matches for the given query', { timeout: 10000 }, () => {
       tree = parser.parse('function one() { two(); function three() {} }')!;
       query = new Query(JavaScript, `
         (function_declaration name: (identifier) @fn-def)
@@ -462,7 +462,7 @@ describe('Query', () => {
   });
 
   describe('Set a timeout', () => {
-    it('returns less than the expected matches', () => {
+    it('returns less than the expected matches', { timeout: 10000 }, () => {
       tree = parser.parse('function foo() while (true) { } }\n'.repeat(1000))!;
       query = new Query(JavaScript, '(function_declaration name: (identifier) @function)');
       const matches = query.matches(tree.rootNode, { timeoutMicros: 1000 });
@@ -538,7 +538,7 @@ describe('Query', () => {
     });
   });
 
-  describe('Executes with a timeout', () => {
+  describe('Executes with a timeout', { timeout: 10000 }, () => {
     it('Returns less than the expected matches', () => {
       tree = parser.parse('function foo() while (true) { } }\n'.repeat(1000))!;
       query = new Query(JavaScript, '(function_declaration) @function');

lib/include/tree_sitter/api.h

@@ -42,7 +42,6 @@ typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
-typedef struct TSLanguageMetadata TSLanguageMetadata;
 typedef struct TSParser TSParser;
 typedef struct TSTree TSTree;
 typedef struct TSQuery TSQuery;

lib/language/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "tree-sitter-language"
 description = "The tree-sitter Language type, used by the library and by language implementations"
-version = "0.1.4"
+version = "0.1.5"
 authors.workspace = true
 edition.workspace = true
-rust-version.workspace = true
+rust-version = "1.76"
 readme = "README.md"
 homepage.workspace = true
 repository.workspace = true

lib/src/get_changed_ranges.c

@@ -34,7 +34,7 @@ bool ts_range_array_intersects(
   uint32_t end_byte
 ) {
   for (unsigned i = start_index; i < self->size; i++) {
-    TSRange *range = &self->contents[i];
+    TSRange *range = array_get(self, i);
     if (range->end_byte > start_byte) {
       if (range->start_byte >= end_byte) break;
       return true;
@@ -108,6 +108,7 @@ typedef struct {
   const TSLanguage *language;
   unsigned visible_depth;
   bool in_padding;
+  Subtree prev_external_token;
 } Iterator;
 
 static Iterator iterator_new(
@@ -127,6 +128,7 @@ static Iterator iterator_new(
     .language = language,
     .visible_depth = 1,
     .in_padding = false,
+    .prev_external_token = NULL_SUBTREE,
   };
 }
 
@@ -157,7 +159,7 @@ static bool iterator_tree_is_visible(const Iterator *self) {
   TreeCursorEntry entry = *array_back(&self->cursor.stack);
   if (ts_subtree_visible(*entry.subtree)) return true;
   if (self->cursor.stack.size > 1) {
-    Subtree parent = *self->cursor.stack.contents[self->cursor.stack.size - 2].subtree;
+    Subtree parent = *array_get(&self->cursor.stack, self->cursor.stack.size - 2)->subtree;
     return ts_language_alias_at(
       self->language,
       parent.ptr->production_id,
@@ -181,10 +183,10 @@ static void iterator_get_visible_state(
   }
 
   for (; i + 1 > 0; i--) {
-    TreeCursorEntry entry = self->cursor.stack.contents[i];
+    TreeCursorEntry entry = *array_get(&self->cursor.stack, i);
 
     if (i > 0) {
-      const Subtree *parent = self->cursor.stack.contents[i - 1].subtree;
+      const Subtree *parent = array_get(&self->cursor.stack, i - 1)->subtree;
       *alias_symbol = ts_language_alias_at(
         self->language,
         parent->ptr->production_id,
@@ -244,6 +246,10 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) {
 
       position = child_right;
       if (!ts_subtree_extra(*child)) structural_child_index++;
+      Subtree last_external_token = ts_subtree_last_external_token(*child);
+      if (last_external_token.ptr) {
+        self->prev_external_token = last_external_token;
+      }
     }
   } while (did_descend);
 
@@ -268,6 +274,10 @@ static void iterator_advance(Iterator *self) {
 
     const Subtree *parent = array_back(&self->cursor.stack)->subtree;
     uint32_t child_index = entry.child_index + 1;
+    Subtree last_external_token = ts_subtree_last_external_token(*entry.subtree);
+    if (last_external_token.ptr) {
+      self->prev_external_token = last_external_token;
+    }
     if (ts_subtree_child_count(*parent) > child_index) {
       Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree));
       uint32_t structural_child_index = entry.structural_child_index;
@@ -313,29 +323,41 @@ static IteratorComparison iterator_compare(
   TSSymbol new_alias_symbol = 0;
   iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
   iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);
+  TSSymbol old_symbol = ts_subtree_symbol(old_tree);
+  TSSymbol new_symbol = ts_subtree_symbol(new_tree);
 
   if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches;
   if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers;
+  if (old_alias_symbol != new_alias_symbol || old_symbol != new_symbol) return IteratorDiffers;
+
+  uint32_t old_size = ts_subtree_size(old_tree).bytes;
+  uint32_t new_size = ts_subtree_size(new_tree).bytes;
+  TSStateId old_state = ts_subtree_parse_state(old_tree);
+  TSStateId new_state = ts_subtree_parse_state(new_tree);
+  bool old_has_external_tokens = ts_subtree_has_external_tokens(old_tree);
+  bool new_has_external_tokens = ts_subtree_has_external_tokens(new_tree);
+  uint32_t old_error_cost = ts_subtree_error_cost(old_tree);
+  uint32_t new_error_cost = ts_subtree_error_cost(new_tree);
 
   if (
-    old_alias_symbol == new_alias_symbol &&
-    ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree)
+    old_start != new_start ||
+    old_symbol == ts_builtin_sym_error ||
+    old_size != new_size ||
+    old_state == TS_TREE_STATE_NONE ||
+    new_state == TS_TREE_STATE_NONE ||
+    ((old_state == ERROR_STATE) != (new_state == ERROR_STATE)) ||
+    old_error_cost != new_error_cost ||
+    old_has_external_tokens != new_has_external_tokens ||
+    ts_subtree_has_changes(old_tree) ||
+    (
+      old_has_external_tokens &&
+      !ts_subtree_external_scanner_state_eq(old_iter->prev_external_token, new_iter->prev_external_token)
+    )
   ) {
-    if (old_start == new_start &&
-        !ts_subtree_has_changes(old_tree) &&
-        ts_subtree_symbol(old_tree) != ts_builtin_sym_error &&
-        ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes &&
-        ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE &&
-        ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE &&
-        (ts_subtree_parse_state(old_tree) == ERROR_STATE) ==
-        (ts_subtree_parse_state(new_tree) == ERROR_STATE)) {
-      return IteratorMatches;
-    } else {
-      return IteratorMayDiffer;
-    }
+    return IteratorMayDiffer;
   }
 
-  return IteratorDiffers;
+  return IteratorMatches;
 }
 
 #ifdef DEBUG_GET_CHANGED_RANGES
@@ -348,8 +370,8 @@ static inline void iterator_print_state(Iterator *self) {
     "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])",
     name, self->in_padding ? "(p)" : "   ",
     self->visible_depth,
-    start.row + 1, start.column,
-    end.row + 1, end.column
+    start.row, start.column,
+    end.row, end.column
   );
 }
 #endif
@@ -380,7 +402,7 @@ unsigned ts_subtree_get_changed_ranges(
 
   do {
     #ifdef DEBUG_GET_CHANGED_RANGES
-    printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column);
+    printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column);
     iterator_print_state(&old_iter);
     printf("\tvs\t");
     iterator_print_state(&new_iter);
@@ -475,9 +497,9 @@ unsigned ts_subtree_get_changed_ranges(
     // Keep track of the current position in the included range differences
     // array in order to avoid scanning the entire array on each iteration.
     while (included_range_difference_index < included_range_differences->size) {
-      const TSRange *range = &included_range_differences->contents[
+      const TSRange *range = array_get(included_range_differences,
         included_range_difference_index
-      ];
+      );
       if (range->end_byte <= position.bytes) {
         included_range_difference_index++;
       } else {

lib/src/parser.c

@@ -193,7 +193,7 @@ static bool ts_parser__breakdown_top_of_stack(
     did_break_down = true;
     pending = false;
     for (uint32_t i = 0; i < pop.size; i++) {
-      StackSlice slice = pop.contents[i];
+      StackSlice slice = *array_get(&pop, i);
       TSStateId state = ts_stack_state(self->stack, slice.version);
       Subtree parent = *array_front(&slice.subtrees);
 
@@ -212,7 +212,7 @@ static bool ts_parser__breakdown_top_of_stack(
       }
 
       for (uint32_t j = 1; j < slice.subtrees.size; j++) {
-        Subtree tree = slice.subtrees.contents[j];
+        Subtree tree = *array_get(&slice.subtrees, j);
         ts_stack_push(self->stack, slice.version, tree, false, state);
       }
 
@@ -556,27 +556,29 @@ static Subtree ts_parser__lex(
           external_scanner_state_len
         );
 
-        // When recovering from an error, ignore any zero-length external tokens
-        // unless they have changed the external scanner's state. This helps to
-        // avoid infinite loops which could otherwise occur, because the lexer is
-        // looking for any possible token, instead of looking for the specific set of
-        // tokens that are valid in some parse state.
+        // Avoid infinite loops caused by the external scanner returning empty tokens.
+        // Empty tokens are needed in some circumstances, e.g. indent/dedent tokens
+        // in Python. Ignore the following classes of empty tokens:
         //
-        // Note that it's possible that the token end position may be *before* the
-        // original position of the lexer because of the way that tokens are positioned
-        // at included range boundaries: when a token is terminated at the start of
-        // an included range, it is marked as ending at the *end* of the preceding
-        // included range.
+        // * Tokens produced during error recovery. When recovering from an error,
+        //   all tokens are allowed, so it's easy to accidentally return unwanted
+        //   empty tokens.
+        // * Tokens that are marked as 'extra' in the grammar. These don't change
+        //   the parse state, so they would definitely cause an infinite loop.
         if (
           self->lexer.token_end_position.bytes <= current_position.bytes &&
-          (error_mode || !ts_stack_has_advanced_since_error(self->stack, version)) &&
           !external_scanner_state_changed
         ) {
-          LOG(
-            "ignore_empty_external_token symbol:%s",
-            SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol])
-          )
-          found_token = false;
+          TSSymbol symbol = self->language->external_scanner.symbol_map[self->lexer.data.result_symbol];
+          TSStateId next_parse_state = ts_language_next_state(self->language, parse_state, symbol);
+          bool token_is_extra = (next_parse_state == parse_state);
+          if (error_mode || !ts_stack_has_advanced_since_error(self->stack, version) || token_is_extra) {
+            LOG(
+              "ignore_empty_external_token symbol:%s",
+              SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol])
+            );
+            found_token = false;
+          }
         }
       }
 
@@ -947,20 +949,22 @@ static StackVersion ts_parser__reduce(
   // children.
   StackSliceArray pop = ts_stack_pop_count(self->stack, version, count);
   uint32_t removed_version_count = 0;
+  uint32_t halted_version_count = ts_stack_halted_version_count(self->stack);
   for (uint32_t i = 0; i < pop.size; i++) {
-    StackSlice slice = pop.contents[i];
+    StackSlice slice = *array_get(&pop, i);
     StackVersion slice_version = slice.version - removed_version_count;
 
     // This is where new versions are added to the parse stack. The versions
     // will all be sorted and truncated at the end of the outer parsing loop.
     // Allow the maximum version count to be temporarily exceeded, but only
     // by a limited threshold.
-    if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW) {
+    if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW + halted_version_count) {
       ts_stack_remove_version(self->stack, slice_version);
       ts_subtree_array_delete(&self->tree_pool, &slice.subtrees);
       removed_version_count++;
       while (i + 1 < pop.size) {
-        StackSlice next_slice = pop.contents[i + 1];
+        LOG("aborting reduce with too many versions")
+        StackSlice next_slice = *array_get(&pop, i + 1);
         if (next_slice.version != slice.version) break;
         ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees);
         i++;
@@ -983,7 +987,7 @@ static StackVersion ts_parser__reduce(
     // choose one of the arrays of trees to be the parent node's children, and
     // delete the rest of the tree arrays.
     while (i + 1 < pop.size) {
-      StackSlice next_slice = pop.contents[i + 1];
+      StackSlice next_slice = *array_get(&pop, i + 1);
       if (next_slice.version != slice.version) break;
       i++;
 
@@ -1025,7 +1029,7 @@ static StackVersion ts_parser__reduce(
     // were previously on top of the stack.
     ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state);
     for (uint32_t j = 0; j < self->trailing_extras.size; j++) {
-      ts_stack_push(self->stack, slice_version, self->trailing_extras.contents[j], false, next_state);
+      ts_stack_push(self->stack, slice_version, *array_get(&self->trailing_extras, j), false, next_state);
     }
 
     for (StackVersion j = 0; j < slice_version; j++) {
@@ -1053,11 +1057,11 @@ static void ts_parser__accept(
 
   StackSliceArray pop = ts_stack_pop_all(self->stack, version);
   for (uint32_t i = 0; i < pop.size; i++) {
-    SubtreeArray trees = pop.contents[i].subtrees;
+    SubtreeArray trees = array_get(&pop, i)->subtrees;
 
     Subtree root = NULL_SUBTREE;
     for (uint32_t j = trees.size - 1; j + 1 > 0; j--) {
-      Subtree tree = trees.contents[j];
+      Subtree tree = *array_get(&trees, j);
       if (!ts_subtree_extra(tree)) {
         ts_assert(!tree.data.is_inline);
         uint32_t child_count = ts_subtree_child_count(tree);
@@ -1092,7 +1096,7 @@ static void ts_parser__accept(
     }
   }
 
-  ts_stack_remove_version(self->stack, pop.contents[0].version);
+  ts_stack_remove_version(self->stack, array_get(&pop, 0)->version);
   ts_stack_halt(self->stack, version);
 }
 
@@ -1158,7 +1162,7 @@ static bool ts_parser__do_all_potential_reductions(
 
     StackVersion reduction_version = STACK_VERSION_NONE;
     for (uint32_t j = 0; j < self->reduce_actions.size; j++) {
-      ReduceAction action = self->reduce_actions.contents[j];
+      ReduceAction action = *array_get(&self->reduce_actions, j);
 
       reduction_version = ts_parser__reduce(
         self, version, action.symbol, action.count,
@@ -1196,7 +1200,7 @@ static bool ts_parser__recover_to_state(
   StackVersion previous_version = STACK_VERSION_NONE;
 
   for (unsigned i = 0; i < pop.size; i++) {
-    StackSlice slice = pop.contents[i];
+    StackSlice slice = *array_get(&pop, i);
 
     if (slice.version == previous_version) {
       ts_subtree_array_delete(&self->tree_pool, &slice.subtrees);
@@ -1214,12 +1218,12 @@ static bool ts_parser__recover_to_state(
     SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version);
     if (error_trees.size > 0) {
       ts_assert(error_trees.size == 1);
-      Subtree error_tree = error_trees.contents[0];
+      Subtree error_tree = *array_get(&error_trees, 0);
       uint32_t error_child_count = ts_subtree_child_count(error_tree);
       if (error_child_count > 0) {
         array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree));
         for (unsigned j = 0; j < error_child_count; j++) {
-          ts_subtree_retain(slice.subtrees.contents[j]);
+          ts_subtree_retain(*array_get(&slice.subtrees, j));
         }
       }
       ts_subtree_array_delete(&self->tree_pool, &error_trees);
@@ -1235,7 +1239,7 @@ static bool ts_parser__recover_to_state(
     }
 
     for (unsigned j = 0; j < self->trailing_extras.size; j++) {
-      Subtree tree = self->trailing_extras.contents[j];
+      Subtree tree = *array_get(&self->trailing_extras, j);
       ts_stack_push(self->stack, slice.version, tree, false, goal_state);
     }
 
@@ -1271,7 +1275,7 @@ static void ts_parser__recover(
   // if the current lookahead token would be valid in that state.
   if (summary && !ts_subtree_is_error(lookahead)) {
     for (unsigned i = 0; i < summary->size; i++) {
-      StackSummaryEntry entry = summary->contents[i];
+      StackSummaryEntry entry = *array_get(summary, i);
 
       if (entry.state == ERROR_STATE) continue;
       if (entry.position.bytes == position.bytes) continue;
@@ -1316,10 +1320,23 @@ static void ts_parser__recover(
   // and subsequently halted. Remove those versions.
   for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) {
     if (!ts_stack_is_active(self->stack, i)) {
+      LOG("removed paused version:%u", i);
       ts_stack_remove_version(self->stack, i--);
+      LOG_STACK();
     }
   }
 
+  // If the parser is still in the error state at the end of the file, just wrap everything
+  // in an ERROR node and terminate.
+  if (ts_subtree_is_eof(lookahead)) {
+    LOG("recover_eof");
+    SubtreeArray children = array_new();
+    Subtree parent = ts_subtree_new_error_node(&children, false, self->language);
+    ts_stack_push(self->stack, version, parent, false, 1);
+    ts_parser__accept(self, version, lookahead);
+    return;
+  }
+
   // If strategy 1 succeeded, a new stack version will have been created which is able to handle
   // the current lookahead token. Now, in addition, try strategy 2 described above: skip the
   // current lookahead token by wrapping it in an ERROR node.
@@ -1340,17 +1357,6 @@ static void ts_parser__recover(
     return;
   }
 
-  // If the parser is still in the error state at the end of the file, just wrap everything
-  // in an ERROR node and terminate.
-  if (ts_subtree_is_eof(lookahead)) {
-    LOG("recover_eof");
-    SubtreeArray children = array_new();
-    Subtree parent = ts_subtree_new_error_node(&children, false, self->language);
-    ts_stack_push(self->stack, version, parent, false, 1);
-    ts_parser__accept(self, version, lookahead);
-    return;
-  }
-
   // Do not recover if the result would clearly be worse than some existing stack version.
   unsigned new_cost =
     current_error_cost + ERROR_COST_PER_SKIPPED_TREE +
@@ -1396,18 +1402,18 @@ static void ts_parser__recover(
     // arbitrarily and discard the rest.
     if (pop.size > 1) {
       for (unsigned i = 1; i < pop.size; i++) {
-        ts_subtree_array_delete(&self->tree_pool, &pop.contents[i].subtrees);
+        ts_subtree_array_delete(&self->tree_pool, &array_get(&pop, i)->subtrees);
       }
-      while (ts_stack_version_count(self->stack) > pop.contents[0].version + 1) {
-        ts_stack_remove_version(self->stack, pop.contents[0].version + 1);
+      while (ts_stack_version_count(self->stack) > array_get(&pop, 0)->version + 1) {
+        ts_stack_remove_version(self->stack, array_get(&pop, 0)->version + 1);
       }
     }
 
-    ts_stack_renumber_version(self->stack, pop.contents[0].version, version);
-    array_push(&pop.contents[0].subtrees, ts_subtree_from_mut(error_repeat));
+    ts_stack_renumber_version(self->stack, array_get(&pop, 0)->version, version);
+    array_push(&array_get(&pop, 0)->subtrees, ts_subtree_from_mut(error_repeat));
     error_repeat = ts_subtree_new_node(
       ts_builtin_sym_error_repeat,
-      &pop.contents[0].subtrees,
+      &array_get(&pop, 0)->subtrees,
       0,
       self->language
     );
@@ -1534,7 +1540,7 @@ static bool ts_parser__check_progress(TSParser *self, Subtree *lookahead, const
   if (self->operation_count >= OP_COUNT_PER_PARSER_TIMEOUT_CHECK) {
     self->operation_count = 0;
   }
-  if (self->parse_options.progress_callback && position != NULL) {
+  if (position != NULL) {
     self->parse_state.current_byte_offset = *position;
     self->parse_state.has_error = self->has_error;
   }
@@ -1616,6 +1622,7 @@ static bool ts_parser__advance(
     // an ambiguous state. REDUCE actions always create a new stack
     // version, whereas SHIFT actions update the existing stack version
     // and terminate this loop.
+    bool did_reduce = false;
     StackVersion last_reduction_version = STACK_VERSION_NONE;
     for (uint32_t i = 0; i < table_entry.action_count; i++) {
       TSParseAction action = table_entry.actions[i];
@@ -1651,6 +1658,7 @@ static bool ts_parser__advance(
             action.reduce.dynamic_precedence, action.reduce.production_id,
             is_fragile, end_of_non_terminal_extra
           );
+          did_reduce = true;
           if (reduction_version != STACK_VERSION_NONE) {
             last_reduction_version = reduction_version;
           }
@@ -1702,9 +1710,12 @@ static bool ts_parser__advance(
       continue;
     }
 
-    // A non-terminal extra rule was reduced and merged into an existing
-    // stack version. This version can be discarded.
-    if (!lookahead.ptr) {
+    // A reduction was performed, but was merged into an existing stack version.
+    // This version can be discarded.
+    if (did_reduce) {
+      if (lookahead.ptr) {
+        ts_subtree_release(&self->tree_pool, lookahead);
+      }
       ts_stack_halt(self->stack, version);
       return true;
     }
@@ -1753,7 +1764,7 @@ static bool ts_parser__advance(
     // versions that exist. If some other version advances successfully, then
     // this version can simply be removed. But if all versions end up paused,
     // then error recovery is needed.
-    LOG("detect_error");
+    LOG("detect_error lookahead:%s", TREE_NAME(lookahead));
     ts_stack_pause(self->stack, version, lookahead);
     return true;
   }
@@ -1842,6 +1853,7 @@ static unsigned ts_parser__condense_stack(TSParser *self) {
           has_unpaused_version = true;
         } else {
           ts_stack_remove_version(self->stack, i);
+          made_changes = true;
           i--;
           n--;
         }
@@ -1877,9 +1889,9 @@ static bool ts_parser__balance_subtree(TSParser *self) {
       return false;
     }
 
-    MutableSubtree tree = self->tree_pool.tree_stack.contents[
+    MutableSubtree tree = *array_get(&self->tree_pool.tree_stack, 
       self->tree_pool.tree_stack.size - 1
-    ];
+    );
 
     if (tree.ptr->repeat_depth > 0) {
       Subtree child1 = ts_subtree_children(tree)[0];
@@ -2128,7 +2140,7 @@ TSTree *ts_parser_parse(
       LOG("parse_after_edit");
       LOG_TREE(self->old_tree);
       for (unsigned i = 0; i < self->included_range_differences.size; i++) {
-        TSRange *range = &self->included_range_differences.contents[i];
+        TSRange *range = array_get(&self->included_range_differences, i);
         LOG("different_included_range %u - %u", range->start_byte, range->end_byte);
       }
     } else {
@@ -2185,7 +2197,7 @@ TSTree *ts_parser_parse(
     }
 
     while (self->included_range_difference_index < self->included_range_differences.size) {
-      TSRange *range = &self->included_range_differences.contents[self->included_range_difference_index];
+      TSRange *range = array_get(&self->included_range_differences, self->included_range_difference_index);
       if (range->end_byte <= position) {
         self->included_range_difference_index++;
       } else {
@@ -2226,6 +2238,8 @@ TSTree *ts_parser_parse_with_options(
   self->parse_options = parse_options;
   self->parse_state.payload = parse_options.payload;
   TSTree *result = ts_parser_parse(self, old_tree, input);
+  // Reset parser options before further parse calls.
+  self->parse_options = (TSParseOptions) {0};
   return result;
 }
 

lib/src/parser.h

@@ -18,7 +18,6 @@ typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
-typedef struct TSLanguageMetadata TSLanguageMetadata;
 typedef struct TSLanguageMetadata {
   uint8_t major_version;
   uint8_t minor_version;

lib/src/portable/endian.h

@@ -18,16 +18,22 @@
 #if defined(HAVE_ENDIAN_H) || \
     defined(__linux__) || \
     defined(__GNU__) || \
+    defined(__illumos__) || \
+    defined(__NetBSD__) || \
     defined(__OpenBSD__) || \
     defined(__CYGWIN__) || \
     defined(__MSYS__) || \
-    defined(__EMSCRIPTEN__)
+    defined(__EMSCRIPTEN__) || \
+    defined(__wasi__)
+
+#if defined(__NetBSD__)
+#define _NETBSD_SOURCE 1
+#endif
 
 # include <endian.h>
 
 #elif defined(HAVE_SYS_ENDIAN_H) || \
     defined(__FreeBSD__) || \
-    defined(__NetBSD__) || \
     defined(__DragonFly__)
 
 # include <sys/endian.h>

lib/src/query.c

@@ -1,3 +1,12 @@
+/*
+ * On NetBSD, defining standard requirements like this removes symbols
+ * from the namespace; however, we need non-standard symbols for
+ * endian.h.
+ */
+#if defined(__NetBSD__) && defined(_POSIX_C_SOURCE)
+#undef _POSIX_C_SOURCE
+#endif
+
 #include "tree_sitter/api.h"
 #include "./alloc.h"
 #include "./array.h"
@@ -428,26 +437,26 @@ static CaptureListPool capture_list_pool_new(void) {
 static void capture_list_pool_reset(CaptureListPool *self) {
   for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
     // This invalid size means that the list is not in use.
-    self->list.contents[i].size = UINT32_MAX;
+    array_get(&self->list, i)->size = UINT32_MAX;
   }
   self->free_capture_list_count = self->list.size;
 }
 
 static void capture_list_pool_delete(CaptureListPool *self) {
   for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
-    array_delete(&self->list.contents[i]);
+    array_delete(array_get(&self->list, i));
   }
   array_delete(&self->list);
 }
 
 static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) {
   if (id >= self->list.size) return &self->empty_list;
-  return &self->list.contents[id];
+  return array_get(&self->list, id);
 }
 
 static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) {
   ts_assert(id < self->list.size);
-  return &self->list.contents[id];
+  return array_get(&self->list, id);
 }
 
 static bool capture_list_pool_is_empty(const CaptureListPool *self) {
@@ -460,8 +469,8 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
   // First see if any already allocated capture list is currently unused.
   if (self->free_capture_list_count > 0) {
     for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
-      if (self->list.contents[i].size == UINT32_MAX) {
-        array_clear(&self->list.contents[i]);
+      if (array_get(&self->list, i)->size == UINT32_MAX) {
+        array_clear(array_get(&self->list, i));
         self->free_capture_list_count--;
         return i;
       }
@@ -482,7 +491,7 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
 
 static void capture_list_pool_release(CaptureListPool *self, uint16_t id) {
   if (id >= self->list.size) return;
-  self->list.contents[id].size = UINT32_MAX;
+  array_get(&self->list, id)->size = UINT32_MAX;
   self->free_capture_list_count++;
 }
 
@@ -765,10 +774,10 @@ static int symbol_table_id_for_name(
   uint32_t length
 ) {
   for (unsigned i = 0; i < self->slices.size; i++) {
-    Slice slice = self->slices.contents[i];
+    Slice slice = *array_get(&self->slices, i);
     if (
       slice.length == length &&
-      !strncmp(&self->characters.contents[slice.offset], name, length)
+      !strncmp(array_get(&self->characters, slice.offset), name, length)
     ) return i;
   }
   return -1;
@@ -779,9 +788,9 @@ static const char *symbol_table_name_for_id(
   uint16_t id,
   uint32_t *length
 ) {
-  Slice slice = self->slices.contents[id];
+  Slice slice = *(array_get(&self->slices,id));
   *length = slice.length;
-  return &self->characters.contents[slice.offset];
+  return array_get(&self->characters, slice.offset);
 }
 
 static uint16_t symbol_table_insert_name(
@@ -796,8 +805,8 @@ static uint16_t symbol_table_insert_name(
     .length = length,
   };
   array_grow_by(&self->characters, length + 1);
-  memcpy(&self->characters.contents[slice.offset], name, length);
-  self->characters.contents[self->characters.size - 1] = 0;
+  memcpy(array_get(&self->characters, slice.offset), name, length);
+  *array_get(&self->characters, self->characters.size - 1) = 0;
   array_push(&self->slices, slice);
   return self->slices.size - 1;
 }
@@ -1109,23 +1118,23 @@ static inline bool ts_query__pattern_map_search(
   while (size > 1) {
     uint32_t half_size = size / 2;
     uint32_t mid_index = base_index + half_size;
-    TSSymbol mid_symbol = self->steps.contents[
-      self->pattern_map.contents[mid_index].step_index
-    ].symbol;
+    TSSymbol mid_symbol = array_get(&self->steps,
+      array_get(&self->pattern_map, mid_index)->step_index
+    )->symbol;
     if (needle > mid_symbol) base_index = mid_index;
     size -= half_size;
   }
 
-  TSSymbol symbol = self->steps.contents[
-    self->pattern_map.contents[base_index].step_index
-  ].symbol;
+  TSSymbol symbol = array_get(&self->steps,
+    array_get(&self->pattern_map, base_index)->step_index
+  )->symbol;
 
   if (needle > symbol) {
     base_index++;
     if (base_index < self->pattern_map.size) {
-      symbol = self->steps.contents[
-        self->pattern_map.contents[base_index].step_index
-      ].symbol;
+      symbol = array_get(&self->steps,
+        array_get(&self->pattern_map, base_index)->step_index
+      )->symbol;
     }
   }
 
@@ -1148,9 +1157,9 @@ static inline void ts_query__pattern_map_insert(
   // initiated first, which allows the ordering of the states array
   // to be maintained more efficiently.
   while (index < self->pattern_map.size) {
-    PatternEntry *entry = &self->pattern_map.contents[index];
+    PatternEntry *entry = array_get(&self->pattern_map, index);
     if (
-      self->steps.contents[entry->step_index].symbol == symbol &&
+      array_get(&self->steps, entry->step_index)->symbol == symbol &&
       entry->pattern_index < new_entry.pattern_index
     ) {
       index++;
@@ -1183,11 +1192,11 @@ static void ts_query__perform_analysis(
     #ifdef DEBUG_ANALYZE_QUERY
       printf("Iteration: %u. Final step indices:", iteration);
       for (unsigned j = 0; j < analysis->final_step_indices.size; j++) {
-        printf(" %4u", analysis->final_step_indices.contents[j]);
+        printf(" %4u", *array_get(&analysis->final_step_indices, j));
       }
       printf("\n");
       for (unsigned j = 0; j < analysis->states.size; j++) {
-        AnalysisState *state = analysis->states.contents[j];
+        AnalysisState *state = *array_get(&analysis->states, j);
         printf("  %3u: step: %u, stack: [", j, state->step_index);
         for (unsigned k = 0; k < state->depth; k++) {
           printf(
@@ -1230,7 +1239,7 @@ static void ts_query__perform_analysis(
 
     analysis_state_set__clear(&analysis->next_states, &analysis->state_pool);
     for (unsigned j = 0; j < analysis->states.size; j++) {
-      AnalysisState * const state = analysis->states.contents[j];
+      AnalysisState * const state = *array_get(&analysis->states, j);
 
       // For efficiency, it's important to avoid processing the same analysis state more
       // than once. To achieve this, keep the states in order of ascending position within
@@ -1253,7 +1262,7 @@ static void ts_query__perform_analysis(
             analysis_state_set__push(
               &analysis->next_states,
               &analysis->state_pool,
-              analysis->states.contents[j]
+              *array_get(&analysis->states, j)
             );
             j++;
           }
@@ -1265,12 +1274,12 @@ static void ts_query__perform_analysis(
       const TSSymbol parent_symbol = analysis_state__top(state)->parent_symbol;
       const TSFieldId parent_field_id = analysis_state__top(state)->field_id;
       const unsigned child_index = analysis_state__top(state)->child_index;
-      const QueryStep * const step = &self->steps.contents[state->step_index];
+      const QueryStep * const step = array_get(&self->steps, state->step_index);
 
       unsigned subgraph_index, exists;
       array_search_sorted_by(subgraphs, .symbol, parent_symbol, &subgraph_index, &exists);
       if (!exists) continue;
-      const AnalysisSubgraph *subgraph = &subgraphs->contents[subgraph_index];
+      const AnalysisSubgraph *subgraph = array_get(subgraphs, subgraph_index);
 
       // Follow every possible path in the parse table, but only visit states that
       // are part of the subgraph for the current symbol.
@@ -1306,7 +1315,8 @@ static void ts_query__perform_analysis(
           &node_index, &exists
         );
         while (node_index < subgraph->nodes.size) {
-          AnalysisSubgraphNode *node = &subgraph->nodes.contents[node_index++];
+          AnalysisSubgraphNode *node = array_get(&subgraph->nodes, node_index);
+          node_index++;
           if (node->state != successor.state || node->child_index != successor.child_index) break;
 
           // Use the subgraph to determine what alias and field will eventually be applied
@@ -1407,7 +1417,7 @@ static void ts_query__perform_analysis(
           if (does_match) {
             for (;;) {
               next_state.step_index++;
-              next_step = &self->steps.contents[next_state.step_index];
+              next_step = array_get(&self->steps, next_state.step_index);
               if (
                 next_step->depth == PATTERN_DONE_MARKER ||
                 next_step->depth <= step->depth
@@ -1431,7 +1441,7 @@ static void ts_query__perform_analysis(
             // record that matching can terminate at this step of the pattern. Otherwise,
             // add this state to the list of states to process on the next iteration.
             if (!next_step->is_dead_end) {
-              bool did_finish_pattern = self->steps.contents[next_state.step_index].depth != step->depth;
+              bool did_finish_pattern = array_get(&self->steps, next_state.step_index)->depth != step->depth;
               if (did_finish_pattern) {
                 array_insert_sorted_by(&analysis->finished_parent_symbols, , state->root_symbol);
               } else if (next_state.depth == 0) {
@@ -1451,7 +1461,7 @@ static void ts_query__perform_analysis(
               next_step->alternative_index > next_state.step_index
             ) {
               next_state.step_index = next_step->alternative_index;
-              next_step = &self->steps.contents[next_state.step_index];
+              next_step = array_get(&self->steps, next_state.step_index);
             } else {
               break;
             }
@@ -1469,9 +1479,9 @@ static void ts_query__perform_analysis(
 static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   Array(uint16_t) non_rooted_pattern_start_steps = array_new();
   for (unsigned i = 0; i < self->pattern_map.size; i++) {
-    PatternEntry *pattern = &self->pattern_map.contents[i];
+    PatternEntry *pattern = array_get(&self->pattern_map, i);
     if (!pattern->is_rooted) {
-      QueryStep *step = &self->steps.contents[pattern->step_index];
+      QueryStep *step = array_get(&self->steps, pattern->step_index);
       if (step->symbol != WILDCARD_SYMBOL) {
         array_push(&non_rooted_pattern_start_steps, i);
       }
@@ -1483,7 +1493,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   // captures, and record the indices of all of the steps that have child steps.
   Array(uint32_t) parent_step_indices = array_new();
   for (unsigned i = 0; i < self->steps.size; i++) {
-    QueryStep *step = &self->steps.contents[i];
+    QueryStep *step = array_get(&self->steps, i);
     if (step->depth == PATTERN_DONE_MARKER) {
       step->parent_pattern_guaranteed = true;
       step->root_pattern_guaranteed = true;
@@ -1494,7 +1504,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
     bool is_wildcard = step->symbol == WILDCARD_SYMBOL;
     step->contains_captures = step->capture_ids[0] != NONE;
     for (unsigned j = i + 1; j < self->steps.size; j++) {
-      QueryStep *next_step = &self->steps.contents[j];
+      QueryStep *next_step = array_get(&self->steps, j);
       if (
         next_step->depth == PATTERN_DONE_MARKER ||
         next_step->depth <= step->depth
@@ -1524,8 +1534,8 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   // parent.
   AnalysisSubgraphArray subgraphs = array_new();
   for (unsigned i = 0; i < parent_step_indices.size; i++) {
-    uint32_t parent_step_index = parent_step_indices.contents[i];
-    TSSymbol parent_symbol = self->steps.contents[parent_step_index].symbol;
+    uint32_t parent_step_index = *array_get(&parent_step_indices, i);
+    TSSymbol parent_symbol = array_get(&self->steps, parent_step_index)->symbol;
     AnalysisSubgraph subgraph = { .symbol = parent_symbol };
     array_insert_sorted_by(&subgraphs, .symbol, subgraph);
   }
@@ -1567,7 +1577,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
                 &exists
               );
               if (exists) {
-                AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index];
+                AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index);
                 if (subgraph->nodes.size == 0 || array_back(&subgraph->nodes)->state != state) {
                   array_push(&subgraph->nodes, ((AnalysisSubgraphNode) {
                     .state = state,
@@ -1604,7 +1614,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
               &exists
             );
             if (exists) {
-              AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index];
+              AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index);
               if (
                 subgraph->start_states.size == 0 ||
                 *array_back(&subgraph->start_states) != state
@@ -1621,7 +1631,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   // from the end states using the predecessor map.
   Array(AnalysisSubgraphNode) next_nodes = array_new();
   for (unsigned i = 0; i < subgraphs.size; i++) {
-    AnalysisSubgraph *subgraph = &subgraphs.contents[i];
+    AnalysisSubgraph *subgraph = array_get(&subgraphs, i);
     if (subgraph->nodes.size == 0) {
       array_delete(&subgraph->start_states);
       array_erase(&subgraphs, i);
@@ -1662,16 +1672,16 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   #ifdef DEBUG_ANALYZE_QUERY
     printf("\nSubgraphs:\n");
     for (unsigned i = 0; i < subgraphs.size; i++) {
-      AnalysisSubgraph *subgraph = &subgraphs.contents[i];
+      AnalysisSubgraph *subgraph = array_get(&subgraphs, i);
       printf("  %u, %s:\n", subgraph->symbol, ts_language_symbol_name(self->language, subgraph->symbol));
       for (unsigned j = 0; j < subgraph->start_states.size; j++) {
         printf(
           "    {state: %u}\n",
-          subgraph->start_states.contents[j]
+          *array_get(&subgraph->start_states, j)
         );
       }
       for (unsigned j = 0; j < subgraph->nodes.size; j++) {
-        AnalysisSubgraphNode *node = &subgraph->nodes.contents[j];
+        AnalysisSubgraphNode *node = array_get(&subgraph->nodes, j);
         printf(
           "    {state: %u, child_index: %u, production_id: %u, done: %d}\n",
           node->state, node->child_index, node->production_id, node->done
@@ -1686,9 +1696,9 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   bool all_patterns_are_valid = true;
   QueryAnalysis analysis = query_analysis__new();
   for (unsigned i = 0; i < parent_step_indices.size; i++) {
-    uint16_t parent_step_index = parent_step_indices.contents[i];
-    uint16_t parent_depth = self->steps.contents[parent_step_index].depth;
-    TSSymbol parent_symbol = self->steps.contents[parent_step_index].symbol;
+    uint16_t parent_step_index = *array_get(&parent_step_indices, i);
+    uint16_t parent_depth = array_get(&self->steps, parent_step_index)->depth;
+    TSSymbol parent_symbol = array_get(&self->steps, parent_step_index)->symbol;
     if (parent_symbol == ts_builtin_sym_error) continue;
 
     // Find the subgraph that corresponds to this pattern's root symbol. If the pattern's
@@ -1700,18 +1710,18 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
       uint32_t j, child_exists;
       array_search_sorted_by(&self->step_offsets, .step_index, first_child_step_index, &j, &child_exists);
       ts_assert(child_exists);
-      *error_offset = self->step_offsets.contents[j].byte_offset;
+      *error_offset = array_get(&self->step_offsets, j)->byte_offset;
       all_patterns_are_valid = false;
       break;
     }
 
     // Initialize an analysis state at every parse state in the table where
     // this parent symbol can occur.
-    AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index];
+    AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index);
     analysis_state_set__clear(&analysis.states, &analysis.state_pool);
     analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool);
     for (unsigned j = 0; j < subgraph->start_states.size; j++) {
-      TSStateId parse_state = subgraph->start_states.contents[j];
+      TSStateId parse_state = *array_get(&subgraph->start_states, j);
       analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) {
         .step_index = parent_step_index + 1,
         .stack = {
@@ -1731,7 +1741,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
     #ifdef DEBUG_ANALYZE_QUERY
       printf(
         "\nWalk states for %s:\n",
-        ts_language_symbol_name(self->language, analysis.states.contents[0]->stack[0].parent_symbol)
+        ts_language_symbol_name(self->language, (*array_get(&analysis.states, 0))->stack[0].parent_symbol)
       );
     #endif
 
@@ -1742,7 +1752,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
     // be considered fallible.
     if (analysis.did_abort) {
       for (unsigned j = parent_step_index + 1; j < self->steps.size; j++) {
-        QueryStep *step = &self->steps.contents[j];
+        QueryStep *step = array_get(&self->steps, j);
         if (
           step->depth <= parent_depth ||
           step->depth == PATTERN_DONE_MARKER
@@ -1763,7 +1773,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
       uint32_t j, impossible_exists;
       array_search_sorted_by(&self->step_offsets, .step_index, impossible_step_index, &j, &impossible_exists);
       if (j >= self->step_offsets.size) j = self->step_offsets.size - 1;
-      *error_offset = self->step_offsets.contents[j].byte_offset;
+      *error_offset = array_get(&self->step_offsets, j)->byte_offset;
       all_patterns_are_valid = false;
       break;
     }
@@ -1771,8 +1781,8 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
     // Mark as fallible any step where a match terminated.
     // Later, this property will be propagated to all of the step's predecessors.
     for (unsigned j = 0; j < analysis.final_step_indices.size; j++) {
-      uint32_t final_step_index = analysis.final_step_indices.contents[j];
-      QueryStep *step = &self->steps.contents[final_step_index];
+      uint32_t final_step_index = *array_get(&analysis.final_step_indices, j);
+      QueryStep *step = array_get(&self->steps, final_step_index);
       if (
         step->depth != PATTERN_DONE_MARKER &&
         step->depth > parent_depth &&
@@ -1787,7 +1797,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   // Mark as indefinite any step with captures that are used in predicates.
   Array(uint16_t) predicate_capture_ids = array_new();
   for (unsigned i = 0; i < self->patterns.size; i++) {
-    QueryPattern *pattern = &self->patterns.contents[i];
+    QueryPattern *pattern = array_get(&self->patterns, i);
 
     // Gather all of the captures that are used in predicates for this pattern.
     array_clear(&predicate_capture_ids);
@@ -1796,7 +1806,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
       end = start + pattern->predicate_steps.length,
       j = start; j < end; j++
     ) {
-      TSQueryPredicateStep *step = &self->predicate_steps.contents[j];
+      TSQueryPredicateStep *step = array_get(&self->predicate_steps, j);
       if (step->type == TSQueryPredicateStepTypeCapture) {
         uint16_t value_id = step->value_id;
         array_insert_sorted_by(&predicate_capture_ids, , value_id);
@@ -1809,7 +1819,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
       end = start + pattern->steps.length,
       j = start; j < end; j++
     ) {
-      QueryStep *step = &self->steps.contents[j];
+      QueryStep *step = array_get(&self->steps, j);
       for (unsigned k = 0; k < MAX_STEP_CAPTURE_COUNT; k++) {
         uint16_t capture_id = step->capture_ids[k];
         if (capture_id == NONE) break;
@@ -1829,7 +1839,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   while (!done) {
     done = true;
     for (unsigned i = self->steps.size - 1; i > 0; i--) {
-      QueryStep *step = &self->steps.contents[i];
+      QueryStep *step = array_get(&self->steps, i);
       if (step->depth == PATTERN_DONE_MARKER) continue;
 
       // Determine if this step is definite or has definite alternatives.
@@ -1842,12 +1852,12 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
         if (step->alternative_index == NONE || step->alternative_index < i) {
           break;
         }
-        step = &self->steps.contents[step->alternative_index];
+        step = array_get(&self->steps, step->alternative_index);
       }
 
       // If not, mark its predecessor as indefinite.
       if (!parent_pattern_guaranteed) {
-        QueryStep *prev_step = &self->steps.contents[i - 1];
+        QueryStep *prev_step = array_get(&self->steps, i - 1);
         if (
           !prev_step->is_dead_end &&
           prev_step->depth != PATTERN_DONE_MARKER &&
@@ -1863,7 +1873,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   #ifdef DEBUG_ANALYZE_QUERY
     printf("Steps:\n");
     for (unsigned i = 0; i < self->steps.size; i++) {
-      QueryStep *step = &self->steps.contents[i];
+      QueryStep *step = array_get(&self->steps, i);
       if (step->depth == PATTERN_DONE_MARKER) {
         printf("  %u: DONE\n", i);
       } else {
@@ -1887,18 +1897,18 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   // prevent certain optimizations with range restrictions.
   analysis.did_abort = false;
   for (uint32_t i = 0; i < non_rooted_pattern_start_steps.size; i++) {
-    uint16_t pattern_entry_index = non_rooted_pattern_start_steps.contents[i];
-    PatternEntry *pattern_entry = &self->pattern_map.contents[pattern_entry_index];
+    uint16_t pattern_entry_index = *array_get(&non_rooted_pattern_start_steps, i);
+    PatternEntry *pattern_entry = array_get(&self->pattern_map, pattern_entry_index);
 
     analysis_state_set__clear(&analysis.states, &analysis.state_pool);
     analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool);
     for (unsigned j = 0; j < subgraphs.size; j++) {
-      AnalysisSubgraph *subgraph = &subgraphs.contents[j];
+      AnalysisSubgraph *subgraph = array_get(&subgraphs, j);
       TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, subgraph->symbol);
       if (metadata.visible || metadata.named) continue;
 
       for (uint32_t k = 0; k < subgraph->start_states.size; k++) {
-        TSStateId parse_state = subgraph->start_states.contents[k];
+        TSStateId parse_state = *array_get(&subgraph->start_states, k);
         analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) {
           .step_index = pattern_entry->step_index,
           .stack = {
@@ -1927,11 +1937,11 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
     );
 
     if (analysis.finished_parent_symbols.size > 0) {
-      self->patterns.contents[pattern_entry->pattern_index].is_non_local = true;
+      array_get(&self->patterns, pattern_entry->pattern_index)->is_non_local = true;
     }
 
     for (unsigned k = 0; k < analysis.finished_parent_symbols.size; k++) {
-      TSSymbol symbol = analysis.finished_parent_symbols.contents[k];
+      TSSymbol symbol = *array_get(&analysis.finished_parent_symbols, k);
       array_insert_sorted_by(&self->repeat_symbols_with_rootless_patterns, , symbol);
     }
   }
@@ -1941,7 +1951,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
       printf("\nRepetition symbols with rootless patterns:\n");
       printf("aborted analysis: %d\n", analysis.did_abort);
       for (unsigned i = 0; i < self->repeat_symbols_with_rootless_patterns.size; i++) {
-        TSSymbol symbol = self->repeat_symbols_with_rootless_patterns.contents[i];
+        TSSymbol symbol = *array_get(&self->repeat_symbols_with_rootless_patterns, i);
         printf("  %u, %s\n", symbol, ts_language_symbol_name(self->language, symbol));
       }
       printf("\n");
@@ -1950,8 +1960,8 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
 
   // Cleanup
   for (unsigned i = 0; i < subgraphs.size; i++) {
-    array_delete(&subgraphs.contents[i].start_states);
-    array_delete(&subgraphs.contents[i].nodes);
+    array_delete(&array_get(&subgraphs, i)->start_states);
+    array_delete(&array_get(&subgraphs, i)->nodes);
   }
   array_delete(&subgraphs);
   query_analysis__delete(&analysis);
@@ -1970,7 +1980,7 @@ static void ts_query__add_negated_fields(
   TSFieldId *field_ids,
   uint16_t field_count
 ) {
-  QueryStep *step = &self->steps.contents[step_index];
+  QueryStep *step = array_get(&self->steps, step_index);
 
   // The negated field array stores a list of field lists, separated by zeros.
   // Try to find the start index of an existing list that matches this new list.
@@ -1978,7 +1988,7 @@ static void ts_query__add_negated_fields(
   unsigned match_count = 0;
   unsigned start_i = 0;
   for (unsigned i = 0; i < self->negated_fields.size; i++) {
-    TSFieldId existing_field_id = self->negated_fields.contents[i];
+    TSFieldId existing_field_id = *array_get(&self->negated_fields, i);
 
     // At each zero value, terminate the match attempt. If we've exactly
     // matched the new field list, then reuse this index. Otherwise,
@@ -2248,10 +2258,10 @@ static TSQueryError ts_query__parse_pattern(
     // For all of the branches except for the last one, add the subsequent branch as an
     // alternative, and link the end of the branch to the current end of the steps.
     for (unsigned i = 0; i < branch_step_indices.size - 1; i++) {
-      uint32_t step_index = branch_step_indices.contents[i];
-      uint32_t next_step_index = branch_step_indices.contents[i + 1];
-      QueryStep *start_step = &self->steps.contents[step_index];
-      QueryStep *end_step = &self->steps.contents[next_step_index - 1];
+      uint32_t step_index = *array_get(&branch_step_indices, i);
+      uint32_t next_step_index = *array_get(&branch_step_indices, i + 1);
+      QueryStep *start_step = array_get(&self->steps, step_index);
+      QueryStep *end_step = array_get(&self->steps, next_step_index - 1);
       start_step->alternative_index = next_step_index;
       end_step->alternative_index = self->steps.size;
       end_step->is_dead_end = true;
@@ -2514,6 +2524,9 @@ static TSQueryError ts_query__parse_pattern(
           child_is_immediate,
           &child_capture_quantifiers
         );
+        // In the event we only parsed a predicate, meaning no new steps were added,
+        // then subtract one so we're not indexing past the end of the array
+        if (step_index == self->steps.size) step_index--;
         if (e == PARENT_DONE) {
           if (stream->next == ')') {
             if (child_is_immediate) {
@@ -2522,19 +2535,19 @@ static TSQueryError ts_query__parse_pattern(
                 return TSQueryErrorSyntax;
               }
               // Mark this step *and* its alternatives as the last child of the parent.
-              QueryStep *last_child_step = &self->steps.contents[last_child_step_index];
+              QueryStep *last_child_step = array_get(&self->steps, last_child_step_index);
               last_child_step->is_last_child = true;
               if (
                 last_child_step->alternative_index != NONE &&
                 last_child_step->alternative_index < self->steps.size
               ) {
-                QueryStep *alternative_step = &self->steps.contents[last_child_step->alternative_index];
+                QueryStep *alternative_step = array_get(&self->steps, last_child_step->alternative_index);
                 alternative_step->is_last_child = true;
                 while (
                   alternative_step->alternative_index != NONE &&
                   alternative_step->alternative_index < self->steps.size
                 ) {
-                  alternative_step = &self->steps.contents[alternative_step->alternative_index];
+                  alternative_step = array_get(&self->steps, alternative_step->alternative_index);
                   alternative_step->is_last_child = true;
                 }
               }
@@ -2640,7 +2653,7 @@ static TSQueryError ts_query__parse_pattern(
     }
 
     uint32_t step_index = starting_step_index;
-    QueryStep *step = &self->steps.contents[step_index];
+    QueryStep *step = array_get(&self->steps, step_index);
     for (;;) {
       step->field = field_id;
       if (
@@ -2649,7 +2662,7 @@ static TSQueryError ts_query__parse_pattern(
         step->alternative_index < self->steps.size
       ) {
         step_index = step->alternative_index;
-        step = &self->steps.contents[step_index];
+        step = array_get(&self->steps, step_index);
       } else {
         break;
       }
@@ -2698,9 +2711,9 @@ static TSQueryError ts_query__parse_pattern(
       // Stop when `step->alternative_index` is `NONE` or it points to
       // `repeat_step` or beyond. Note that having just been pushed,
       // `repeat_step` occupies slot `self->steps.size - 1`.
-      QueryStep *step = &self->steps.contents[starting_step_index];
+      QueryStep *step = array_get(&self->steps, starting_step_index);
       while (step->alternative_index != NONE && step->alternative_index < self->steps.size - 1) {
-        step = &self->steps.contents[step->alternative_index];
+        step = array_get(&self->steps, step->alternative_index);
       }
       step->alternative_index = self->steps.size;
     }
@@ -2712,9 +2725,9 @@ static TSQueryError ts_query__parse_pattern(
       stream_advance(stream);
       stream_skip_whitespace(stream);
 
-      QueryStep *step = &self->steps.contents[starting_step_index];
+      QueryStep *step = array_get(&self->steps, starting_step_index);
       while (step->alternative_index != NONE && step->alternative_index < self->steps.size) {
-        step = &self->steps.contents[step->alternative_index];
+        step = array_get(&self->steps, step->alternative_index);
       }
       step->alternative_index = self->steps.size;
     }
@@ -2740,7 +2753,7 @@ static TSQueryError ts_query__parse_pattern(
 
       uint32_t step_index = starting_step_index;
       for (;;) {
-        QueryStep *step = &self->steps.contents[step_index];
+        QueryStep *step = array_get(&self->steps, step_index);
         query_step__add_capture(step, capture_id);
         if (
           step->alternative_index != NONE &&
@@ -2838,14 +2851,14 @@ TSQuery *ts_query_new(
     // Maintain a map that can look up patterns for a given root symbol.
     uint16_t wildcard_root_alternative_index = NONE;
     for (;;) {
-      QueryStep *step = &self->steps.contents[start_step_index];
+      QueryStep *step = array_get(&self->steps, start_step_index);
 
       // If a pattern has a wildcard at its root, but it has a non-wildcard child,
       // then optimize the matching process by skipping matching the wildcard.
       // Later, during the matching process, the query cursor will check that
       // there is a parent node, and capture it if necessary.
       if (step->symbol == WILDCARD_SYMBOL && step->depth == 0 && !step->field) {
-        QueryStep *second_step = &self->steps.contents[start_step_index + 1];
+        QueryStep *second_step = array_get(&self->steps, start_step_index + 1);
         if (second_step->symbol != WILDCARD_SYMBOL && second_step->depth == 1 && !second_step->is_immediate) {
           wildcard_root_alternative_index = step->alternative_index;
           start_step_index += 1;
@@ -2860,7 +2873,7 @@ TSQuery *ts_query_new(
       uint32_t start_depth = step->depth;
       bool is_rooted = start_depth == 0;
       for (uint32_t step_index = start_step_index + 1; step_index < self->steps.size; step_index++) {
-        QueryStep *child_step = &self->steps.contents[step_index];
+        QueryStep *child_step = array_get(&self->steps, step_index);
         if (child_step->is_dead_end) break;
         if (child_step->depth == start_depth) {
           is_rooted = false;
@@ -2964,26 +2977,24 @@ const TSQueryPredicateStep *ts_query_predicates_for_pattern(
   uint32_t pattern_index,
   uint32_t *step_count
 ) {
-  Slice slice = self->patterns.contents[pattern_index].predicate_steps;
+  Slice slice = array_get(&self->patterns, pattern_index)->predicate_steps;
   *step_count = slice.length;
-  if (self->predicate_steps.contents == NULL) {
-    return NULL;
-  }
-  return &self->predicate_steps.contents[slice.offset];
+  if (slice.length == 0) return NULL;
+  return array_get(&self->predicate_steps, slice.offset);
 }
 
 uint32_t ts_query_start_byte_for_pattern(
   const TSQuery *self,
   uint32_t pattern_index
 ) {
-  return self->patterns.contents[pattern_index].start_byte;
+  return array_get(&self->patterns, pattern_index)->start_byte;
 }
 
 uint32_t ts_query_end_byte_for_pattern(
   const TSQuery *self,
   uint32_t pattern_index
 ) {
-  return self->patterns.contents[pattern_index].end_byte;
+  return array_get(&self->patterns, pattern_index)->end_byte;
 }
 
 bool ts_query_is_pattern_rooted(
@@ -2991,7 +3002,7 @@ bool ts_query_is_pattern_rooted(
   uint32_t pattern_index
 ) {
   for (unsigned i = 0; i < self->pattern_map.size; i++) {
-    PatternEntry *entry = &self->pattern_map.contents[i];
+    PatternEntry *entry = array_get(&self->pattern_map, i);
     if (entry->pattern_index == pattern_index) {
       if (!entry->is_rooted) return false;
     }
@@ -3004,7 +3015,7 @@ bool ts_query_is_pattern_non_local(
   uint32_t pattern_index
 ) {
   if (pattern_index < self->patterns.size) {
-    return self->patterns.contents[pattern_index].is_non_local;
+    return array_get(&self->patterns, pattern_index)->is_non_local;
   } else {
     return false;
   }
@@ -3016,12 +3027,12 @@ bool ts_query_is_pattern_guaranteed_at_step(
 ) {
   uint32_t step_index = UINT32_MAX;
   for (unsigned i = 0; i < self->step_offsets.size; i++) {
-    StepOffset *step_offset = &self->step_offsets.contents[i];
+    StepOffset *step_offset = array_get(&self->step_offsets, i);
     if (step_offset->byte_offset > byte_offset) break;
     step_index = step_offset->step_index;
   }
   if (step_index < self->steps.size) {
-    return self->steps.contents[step_index].root_pattern_guaranteed;
+    return array_get(&self->steps, step_index)->root_pattern_guaranteed;
   } else {
     return false;
   }
@@ -3032,8 +3043,8 @@ bool ts_query__step_is_fallible(
   uint16_t step_index
 ) {
   ts_assert((uint32_t)step_index + 1 < self->steps.size);
-  QueryStep *step = &self->steps.contents[step_index];
-  QueryStep *next_step = &self->steps.contents[step_index + 1];
+  QueryStep *step = array_get(&self->steps, step_index);
+  QueryStep *next_step = array_get(&self->steps, step_index + 1);
   return (
     next_step->depth != PATTERN_DONE_MARKER &&
     next_step->depth > step->depth &&
@@ -3051,7 +3062,7 @@ void ts_query_disable_capture(
   int id = symbol_table_id_for_name(&self->captures, name, length);
   if (id != -1) {
     for (unsigned i = 0; i < self->steps.size; i++) {
-      QueryStep *step = &self->steps.contents[i];
+      QueryStep *step = array_get(&self->steps, i);
       query_step__remove_capture(step, id);
     }
   }
@@ -3064,7 +3075,7 @@ void ts_query_disable_pattern(
   // Remove the given pattern from the pattern map. Its steps will still
   // be in the `steps` array, but they will never be read.
   for (unsigned i = 0; i < self->pattern_map.size; i++) {
-    PatternEntry *pattern = &self->pattern_map.contents[i];
+    PatternEntry *pattern = array_get(&self->pattern_map, i);
     if (pattern->pattern_index == pattern_index) {
       array_erase(&self->pattern_map, i);
       i--;
@@ -3141,7 +3152,7 @@ void ts_query_cursor_exec(
   if (query) {
     LOG("query steps:\n");
     for (unsigned i = 0; i < query->steps.size; i++) {
-      QueryStep *step = &query->steps.contents[i];
+      QueryStep *step = array_get(&query->steps, i);
       LOG("  %u: {", i);
       if (step->depth == PATTERN_DONE_MARKER) {
         LOG("DONE");
@@ -3246,7 +3257,7 @@ static bool ts_query_cursor__first_in_progress_capture(
   *byte_offset = UINT32_MAX;
   *pattern_index = UINT32_MAX;
   for (unsigned i = 0; i < self->states.size; i++) {
-    QueryState *state = &self->states.contents[i];
+    QueryState *state = array_get(&self->states, i);
     if (state->dead) continue;
 
     const CaptureList *captures = capture_list_pool_get(
@@ -3257,7 +3268,7 @@ static bool ts_query_cursor__first_in_progress_capture(
       continue;
     }
 
-    TSNode node = captures->contents[state->consumed_capture_count].node;
+    TSNode node = array_get(captures, state->consumed_capture_count)->node;
     if (
       ts_node_end_byte(node) <= self->start_byte ||
       point_lte(ts_node_end_point(node), self->start_point)
@@ -3273,7 +3284,7 @@ static bool ts_query_cursor__first_in_progress_capture(
       node_start_byte < *byte_offset ||
       (node_start_byte == *byte_offset && state->pattern_index < *pattern_index)
     ) {
-      QueryStep *step = &self->query->steps.contents[state->step_index];
+      QueryStep *step = array_get(&self->query->steps, state->step_index);
       if (is_definite) {
         // We're being a bit conservative here by asserting that the following step
         // is not immediate, because this capture might end up being discarded if the
@@ -3329,8 +3340,8 @@ void ts_query_cursor__compare_captures(
   for (;;) {
     if (i < left_captures->size) {
       if (j < right_captures->size) {
-        TSQueryCapture *left = &left_captures->contents[i];
-        TSQueryCapture *right = &right_captures->contents[j];
+        TSQueryCapture *left = array_get(left_captures, i);
+        TSQueryCapture *right = array_get(right_captures, j);
         if (left->node.id == right->node.id && left->index == right->index) {
           i++;
           j++;
@@ -3369,7 +3380,7 @@ static void ts_query_cursor__add_state(
   TSQueryCursor *self,
   const PatternEntry *pattern
 ) {
-  QueryStep *step = &self->query->steps.contents[pattern->step_index];
+  QueryStep *step = array_get(&self->query->steps, pattern->step_index);
   uint32_t start_depth = self->depth - step->depth;
 
   // Keep the states array in ascending order of start_depth and pattern_index,
@@ -3393,7 +3404,7 @@ static void ts_query_cursor__add_state(
   // need to execute in order to keep the states ordered by pattern_index.
   uint32_t index = self->states.size;
   while (index > 0) {
-    QueryState *prev_state = &self->states.contents[index - 1];
+    QueryState *prev_state = array_get(&self->states, index - 1);
     if (prev_state->start_depth < start_depth) break;
     if (prev_state->start_depth == start_depth) {
       // Avoid inserting an unnecessary duplicate state, which would be
@@ -3457,7 +3468,7 @@ static CaptureList *ts_query_cursor__prepare_to_capture(
           "  abandon state. index:%u, pattern:%u, offset:%u.\n",
           state_index, pattern_index, byte_offset
         );
-        QueryState *other_state = &self->states.contents[state_index];
+        QueryState *other_state = array_get(&self->states, state_index);
         state->capture_list_id = other_state->capture_list_id;
         other_state->capture_list_id = NONE;
         other_state->dead = true;
@@ -3527,8 +3538,8 @@ static QueryState *ts_query_cursor__copy_state(
   }
 
   array_insert(&self->states, state_index + 1, copy);
-  *state_ref = &self->states.contents[state_index];
-  return &self->states.contents[state_index + 1];
+  *state_ref = array_get(&self->states, state_index);
+  return array_get(&self->states, state_index + 1);
 }
 
 static inline bool ts_query_cursor__should_descend(
@@ -3543,8 +3554,8 @@ static inline bool ts_query_cursor__should_descend(
   // If there are in-progress matches whose remaining steps occur
   // deeper in the tree, then descend.
   for (unsigned i = 0; i < self->states.size; i++) {
-    QueryState *state = &self->states.contents[i];;
-    QueryStep *next_step = &self->query->steps.contents[state->step_index];
+    QueryState *state = array_get(&self->states, i);
+    QueryStep *next_step = array_get(&self->query->steps, state->step_index);
     if (
       next_step->depth != PATTERN_DONE_MARKER &&
       state->start_depth + next_step->depth > self->depth
@@ -3638,8 +3649,8 @@ static inline bool ts_query_cursor__advance(
         // After leaving a node, remove any states that cannot make further progress.
         uint32_t deleted_count = 0;
         for (unsigned i = 0, n = self->states.size; i < n; i++) {
-          QueryState *state = &self->states.contents[i];
-          QueryStep *step = &self->query->steps.contents[state->step_index];
+          QueryState *state = array_get(&self->states, i);
+          QueryStep *step = array_get(&self->query->steps, state->step_index);
 
           // If a state completed its pattern inside of this node, but was deferred from finishing
           // in order to search for longer matches, mark it as finished.
@@ -3672,7 +3683,7 @@ static inline bool ts_query_cursor__advance(
           }
 
           else if (deleted_count > 0) {
-            self->states.contents[i - deleted_count] = *state;
+            *array_get(&self->states, i - deleted_count) = *state;
           }
         }
         self->states.size -= deleted_count;
@@ -3775,11 +3786,11 @@ static inline bool ts_query_cursor__advance(
         // Add new states for any patterns whose root node is a wildcard.
         if (!node_is_error) {
           for (unsigned i = 0; i < self->query->wildcard_root_pattern_count; i++) {
-            PatternEntry *pattern = &self->query->pattern_map.contents[i];
+            PatternEntry *pattern = array_get(&self->query->pattern_map, i);
 
             // If this node matches the first step of the pattern, then add a new
             // state at the start of this pattern.
-            QueryStep *step = &self->query->steps.contents[pattern->step_index];
+            QueryStep *step = array_get(&self->query->steps, pattern->step_index);
             uint32_t start_depth = self->depth - step->depth;
             if (
               (pattern->is_rooted ?
@@ -3797,9 +3808,9 @@ static inline bool ts_query_cursor__advance(
         // Add new states for any patterns whose root node matches this node.
         unsigned i;
         if (ts_query__pattern_map_search(self->query, symbol, &i)) {
-          PatternEntry *pattern = &self->query->pattern_map.contents[i];
+          PatternEntry *pattern = array_get(&self->query->pattern_map, i);
 
-          QueryStep *step = &self->query->steps.contents[pattern->step_index];
+          QueryStep *step = array_get(&self->query->steps, pattern->step_index);
           uint32_t start_depth = self->depth - step->depth;
           do {
             // If this node matches the first step of the pattern, then add a new
@@ -3817,15 +3828,15 @@ static inline bool ts_query_cursor__advance(
             // Advance to the next pattern whose root node matches this node.
             i++;
             if (i == self->query->pattern_map.size) break;
-            pattern = &self->query->pattern_map.contents[i];
-            step = &self->query->steps.contents[pattern->step_index];
+            pattern = array_get(&self->query->pattern_map, i);
+            step = array_get(&self->query->steps, pattern->step_index);
           } while (step->symbol == symbol);
         }
 
         // Update all of the in-progress states with current node.
         for (unsigned j = 0, copy_count = 0; j < self->states.size; j += 1 + copy_count) {
-          QueryState *state = &self->states.contents[j];
-          QueryStep *step = &self->query->steps.contents[state->step_index];
+          QueryState *state = array_get(&self->states, j);
+          QueryStep *step = array_get(&self->query->steps, state->step_index);
           state->has_in_progress_alternatives = false;
           copy_count = 0;
 
@@ -3874,7 +3885,7 @@ static inline bool ts_query_cursor__advance(
           }
 
           if (step->negated_field_list_id) {
-            TSFieldId *negated_field_ids = &self->query->negated_fields.contents[step->negated_field_list_id];
+            TSFieldId *negated_field_ids = array_get(&self->query->negated_fields, step->negated_field_list_id);
             for (;;) {
               TSFieldId negated_field_id = *negated_field_ids;
               if (negated_field_id) {
@@ -3975,7 +3986,7 @@ static inline bool ts_query_cursor__advance(
             state->step_index
           );
 
-          QueryStep *next_step = &self->query->steps.contents[state->step_index];
+          QueryStep *next_step = array_get(&self->query->steps, state->step_index);
 
           // For a given step, if the current symbol is the wildcard symbol, `_`, and it is **not**
           // named, meaning it should capture anonymous nodes, **and** the next step is immediate,
@@ -3998,8 +4009,8 @@ static inline bool ts_query_cursor__advance(
           // so this is an interactive process.
           unsigned end_index = j + 1;
           for (unsigned k = j; k < end_index; k++) {
-            QueryState *child_state = &self->states.contents[k];
-            QueryStep *child_step = &self->query->steps.contents[child_state->step_index];
+            QueryState *child_state = array_get(&self->states, k);
+            QueryStep *child_step = array_get(&self->query->steps, child_state->step_index);
             if (child_step->alternative_index != NONE) {
               // A "dead-end" step exists only to add a non-sequential jump into the step sequence,
               // via its alternative index. When a state reaches a dead-end step, it jumps straight
@@ -4040,7 +4051,7 @@ static inline bool ts_query_cursor__advance(
         }
 
         for (unsigned j = 0; j < self->states.size; j++) {
-          QueryState *state = &self->states.contents[j];
+          QueryState *state = array_get(&self->states, j);
           if (state->dead) {
             array_erase(&self->states, j);
             j--;
@@ -4052,7 +4063,7 @@ static inline bool ts_query_cursor__advance(
           // one state has a strict subset of another state's captures.
           bool did_remove = false;
           for (unsigned k = j + 1; k < self->states.size; k++) {
-            QueryState *other_state = &self->states.contents[k];
+            QueryState *other_state = array_get(&self->states, k);
 
             // Query states are kept in ascending order of start_depth and pattern_index.
             // Since the longest-match criteria is only used for deduping matches of the same
@@ -4112,7 +4123,7 @@ static inline bool ts_query_cursor__advance(
               state->step_index,
               capture_list_pool_get(&self->capture_list_pool, state->capture_list_id)->size
             );
-            QueryStep *next_step = &self->query->steps.contents[state->step_index];
+            QueryStep *next_step = array_get(&self->query->steps, state->step_index);
             if (next_step->depth == PATTERN_DONE_MARKER) {
               if (state->has_in_progress_alternatives) {
                 LOG("  defer finishing pattern %u\n", state->pattern_index);
@@ -4157,7 +4168,7 @@ bool ts_query_cursor_next_match(
     }
   }
 
-  QueryState *state = &self->finished_states.contents[0];
+  QueryState *state = array_get(&self->finished_states, 0);
   if (state->id == UINT32_MAX) state->id = self->next_state_id++;
   match->id = state->id;
   match->pattern_index = state->pattern_index;
@@ -4177,7 +4188,7 @@ void ts_query_cursor_remove_match(
   uint32_t match_id
 ) {
   for (unsigned i = 0; i < self->finished_states.size; i++) {
-    const QueryState *state = &self->finished_states.contents[i];
+    const QueryState *state = array_get(&self->finished_states, i);
     if (state->id == match_id) {
       capture_list_pool_release(
         &self->capture_list_pool,
@@ -4191,7 +4202,7 @@ void ts_query_cursor_remove_match(
   // Remove unfinished query states as well to prevent future
   // captures for a match being removed.
   for (unsigned i = 0; i < self->states.size; i++) {
-    const QueryState *state = &self->states.contents[i];
+    const QueryState *state = array_get(&self->states, i);
     if (state->id == match_id) {
       capture_list_pool_release(
         &self->capture_list_pool,
@@ -4231,7 +4242,7 @@ bool ts_query_cursor_next_capture(
     uint32_t first_finished_capture_byte = first_unfinished_capture_byte;
     uint32_t first_finished_pattern_index = first_unfinished_pattern_index;
     for (unsigned i = 0; i < self->finished_states.size;) {
-      QueryState *state = &self->finished_states.contents[i];
+      QueryState *state = array_get(&self->finished_states, i);
       const CaptureList *captures = capture_list_pool_get(
         &self->capture_list_pool,
         state->capture_list_id
@@ -4247,7 +4258,7 @@ bool ts_query_cursor_next_capture(
         continue;
       }
 
-      TSNode node = captures->contents[state->consumed_capture_count].node;
+      TSNode node = array_get(captures, state->consumed_capture_count)->node;
 
       bool node_precedes_range = (
         ts_node_end_byte(node) <= self->start_byte ||
@@ -4287,7 +4298,7 @@ bool ts_query_cursor_next_capture(
     if (first_finished_state) {
       state = first_finished_state;
     } else if (first_unfinished_state_is_definite) {
-      state = &self->states.contents[first_unfinished_state_index];
+      state = array_get(&self->states, first_unfinished_state_index);
     } else {
       state = NULL;
     }
@@ -4316,7 +4327,7 @@ bool ts_query_cursor_next_capture(
       );
       capture_list_pool_release(
         &self->capture_list_pool,
-        self->states.contents[first_unfinished_state_index].capture_list_id
+        array_get(&self->states, first_unfinished_state_index)->capture_list_id
       );
       array_erase(&self->states, first_unfinished_state_index);
     }

lib/src/stack.c

@@ -290,8 +290,8 @@ static StackVersion ts_stack__add_version(
 ) {
   StackHead head = {
     .node = node,
-    .node_count_at_last_error = self->heads.contents[original_version].node_count_at_last_error,
-    .last_external_token = self->heads.contents[original_version].last_external_token,
+    .node_count_at_last_error = array_get(&self->heads, original_version)->node_count_at_last_error,
+    .last_external_token = array_get(&self->heads, original_version)->last_external_token,
     .status = StackStatusActive,
     .lookahead_when_paused = NULL_SUBTREE,
   };
@@ -308,8 +308,8 @@ static void ts_stack__add_slice(
   SubtreeArray *subtrees
 ) {
   for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) {
-    StackVersion version = self->slices.contents[i].version;
-    if (self->heads.contents[version].node == node) {
+    StackVersion version = array_get(&self->slices, i)->version;
+    if (array_get(&self->heads, version)->node == node) {
       StackSlice slice = {*subtrees, version};
       array_insert(&self->slices, i + 1, slice);
       return;
@@ -349,7 +349,7 @@ static StackSliceArray stack__iter(
 
   while (self->iterators.size > 0) {
     for (uint32_t i = 0, size = self->iterators.size; i < size; i++) {
-      StackIterator *iterator = &self->iterators.contents[i];
+      StackIterator *iterator = array_get(&self->iterators, i);
       StackNode *node = iterator->node;
 
       StackAction action = callback(payload, iterator);
@@ -384,11 +384,11 @@ static StackSliceArray stack__iter(
         StackLink link;
         if (j == node->link_count) {
           link = node->links[0];
-          next_iterator = &self->iterators.contents[i];
+          next_iterator = array_get(&self->iterators, i);
         } else {
           if (self->iterators.size >= MAX_ITERATOR_COUNT) continue;
           link = node->links[j];
-          StackIterator current_iterator = self->iterators.contents[i];
+          StackIterator current_iterator = *array_get(&self->iterators, i);
           array_push(&self->iterators, current_iterator);
           next_iterator = array_back(&self->iterators);
           ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees);
@@ -444,12 +444,12 @@ void ts_stack_delete(Stack *self) {
     array_delete(&self->iterators);
   stack_node_release(self->base_node, &self->node_pool, self->subtree_pool);
   for (uint32_t i = 0; i < self->heads.size; i++) {
-    stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
+    stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool);
   }
   array_clear(&self->heads);
   if (self->node_pool.contents) {
     for (uint32_t i = 0; i < self->node_pool.size; i++)
-      ts_free(self->node_pool.contents[i]);
+      ts_free(*array_get(&self->node_pool, i));
     array_delete(&self->node_pool);
   }
   array_delete(&self->heads);
@@ -460,6 +460,17 @@ uint32_t ts_stack_version_count(const Stack *self) {
   return self->heads.size;
 }
 
+uint32_t ts_stack_halted_version_count(Stack *self) {
+  uint32_t count = 0;
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    StackHead *head = array_get(&self->heads, i);
+    if (head->status == StackStatusHalted) {
+      count++;
+    }
+  }
+  return count;
+}
+
 TSStateId ts_stack_state(const Stack *self, StackVersion version) {
   return array_get(&self->heads, version)->node->state;
 }
@@ -524,6 +535,7 @@ StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t c
   return stack__iter(self, version, pop_count_callback, &count, (int)count);
 }
 
+
 forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) {
   (void)payload;
   if (iterator->subtree_count >= 1) {
@@ -540,8 +552,8 @@ forceinline StackAction pop_pending_callback(void *payload, const StackIterator
 StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) {
   StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0);
   if (pop.size > 0) {
-    ts_stack_renumber_version(self, pop.contents[0].version, version);
-    pop.contents[0].version = version;
+    ts_stack_renumber_version(self, array_get(&pop, 0)->version, version);
+    array_get(&pop, 0)->version = version;
   }
   return pop;
 }
@@ -549,7 +561,7 @@ StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) {
 forceinline StackAction pop_error_callback(void *payload, const StackIterator *iterator) {
   if (iterator->subtrees.size > 0) {
     bool *found_error = payload;
-    if (!*found_error && ts_subtree_is_error(iterator->subtrees.contents[0])) {
+    if (!*found_error && ts_subtree_is_error(*array_get(&iterator->subtrees, 0))) {
       *found_error = true;
       return StackActionPop | StackActionStop;
     } else {
@@ -568,8 +580,8 @@ SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
       StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1);
       if (pop.size > 0) {
         ts_assert(pop.size == 1);
-        ts_stack_renumber_version(self, pop.contents[0].version, version);
-        return pop.contents[0].subtrees;
+        ts_stack_renumber_version(self, array_get(&pop, 0)->version, version);
+        return array_get(&pop, 0)->subtrees;
       }
       break;
     }
@@ -597,7 +609,7 @@ forceinline StackAction summarize_stack_callback(void *payload, const StackItera
   unsigned depth = iterator->subtree_count;
   if (depth > session->max_depth) return StackActionStop;
   for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) {
-    StackSummaryEntry entry = session->summary->contents[i];
+    StackSummaryEntry entry = *array_get(session->summary, i);
     if (entry.depth < depth) break;
     if (entry.depth == depth && entry.state == state) return StackActionNone;
   }
@@ -616,7 +628,7 @@ void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_dep
   };
   array_init(session.summary);
   stack__iter(self, version, summarize_stack_callback, &session, -1);
-  StackHead *head = &self->heads.contents[version];
+  StackHead *head = array_get(&self->heads, version);
   if (head->summary) {
     array_delete(head->summary);
     ts_free(head->summary);
@@ -665,8 +677,8 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
   if (v1 == v2) return;
   ts_assert(v2 < v1);
   ts_assert((uint32_t)v1 < self->heads.size);
-  StackHead *source_head = &self->heads.contents[v1];
-  StackHead *target_head = &self->heads.contents[v2];
+  StackHead *source_head = array_get(&self->heads, v1);
+  StackHead *target_head = array_get(&self->heads, v2);
   if (target_head->summary && !source_head->summary) {
     source_head->summary = target_head->summary;
     target_head->summary = NULL;
@@ -677,14 +689,15 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
 }
 
 void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) {
-  StackHead temporary_head = self->heads.contents[v1];
-  self->heads.contents[v1] = self->heads.contents[v2];
-  self->heads.contents[v2] = temporary_head;
+  StackHead temporary_head = *array_get(&self->heads, v1);
+  *array_get(&self->heads, v1) = *array_get(&self->heads, v2);
+  *array_get(&self->heads, v2) = temporary_head;
 }
 
 StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
   ts_assert(version < self->heads.size);
-  array_push(&self->heads, self->heads.contents[version]);
+  StackHead version_head = *array_get(&self->heads, version);
+  array_push(&self->heads, version_head);
   StackHead *head = array_back(&self->heads);
   stack_node_retain(head->node);
   if (head->last_external_token.ptr) ts_subtree_retain(head->last_external_token);
@@ -694,8 +707,8 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
 
 bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
   if (!ts_stack_can_merge(self, version1, version2)) return false;
-  StackHead *head1 = &self->heads.contents[version1];
-  StackHead *head2 = &self->heads.contents[version2];
+  StackHead *head1 = array_get(&self->heads, version1);
+  StackHead *head2 = array_get(&self->heads, version2);
   for (uint32_t i = 0; i < head2->node->link_count; i++) {
     stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool);
   }
@@ -707,8 +720,8 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
 }
 
 bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) {
-  StackHead *head1 = &self->heads.contents[version1];
-  StackHead *head2 = &self->heads.contents[version2];
+  StackHead *head1 = array_get(&self->heads, version1);
+  StackHead *head2 = array_get(&self->heads, version2);
   return
     head1->status == StackStatusActive &&
     head2->status == StackStatusActive &&
@@ -753,7 +766,7 @@ Subtree ts_stack_resume(Stack *self, StackVersion version) {
 void ts_stack_clear(Stack *self) {
   stack_node_retain(self->base_node);
   for (uint32_t i = 0; i < self->heads.size; i++) {
-    stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
+    stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool);
   }
   array_clear(&self->heads);
   array_push(&self->heads, ((StackHead) {
@@ -776,7 +789,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
 
   array_clear(&self->iterators);
   for (uint32_t i = 0; i < self->heads.size; i++) {
-    StackHead *head = &self->heads.contents[i];
+    StackHead *head = array_get(&self->heads, i);
     if (head->status == StackStatusHalted) continue;
 
     fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i);
@@ -794,7 +807,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
 
     if (head->summary) {
       fprintf(f, "\nsummary:");
-      for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", head->summary->contents[j].state);
+      for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", array_get(head->summary, j)->state);
     }
 
     if (head->last_external_token.ptr) {
@@ -815,11 +828,11 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
     all_iterators_done = true;
 
     for (uint32_t i = 0; i < self->iterators.size; i++) {
-      StackIterator iterator = self->iterators.contents[i];
+      StackIterator iterator = *array_get(&self->iterators, i);
       StackNode *node = iterator.node;
 
       for (uint32_t j = 0; j < visited_nodes.size; j++) {
-        if (visited_nodes.contents[j] == node) {
+        if (*array_get(&visited_nodes, j) == node) {
           node = NULL;
           break;
         }
@@ -878,7 +891,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
 
         StackIterator *next_iterator;
         if (j == 0) {
-          next_iterator = &self->iterators.contents[i];
+          next_iterator = array_get(&self->iterators, i);
         } else {
           array_push(&self->iterators, iterator);
           next_iterator = array_back(&self->iterators);

lib/src/stack.h

@@ -36,6 +36,9 @@ void ts_stack_delete(Stack *self);
 // Get the stack's current number of versions.
 uint32_t ts_stack_version_count(const Stack *self);
 
+// Get the stack's current number of halted versions.
+uint32_t ts_stack_halted_version_count(Stack *self);
+
 // Get the state at the top of the given version of the stack. If the stack is
 // empty, this returns the initial state, 0.
 TSStateId ts_stack_state(const Stack *self, StackVersion version);

lib/src/subtree.c

@@ -73,14 +73,14 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) {
     dest->contents = ts_calloc(self.capacity, sizeof(Subtree));
     memcpy(dest->contents, self.contents, self.size * sizeof(Subtree));
     for (uint32_t i = 0; i < self.size; i++) {
-      ts_subtree_retain(dest->contents[i]);
+      ts_subtree_retain(*array_get(dest, i));
     }
   }
 }
 
 void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self) {
   for (uint32_t i = 0; i < self->size; i++) {
-    ts_subtree_release(pool, self->contents[i]);
+    ts_subtree_release(pool, *array_get(self, i));
   }
   array_clear(self);
 }
@@ -96,7 +96,7 @@ void ts_subtree_array_remove_trailing_extras(
 ) {
   array_clear(destination);
   while (self->size > 0) {
-    Subtree last = self->contents[self->size - 1];
+    Subtree last = *array_get(self, self->size - 1);
     if (ts_subtree_extra(last)) {
       self->size--;
       array_push(destination, last);
@@ -110,9 +110,9 @@ void ts_subtree_array_remove_trailing_extras(
 void ts_subtree_array_reverse(SubtreeArray *self) {
   for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) {
     size_t reverse_index = self->size - 1 - i;
-    Subtree swap = self->contents[i];
-    self->contents[i] = self->contents[reverse_index];
-    self->contents[reverse_index] = swap;
+    Subtree swap = *array_get(self, i);
+    *array_get(self, i) = *array_get(self, reverse_index);
+    *array_get(self, reverse_index) = swap;
   }
 }
 
@@ -127,7 +127,7 @@ SubtreePool ts_subtree_pool_new(uint32_t capacity) {
 void ts_subtree_pool_delete(SubtreePool *self) {
   if (self->free_trees.contents) {
     for (unsigned i = 0; i < self->free_trees.size; i++) {
-      ts_free(self->free_trees.contents[i].ptr);
+      ts_free(array_get(&self->free_trees, i)->ptr);
     }
     array_delete(&self->free_trees);
   }
@@ -407,7 +407,12 @@ void ts_subtree_summarize_children(
     self.ptr->dynamic_precedence += ts_subtree_dynamic_precedence(child);
     self.ptr->visible_descendant_count += ts_subtree_visible_descendant_count(child);
 
-    if (alias_sequence && alias_sequence[structural_index] != 0 && !ts_subtree_extra(child)) {
+    if (
+      !ts_subtree_extra(child) &&
+      ts_subtree_symbol(child) != 0 &&
+      alias_sequence &&
+      alias_sequence[structural_index] != 0
+    ) {
       self.ptr->visible_descendant_count++;
       self.ptr->visible_child_count++;
       if (ts_language_symbol_metadata(language, alias_sequence[structural_index]).named) {

lib/src/tree_cursor.c

@@ -16,11 +16,11 @@ typedef struct {
 // CursorChildIterator
 
 static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) {
-  TreeCursorEntry *entry = &self->stack.contents[index];
+  TreeCursorEntry *entry = array_get(&self->stack, index);
   if (index == 0 || ts_subtree_visible(*entry->subtree)) {
     return true;
   } else if (!ts_subtree_extra(*entry->subtree)) {
-    TreeCursorEntry *parent_entry = &self->stack.contents[index - 1];
+    TreeCursorEntry *parent_entry = array_get(&self->stack, index - 1);
     return ts_language_alias_at(
       self->tree->language,
       parent_entry->subtree->ptr->production_id,
@@ -129,14 +129,17 @@ static inline bool ts_tree_cursor_child_iterator_previous(
   };
   *visible = ts_subtree_visible(*child);
   bool extra = ts_subtree_extra(*child);
-  if (!extra && self->alias_sequence) {
-    *visible |= self->alias_sequence[self->structural_child_index];
-    self->structural_child_index--;
-  }
 
   self->position = length_backtrack(self->position, ts_subtree_padding(*child));
   self->child_index--;
 
+  if (!extra && self->alias_sequence) {
+    *visible |= self->alias_sequence[self->structural_child_index];
+    if (self->structural_child_index > 0) {
+      self->structural_child_index--;
+    }
+  }
+
   // unsigned can underflow so compare it to child_count
   if (self->child_index < self->parent.ptr->child_count) {
     Subtree previous_child = ts_subtree_children(self->parent)[self->child_index];
@@ -304,8 +307,9 @@ int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint go
 }
 
 TreeCursorStep ts_tree_cursor_goto_sibling_internal(
-    TSTreeCursor *_self,
-    bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) {
+  TSTreeCursor *_self,
+  bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)
+) {
   TreeCursor *self = (TreeCursor *)_self;
   uint32_t initial_size = self->stack.size;
 
@@ -370,7 +374,7 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self
     return step;
 
   // restore position from the parent node
-  const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2];
+  const TreeCursorEntry *parent = array_get(&self->stack, self->stack.size - 2);
   Length position = parent->position;
   uint32_t child_index = array_back(&self->stack)->child_index;
   const Subtree *children = ts_subtree_children((*(parent->subtree)));
@@ -421,7 +425,7 @@ void ts_tree_cursor_goto_descendant(
   // Ascend to the lowest ancestor that contains the goal node.
   for (;;) {
     uint32_t i = self->stack.size - 1;
-    TreeCursorEntry *entry = &self->stack.contents[i];
+    TreeCursorEntry *entry = array_get(&self->stack, i);
     uint32_t next_descendant_index =
       entry->descendant_index +
       (ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) +
@@ -475,7 +479,7 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
   bool is_extra = ts_subtree_extra(*last_entry->subtree);
   TSSymbol alias_symbol = is_extra ? 0 : self->root_alias_symbol;
   if (self->stack.size > 1 && !is_extra) {
-    TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2];
+    TreeCursorEntry *parent_entry = array_get(&self->stack, self->stack.size - 2);
     alias_symbol = ts_language_alias_at(
       self->tree->language,
       parent_entry->subtree->ptr->production_id,
@@ -512,8 +516,8 @@ void ts_tree_cursor_current_status(
   // Walk up the tree, visiting the current node and its invisible ancestors,
   // because fields can refer to nodes through invisible *wrapper* nodes,
   for (unsigned i = self->stack.size - 1; i > 0; i--) {
-    TreeCursorEntry *entry = &self->stack.contents[i];
-    TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
+    TreeCursorEntry *entry = array_get(&self->stack, i);
+    TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1);
 
     const TSSymbol *alias_sequence = ts_language_alias_sequence(
       self->tree->language,
@@ -626,11 +630,11 @@ uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) {
 TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) {
   const TreeCursor *self = (const TreeCursor *)_self;
   for (int i = (int)self->stack.size - 2; i >= 0; i--) {
-    TreeCursorEntry *entry = &self->stack.contents[i];
+    TreeCursorEntry *entry = array_get(&self->stack, i);
     bool is_visible = true;
     TSSymbol alias_symbol = 0;
     if (i > 0) {
-      TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
+      TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1);
       alias_symbol = ts_language_alias_at(
         self->tree->language,
         parent_entry->subtree->ptr->production_id,
@@ -655,8 +659,8 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
 
   // Walk up the tree, visiting the current node and its invisible ancestors.
   for (unsigned i = self->stack.size - 1; i > 0; i--) {
-    TreeCursorEntry *entry = &self->stack.contents[i];
-    TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
+    TreeCursorEntry *entry = array_get(&self->stack, i);
+    TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1);
 
     // Stop walking up when another visible node is found.
     if (

lib/src/wasm_store.c

@@ -946,7 +946,7 @@ void ts_wasm_store_delete(TSWasmStore *self) {
   wasmtime_store_delete(self->store);
   wasm_engine_delete(self->engine);
   for (unsigned i = 0; i < self->language_instances.size; i++) {
-    LanguageWasmInstance *instance = &self->language_instances.contents[i];
+    LanguageWasmInstance *instance = array_get(&self->language_instances, i);
     language_id_delete(instance->language_id);
   }
   array_delete(&self->language_instances);
@@ -956,7 +956,7 @@ void ts_wasm_store_delete(TSWasmStore *self) {
 size_t ts_wasm_store_language_count(const TSWasmStore *self) {
   size_t result = 0;
   for (unsigned i = 0; i < self->language_instances.size; i++) {
-    const WasmLanguageId *id = self->language_instances.contents[i].language_id;
+    const WasmLanguageId *id = array_get(&self->language_instances, i)->language_id;
     if (!id->is_language_deleted) {
       result++;
     }
@@ -1220,6 +1220,10 @@ const TSLanguage *ts_wasm_store_load_language(
   const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
   memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory));
 
+  bool has_supertypes =
+    wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS &&
+    wasm_language.supertype_count > 0;
+
   int32_t addresses[] = {
     wasm_language.parse_table,
     wasm_language.small_parse_table,
@@ -1239,9 +1243,9 @@ const TSLanguage *ts_wasm_store_load_language(
     wasm_language.primary_state_ids,
     wasm_language.name,
     wasm_language.reserved_words,
-    wasm_language.supertype_symbols,
-    wasm_language.supertype_map_entries,
-    wasm_language.supertype_map_slices,
+    has_supertypes ? wasm_language.supertype_symbols : 0,
+    has_supertypes ? wasm_language.supertype_map_entries : 0,
+    has_supertypes ? wasm_language.supertype_map_slices : 0,
     wasm_language.external_token_count > 0 ? wasm_language.external_scanner.states : 0,
     wasm_language.external_token_count > 0 ? wasm_language.external_scanner.symbol_map : 0,
     wasm_language.external_token_count > 0 ? wasm_language.external_scanner.create : 0,
@@ -1331,7 +1335,7 @@ const TSLanguage *ts_wasm_store_load_language(
     );
   }
 
-  if (language->supertype_count > 0) {
+  if (has_supertypes) {
     language->supertype_symbols = copy(
       &memory[wasm_language.supertype_symbols],
       wasm_language.supertype_count * sizeof(TSSymbol)
@@ -1446,7 +1450,7 @@ const TSLanguage *ts_wasm_store_load_language(
 
   // Clear out any instances of languages that have been deleted.
   for (unsigned i = 0; i < self->language_instances.size; i++) {
-    WasmLanguageId *id = self->language_instances.contents[i].language_id;
+    WasmLanguageId *id = array_get(&self->language_instances, i)->language_id;
     if (id->is_language_deleted) {
       language_id_delete(id);
       array_erase(&self->language_instances, i);
@@ -1487,7 +1491,7 @@ bool ts_wasm_store_add_language(
   // instances of languages that have been deleted.
   bool exists = false;
   for (unsigned i = 0; i < self->language_instances.size; i++) {
-    WasmLanguageId *id = self->language_instances.contents[i].language_id;
+    WasmLanguageId *id = array_get(&self->language_instances, i)->language_id;
     if (id->is_language_deleted) {
       language_id_delete(id);
       array_erase(&self->language_instances, i);
@@ -1558,7 +1562,7 @@ bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *la
   uint32_t instance_index;
   if (!ts_wasm_store_add_language(self, language, &instance_index)) return false;
   self->current_lexer = lexer;
-  self->current_instance = &self->language_instances.contents[instance_index];
+  self->current_instance = array_get(&self->language_instances, instance_index);
   self->has_error = false;
   ts_wasm_store_reset_heap(self);
   return true;
@@ -1609,13 +1613,22 @@ static void ts_wasm_store__call(
   }
 }
 
+// The data fields of TSLexer, without the function pointers.
+//
+// This portion of the struct needs to be copied in and out
+// of wasm memory before and after calling a scan function.
+typedef struct {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+} TSLexerDataPrefix;
+
 static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned function_index, TSStateId state) {
   wasmtime_context_t *context = wasmtime_store_context(self->store);
   uint8_t *memory_data = wasmtime_memory_data(context, &self->memory);
   memcpy(
     &memory_data[self->lexer_address],
-    &self->current_lexer->lookahead,
-    sizeof(self->current_lexer->lookahead)
+    self->current_lexer,
+    sizeof(TSLexerDataPrefix)
   );
 
   wasmtime_val_raw_t args[2] = {
@@ -1627,9 +1640,9 @@ static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned functio
   bool result = args[0].i32;
 
   memcpy(
-    &self->current_lexer->lookahead,
+    self->current_lexer,
     &memory_data[self->lexer_address],
-    sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol)
+    sizeof(TSLexerDataPrefix)
   );
   return result;
 }
@@ -1674,8 +1687,8 @@ bool ts_wasm_store_call_scanner_scan(
 
   memcpy(
     &memory_data[self->lexer_address],
-    &self->current_lexer->lookahead,
-    sizeof(self->current_lexer->lookahead)
+    self->current_lexer,
+    sizeof(TSLexerDataPrefix)
   );
 
   uint32_t valid_tokens_address =
@@ -1690,9 +1703,9 @@ bool ts_wasm_store_call_scanner_scan(
   if (self->has_error) return false;
 
   memcpy(
-    &self->current_lexer->lookahead,
+    self->current_lexer,
     &memory_data[self->lexer_address],
-    sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol)
+    sizeof(TSLexerDataPrefix)
   );
   return args[0].i32;
 }

test/fixtures/test_grammars/aliases_in_root/corpus.txt

@@ -0,0 +1,13 @@
+======================================
+Aliases within the root node
+======================================
+
+# this is a comment
+foo foo
+
+---
+
+(document
+  (comment)
+  (bar)
+  (foo))

test/fixtures/test_grammars/aliases_in_root/grammar.js

@@ -0,0 +1,19 @@
+module.exports = grammar({
+    name: 'aliases_in_root',
+
+    extras: $ => [
+      /\s/,
+      $.comment,
+    ],
+
+    rules: {
+        document: $ => seq(
+          alias($.foo, $.bar),
+          $.foo,
+        ),
+
+        foo: $ => "foo",
+
+        comment: $ => /#.*/
+    }
+});

test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A document
+==========================
+
+a b
+
+---
+
+(document)

test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js

@@ -0,0 +1,11 @@
+module.exports = grammar({
+    name: 'epsilon_external_extra_tokens',
+
+    extras: $ => [/\s/, $.comment],
+
+    externals: $ => [$.comment],
+
+    rules: {
+        document: $ => seq('a', 'b'),
+    }
+});

test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c

@@ -0,0 +1,33 @@
+#include "tree_sitter/parser.h"
+
+enum TokenType {
+  COMMENT
+};
+
+void *tree_sitter_epsilon_external_extra_tokens_external_scanner_create(void) {
+  return NULL;
+}
+
+bool tree_sitter_epsilon_external_extra_tokens_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = COMMENT;
+  return true;
+}
+
+unsigned tree_sitter_epsilon_external_extra_tokens_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_epsilon_external_extra_tokens_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_epsilon_external_extra_tokens_external_scanner_destroy(void *payload) {}

xtask/src/bump.rs

@@ -127,6 +127,11 @@ pub fn run(args: BumpVersion) -> Result<()> {
         }
         next_version
     };
+    if next_version <= current_version {
+        return Err(anyhow!(format!(
+            "Next version {next_version} must be greater than current version {current_version}"
+        )));
+    }
 
     println!("Bumping from {current_version} to {next_version}");
     update_crates(&current_version, &next_version)?;

xtask/src/clippy.rs

@@ -6,7 +6,7 @@ use crate::{bail_on_err, Clippy};
 
 pub fn run(args: &Clippy) -> Result<()> {
     let mut clippy_command = Command::new("cargo");
-    clippy_command.arg("+nightly").arg("clippy");
+    clippy_command.arg("clippy");
 
     if let Some(package) = args.package.as_ref() {
         clippy_command.args(["--package", package]);

xtask/src/main.rs

@@ -144,7 +144,7 @@ struct Test {
     iterations: Option<u32>,
     /// Set the seed used to control random behavior.
     #[arg(long, short)]
-    seed: Option<u32>,
+    seed: Option<usize>,
     /// Print parsing log to stderr.
     #[arg(long, short)]
     debug: bool,

xtask/src/test.rs

@@ -65,13 +65,17 @@ pub fn run(args: &Test) -> Result<()> {
     }
 
     if args.g {
-        let cargo_cmd = Command::new("cargo")
+        let mut cargo_cmd = Command::new("cargo");
+        cargo_cmd
             .arg("test")
             .arg(test_flags)
             .arg("--no-run")
-            .arg("--message-format=json")
-            .stdout(Stdio::piped())
-            .spawn()?;
+            .arg("--message-format=json");
+
+        #[cfg(target_os = "windows")]
+        cargo_cmd.arg("--").arg("--test-threads=1");
+
+        let cargo_cmd = cargo_cmd.stdout(Stdio::piped()).spawn()?;
 
         let jq_cmd = Command::new("jq")
             .arg("-rs")
@@ -97,8 +101,15 @@ pub fn run(args: &Test) -> Result<()> {
             cargo_cmd.arg(test_flags);
         }
         cargo_cmd.args(&args.args);
+
+        #[cfg(target_os = "windows")]
+        cargo_cmd.arg("--").arg("--test-threads=1");
+
         if args.nocapture {
-            cargo_cmd.arg("--").arg("--nocapture");
+            #[cfg(not(target_os = "windows"))]
+            cargo_cmd.arg("--");
+
+            cargo_cmd.arg("--nocapture");
         }
         bail_on_err(
             &cargo_cmd.spawn()?.wait_with_output()?,