0.25.4

(cherry picked from commit 31b9717ca3)

.github/workflows/build.yml

@@ -195,6 +195,13 @@ jobs:
         npm run build
         npm run build:debug
 
+    - name: Check no_std builds
+      if: ${{ !matrix.no-run && inputs.run-test }}
+      shell: bash
+      run: |
+        cd lib
+        $BUILD_CMD check --no-default-features
+
     - name: Build target
       run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }}
 

.github/workflows/ci.yml

@@ -32,6 +32,7 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: stable
+          components: clippy, rustfmt
 
       - name: Set up nightly Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -44,6 +45,11 @@ jobs:
           make lint
           make lint-web
 
+      - name: Lint files (Rust nightly)
+        continue-on-error: true
+        run: |
+          make lint-nightly
+
   sanitize:
     uses: ./.github/workflows/sanitize.yml
 

Cargo.toml

@@ -3,6 +3,7 @@ default-members = ["cli"]
 members = [
   "cli",
   "cli/config",
+  "cli/generate",
   "cli/loader",
   "lib",
   "lib/language",
@@ -13,7 +14,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.25.1"
+version = "0.25.4"
 authors = [
   "Max Brunsfeld <maxbrunsfeld@gmail.com>",
   "Amaan Qureshi <amaanq12@gmail.com>",
@@ -74,6 +75,7 @@ too_many_lines = "allow"
 unnecessary_wraps = "allow"
 unused_self = "allow"
 used_underscore_items = "allow"
+obfuscated_if_else = "allow"
 
 [profile.optimize]
 inherits = "release"

Makefile

@@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT)
 $(error Windows is not supported)
 endif
 
-VERSION := 0.25.1
+VERSION := 0.25.4
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -104,6 +104,12 @@ test-wasm:
 	cargo xtask test-wasm
 
 lint:
+	cargo update --workspace --locked --quiet
+	cargo check --workspace --all-targets
+	cargo +nightly fmt --all --check
+	cargo +stable clippy --workspace --all-targets -- -D warnings
+
+lint-nightly:
 	cargo update --workspace --locked --quiet
 	cargo check --workspace --all-targets
 	cargo +nightly fmt --all --check

build.zig.zon

@@ -1,6 +1,6 @@
 .{
   .name = "tree-sitter",
-  .version = "0.25.1",
+  .version = "0.25.4",
   .paths = .{
     "build.zig",
     "build.zig.zon",

cli/benches/benchmark.rs

@@ -112,7 +112,7 @@ fn main() {
 
             parse(path, max_path_length, |source| {
                 Query::new(&language, str::from_utf8(source).unwrap())
-                    .with_context(|| format!("Query file path: {path:?}"))
+                    .with_context(|| format!("Query file path: {}", path.display()))
                     .expect("Failed to parse query");
             });
         }
@@ -201,7 +201,7 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
     );
 
     let source_code = fs::read(path)
-        .with_context(|| format!("Failed to read {path:?}"))
+        .with_context(|| format!("Failed to read {}", path.display()))
         .unwrap();
     let time = Instant::now();
     for _ in 0..*REPETITION_COUNT {
@@ -221,6 +221,6 @@ fn get_language(path: &Path) -> Language {
     let src_path = GRAMMARS_DIR.join(path).join("src");
     TEST_LOADER
         .load_language_at_path(CompileConfig::new(&src_path, None, None))
-        .with_context(|| format!("Failed to load language at path {src_path:?}"))
+        .with_context(|| format!("Failed to load language at path {}", src_path.display()))
         .unwrap()
 }

cli/generate/Cargo.toml

@@ -29,6 +29,8 @@ serde.workspace = true
 serde_json.workspace = true
 smallbitvec.workspace = true
 thiserror.workspace = true
-url.workspace = true
 
 tree-sitter.workspace = true
+
+[target.'cfg(windows)'.dependencies]
+url = { version = "2.5.4", default-features = false }

cli/generate/src/lib.rs

@@ -183,7 +183,8 @@ pub fn generate_parser_in_directory(
     if grammar_path.file_name().unwrap() != "grammar.json" {
         fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| {
             GenerateError::IO(format!(
-                "Failed to write grammar.json to {src_path:?} -- {e}"
+                "Failed to write grammar.json to {} -- {e}",
+                src_path.display()
             ))
         })?;
     }

cli/generate/src/prepare_grammar/extract_tokens.rs

@@ -28,8 +28,8 @@ unless they are used only as the grammar's start rule.
     NonSymbolExternalToken,
     #[error("Non-terminal symbol '{0}' cannot be used as the word token, because its rule is duplicated in '{1}'")]
     NonTerminalWordToken(String, String),
-    #[error("Reserved words must be tokens")]
-    NonTokenReservedWord,
+    #[error("Reserved word '{0}' must be a token")]
+    NonTokenReservedWord(String),
 }
 
 pub(super) fn extract_tokens(
@@ -188,7 +188,12 @@ pub(super) fn extract_tokens(
             {
                 reserved_words.push(Symbol::terminal(index));
             } else {
-                Err(ExtractTokensError::NonTokenReservedWord)?;
+                let token_name = match &reserved_rule {
+                    Rule::String(s) => s.clone(),
+                    Rule::Pattern(p, _) => p.clone(),
+                    _ => "unknown".to_string(),
+                };
+                Err(ExtractTokensError::NonTokenReservedWord(token_name))?;
             }
         }
         reserved_word_contexts.push(ReservedWordContext {

cli/generate/src/render.rs

@@ -328,7 +328,7 @@ impl Generator {
         );
         add_line!(
             self,
-            "/* Automatically generated by tree-sitter v{version} */",
+            "/* Automatically @generated by tree-sitter v{version} */",
         );
         add_line!(self, "");
     }
@@ -1111,7 +1111,11 @@ impl Generator {
             return;
         }
 
-        add_line!(self, "const TSCharacterRange {}[] = {{", info.constant_name);
+        add_line!(
+            self,
+            "static const TSCharacterRange {}[] = {{",
+            info.constant_name
+        );
 
         indent!(self);
         for (ix, range) in characters.ranges().enumerate() {
@@ -1847,11 +1851,11 @@ impl Generator {
                         '\u{007F}' => "DEL",
                         '\u{FEFF}' => "BOM",
                         '\u{0080}'..='\u{FFFF}' => {
-                            result.push_str(&format!("u{:04x}", c as u32));
+                            write!(result, "u{:04x}", c as u32).unwrap();
                             break 'special_chars;
                         }
                         '\u{10000}'..='\u{10FFFF}' => {
-                            result.push_str(&format!("U{:08x}", c as u32));
+                            write!(result, "U{:08x}", c as u32).unwrap();
                             break 'special_chars;
                         }
                         '0'..='9' | 'a'..='z' | 'A'..='Z' | '_' => unreachable!(),
@@ -1882,11 +1886,9 @@ impl Generator {
                 '\r' => result += "\\r",
                 '\t' => result += "\\t",
                 '\0' => result += "\\0",
-                '\u{0001}'..='\u{001f}' => result += &format!("\\x{:02x}", c as u32),
-                '\u{007F}'..='\u{FFFF}' => result += &format!("\\u{:04x}", c as u32),
-                '\u{10000}'..='\u{10FFFF}' => {
-                    result.push_str(&format!("\\U{:08x}", c as u32));
-                }
+                '\u{0001}'..='\u{001f}' => write!(result, "\\x{:02x}", c as u32).unwrap(),
+                '\u{007F}'..='\u{FFFF}' => write!(result, "\\u{:04x}", c as u32).unwrap(),
+                '\u{10000}'..='\u{10FFFF}' => write!(result, "\\U{:08x}", c as u32).unwrap(),
                 _ => result.push(c),
             }
         }

cli/generate/src/rules.rs

@@ -306,7 +306,6 @@ impl Symbol {
 }
 
 impl From<Symbol> for Rule {
-    #[must_use]
     fn from(symbol: Symbol) -> Self {
         Self::Symbol(symbol)
     }

cli/loader/emscripten-version

@@ -1 +1 @@
-4.0.1
+4.0.4

cli/loader/src/lib.rs

@@ -561,8 +561,8 @@ impl Loader {
                 // If multiple language configurations match, then determine which
                 // one to use by applying the configurations' content regexes.
                 else {
-                    let file_contents =
-                        fs::read(path).with_context(|| format!("Failed to read path {path:?}"))?;
+                    let file_contents = fs::read(path)
+                        .with_context(|| format!("Failed to read path {}", path.display()))?;
                     let file_contents = String::from_utf8_lossy(&file_contents);
                     let mut best_score = -2isize;
                     let mut best_configuration_id = None;
@@ -780,8 +780,8 @@ impl Loader {
         if recompile {
             fs::create_dir_all(lock_path.parent().unwrap()).with_context(|| {
                 format!(
-                    "Failed to create directory {:?}",
-                    lock_path.parent().unwrap()
+                    "Failed to create directory {}",
+                    lock_path.parent().unwrap().display()
                 )
             })?;
             let lock_file = fs::OpenOptions::new()
@@ -799,7 +799,7 @@ impl Loader {
         }
 
         let library = unsafe { Library::new(&output_path) }
-            .with_context(|| format!("Error opening dynamic library {output_path:?}"))?;
+            .with_context(|| format!("Error opening dynamic library {}", output_path.display()))?;
         let language = unsafe {
             let language_fn = library
                 .get::<Symbol<unsafe extern "C" fn() -> Language>>(language_fn_name.as_bytes())
@@ -1564,7 +1564,7 @@ impl LanguageConfiguration<'_> {
         error.row = source[range.start..offset_within_section]
             .matches('\n')
             .count();
-        Error::from(error).context(format!("Error in query file {path:?}"))
+        Error::from(error).context(format!("Error in query file {}", path.display()))
     }
 
     #[allow(clippy::type_complexity)]
@@ -1581,7 +1581,7 @@ impl LanguageConfiguration<'_> {
                 let abs_path = self.root_path.join(path);
                 let prev_query_len = query.len();
                 query += &fs::read_to_string(&abs_path)
-                    .with_context(|| format!("Failed to read query file {path:?}"))?;
+                    .with_context(|| format!("Failed to read query file {}", path.display()))?;
                 path_ranges.push((path.clone(), prev_query_len..query.len()));
             }
         } else {
@@ -1599,7 +1599,7 @@ impl LanguageConfiguration<'_> {
             let path = queries_path.join(default_path);
             if path.exists() {
                 query = fs::read_to_string(&path)
-                    .with_context(|| format!("Failed to read query file {path:?}"))?;
+                    .with_context(|| format!("Failed to read query file {}", path.display()))?;
                 path_ranges.push((PathBuf::from(default_path), 0..query.len()));
             }
         }
@@ -1612,8 +1612,8 @@ fn needs_recompile(lib_path: &Path, paths_to_check: &[PathBuf]) -> Result<bool>
     if !lib_path.exists() {
         return Ok(true);
     }
-    let lib_mtime =
-        mtime(lib_path).with_context(|| format!("Failed to read mtime of {lib_path:?}"))?;
+    let lib_mtime = mtime(lib_path)
+        .with_context(|| format!("Failed to read mtime of {}", lib_path.display()))?;
     for path in paths_to_check {
         if mtime(path)? > lib_mtime {
             return Ok(true);

cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.25.1",
+  "version": "0.25.4",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

cli/src/fuzz/allocations.rs

@@ -109,7 +109,7 @@ unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_
     let result = realloc(ptr, size);
     if ptr.is_null() {
         record_alloc(result);
-    } else if ptr != result {
+    } else if !core::ptr::eq(ptr, result) {
         record_dealloc(ptr);
         record_alloc(result);
     }

cli/src/fuzz/mod.rs

@@ -56,7 +56,9 @@ fn regex_env_var(name: &'static str) -> Option<Regex> {
 pub fn new_seed() -> usize {
     int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
         let mut rng = rand::thread_rng();
-        rng.gen::<usize>()
+        let seed = rng.gen::<usize>();
+        eprintln!("Seed: {seed}");
+        seed
     })
 }
 

cli/src/highlight.rs

@@ -1,5 +1,5 @@
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeMap, HashSet},
     fmt::Write,
     fs,
     io::{self, Write as _},
@@ -82,7 +82,7 @@ impl<'de> Deserialize<'de> for Theme {
     {
         let mut styles = Vec::new();
         let mut highlight_names = Vec::new();
-        if let Ok(colors) = HashMap::<String, Value>::deserialize(deserializer) {
+        if let Ok(colors) = BTreeMap::<String, Value>::deserialize(deserializer) {
             highlight_names.reserve(colors.len());
             styles.reserve(colors.len());
             for (name, style_value) in colors {
@@ -127,7 +127,7 @@ impl Serialize for Theme {
                 || effects.contains(Effects::ITALIC)
                 || effects.contains(Effects::UNDERLINE)
             {
-                let mut style_json = HashMap::new();
+                let mut style_json = BTreeMap::new();
                 if let Some(color) = color {
                     style_json.insert("color", color);
                 }

cli/src/input.rs

@@ -89,8 +89,8 @@ pub fn get_input(
                 let Some(path_str) = path.to_str() else {
                     bail!("Invalid path: {}", path.display());
                 };
-                let paths =
-                    glob(path_str).with_context(|| format!("Invalid glob pattern {path:?}"))?;
+                let paths = glob(path_str)
+                    .with_context(|| format!("Invalid glob pattern {}", path.display()))?;
                 for path in paths {
                     incorporate_path(path?, positive);
                 }

cli/src/main.rs

@@ -206,7 +206,8 @@ struct Parse {
     #[arg(long, short)]
     pub quiet: bool,
     #[allow(clippy::doc_markdown)]
-    /// Apply edits in the format: \"row, col delcount insert_text\"
+    /// Apply edits in the format: \"row,col|position delcount insert_text\", can be supplied
+    /// multiple times
     #[arg(
         long,
         num_args = 1..,

cli/src/parse.rs

@@ -29,18 +29,28 @@ pub struct Stats {
 impl fmt::Display for Stats {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let duration_us = self.total_duration.as_micros();
+        let success_rate = if self.total_parses > 0 {
+            format!(
+                "{:.2}%",
+                ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0,
+            )
+        } else {
+            "N/A".to_string()
+        };
+        let duration_str = match (self.total_parses, duration_us) {
+            (0, _) => "N/A".to_string(),
+            (_, 0) => "0 bytes/ms".to_string(),
+            (_, _) => format!(
+                "{} bytes/ms",
+                ((self.total_bytes as u128) * 1_000) / duration_us
+            ),
+        };
         writeln!(
             f,
-            "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {:.2}%; average speed: {} bytes/ms",
+            "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {success_rate}; average speed: {duration_str}",
             self.total_parses,
             self.successful_parses,
             self.total_parses - self.successful_parses,
-            ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0,
-            if duration_us != 0 {
-                ((self.total_bytes as u128) * 1_000) / duration_us
-            } else {
-                0
-            }
         )
     }
 }
@@ -225,7 +235,7 @@ pub struct ParseStats {
     pub cumulative_stats: Stats,
 }
 
-#[derive(Serialize, ValueEnum, Debug, Clone, Default, Eq, PartialEq)]
+#[derive(Serialize, ValueEnum, Debug, Copy, Clone, Default, Eq, PartialEq)]
 pub enum ParseDebugType {
     #[default]
     Quiet,
@@ -273,10 +283,11 @@ pub fn parse_file_at_path(
     }
     // Log to stderr if `--debug` was passed
     else if opts.debug != ParseDebugType::Quiet {
-        let mut curr_version: usize = 0usize;
+        let mut curr_version: usize = 0;
         let use_color = std::env::var("NO_COLOR").map_or(true, |v| v != "1");
-        parser.set_logger(Some(Box::new(|log_type, message| {
-            if opts.debug == ParseDebugType::Normal {
+        let debug = opts.debug;
+        parser.set_logger(Some(Box::new(move |log_type, message| {
+            if debug == ParseDebugType::Normal {
                 if log_type == LogType::Lex {
                     write!(&mut io::stderr(), "  ").unwrap();
                 }
@@ -686,19 +697,23 @@ pub fn parse_file_at_path(
             if let Some(node) = first_error {
                 let start = node.start_position();
                 let end = node.end_position();
+                let mut node_text = String::new();
+                for c in node.kind().chars() {
+                    if let Some(escaped) = escape_invisible(c) {
+                        node_text += escaped;
+                    } else {
+                        node_text.push(c);
+                    }
+                }
                 write!(&mut stdout, "\t(")?;
                 if node.is_missing() {
                     if node.is_named() {
-                        write!(&mut stdout, "MISSING {}", node.kind())?;
+                        write!(&mut stdout, "MISSING {node_text}")?;
                     } else {
-                        write!(
-                            &mut stdout,
-                            "MISSING \"{}\"",
-                            node.kind().replace('\n', "\\n")
-                        )?;
+                        write!(&mut stdout, "MISSING \"{node_text}\"")?;
                     }
                 } else {
-                    write!(&mut stdout, "{}", node.kind())?;
+                    write!(&mut stdout, "{node_text}")?;
                 }
                 write!(
                     &mut stdout,

cli/src/query.rs

@@ -34,7 +34,7 @@ pub fn query_file_at_path(
     let mut stdout = stdout.lock();
 
     let query_source = fs::read_to_string(query_path)
-        .with_context(|| format!("Error reading query file {query_path:?}"))?;
+        .with_context(|| format!("Error reading query file {}", query_path.display()))?;
     let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?;
 
     let mut query_cursor = QueryCursor::new();
@@ -55,7 +55,7 @@ pub fn query_file_at_path(
     }
 
     let source_code =
-        fs::read(path).with_context(|| format!("Error reading source file {path:?}"))?;
+        fs::read(path).with_context(|| format!("Error reading source file {}", path.display()))?;
     let tree = parser.parse(&source_code, None).unwrap();
 
     let start = Instant::now();

cli/src/templates/lib.rs

@@ -1,7 +1,7 @@
-//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library.
+//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter] parsing library.
 //!
-//! Typically, you will use the [LANGUAGE][] constant to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
+//! Typically, you will use the [`LANGUAGE`] constant to add this language to a
+//! tree-sitter [`Parser`], and then use the parser to parse some code:
 //!
 //! ```
 //! let code = r#"
@@ -15,7 +15,7 @@
 //! assert!(!tree.root_node().has_error());
 //! ```
 //!
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [`Parser`]: https://docs.rs/tree-sitter/RUST_BINDING_VERSION/tree_sitter/struct.Parser.html
 //! [tree-sitter]: https://tree-sitter.github.io/
 
 use tree_sitter_language::LanguageFn;
@@ -24,12 +24,10 @@ extern "C" {
     fn tree_sitter_PARSER_NAME() -> *const ();
 }
 
-/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar.
-///
-/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
+/// The tree-sitter [`LanguageFn`] for this grammar.
 pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) };
 
-/// The content of the [`node-types.json`][] file for this grammar.
+/// The content of the [`node-types.json`] file for this grammar.
 ///
 /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
 pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");

cli/src/templates/makefile

@@ -77,7 +77,9 @@ install: all
 	install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER)
 	ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR)
 	ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT)
+ifneq ($(wildcard queries/*.scm),)
 	install -m644 queries/*.scm '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME
+endif
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \

cli/src/test_highlight.rs

@@ -172,7 +172,7 @@ pub fn iterate_assertions(
             let mut j = i;
             while let (false, Some(highlight)) = (passed, highlights.get(j)) {
                 end_column = position.column + length - 1;
-                if highlight.0.column > end_column {
+                if highlight.0.row >= position.row && highlight.0.column > end_column {
                     break 'highlight_loop;
                 }
 

cli/src/tests/async_context_test.rs

@@ -238,7 +238,7 @@ async fn yield_now() {
     SimpleYieldNow { yielded: false }.await;
 }
 
-pub fn noop_waker() -> Waker {
+pub const fn noop_waker() -> Waker {
     const VTABLE: RawWakerVTable = RawWakerVTable::new(
         // Cloning just returns a new no-op raw waker
         |_| RAW,

cli/src/tests/corpus_test.rs

@@ -23,7 +23,7 @@ use crate::{
 };
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_bash(seed: usize) {
+fn test_corpus_for_bash_language(seed: usize) {
     test_language_corpus(
         "bash",
         seed,
@@ -39,73 +39,77 @@ fn test_corpus_for_bash(seed: usize) {
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_c(seed: usize) {
+fn test_corpus_for_c_language(seed: usize) {
     test_language_corpus("c", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_cpp(seed: usize) {
+fn test_corpus_for_cpp_language(seed: usize) {
     test_language_corpus("cpp", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_embedded_template(seed: usize) {
+fn test_corpus_for_embedded_template_language(seed: usize) {
     test_language_corpus("embedded-template", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_go(seed: usize) {
+fn test_corpus_for_go_language(seed: usize) {
     test_language_corpus("go", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_html(seed: usize) {
+fn test_corpus_for_html_language(seed: usize) {
     test_language_corpus("html", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_java(seed: usize) {
-    test_language_corpus("java", seed, None, None);
+fn test_corpus_for_java_language(seed: usize) {
+    test_language_corpus(
+        "java",
+        seed,
+        Some(&["java - corpus - expressions - switch with unnamed pattern variable"]),
+        None,
+    );
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_javascript(seed: usize) {
+fn test_corpus_for_javascript_language(seed: usize) {
     test_language_corpus("javascript", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_json(seed: usize) {
+fn test_corpus_for_json_language(seed: usize) {
     test_language_corpus("json", seed, None, None);
 }
 
-#[ignore]
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_php(seed: usize) {
-    test_language_corpus("php", seed, None, None);
+fn test_corpus_for_php_language(seed: usize) {
+    test_language_corpus("php", seed, None, Some("php"));
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_python(seed: usize) {
+fn test_corpus_for_python_language(seed: usize) {
     test_language_corpus("python", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_ruby(seed: usize) {
+fn test_corpus_for_ruby_language(seed: usize) {
     test_language_corpus("ruby", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_rust(seed: usize) {
+fn test_corpus_for_rust_language(seed: usize) {
     test_language_corpus("rust", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_typescript(seed: usize) {
+fn test_corpus_for_typescript_language(seed: usize) {
     test_language_corpus("typescript", seed, None, Some("typescript"));
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_tsx(seed: usize) {
+fn test_corpus_for_tsx_language(seed: usize) {
     test_language_corpus("typescript", seed, None, Some("tsx"));
 }
 

cli/src/tests/helpers/allocations.rs

@@ -108,7 +108,7 @@ unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_
     let result = realloc(ptr, size);
     if ptr.is_null() {
         record_alloc(result);
-    } else if ptr != result {
+    } else if !core::ptr::eq(ptr, result) {
         record_dealloc(ptr);
         record_alloc(result);
     }

cli/src/tests/language_test.rs

@@ -152,6 +152,7 @@ fn test_supertypes() {
                         "_literal_pattern",
                         "captured_pattern",
                         "const_block",
+                        "generic_pattern",
                         "identifier",
                         "macro_invocation",
                         "mut_pattern",

cli/src/tests/query_test.rs

@@ -5621,3 +5621,14 @@ const foo = [
     assert_eq!(matches.len(), 1);
     assert_eq!(matches[0].1, captures);
 }
+
+#[test]
+fn test_query_with_predicate_causing_oob_access() {
+    let language = get_language("rust");
+
+    let query = "(call_expression
+     function: (scoped_identifier
+       path: (scoped_identifier (identifier) @_regex (#any-of? @_regex \"Regex\" \"RegexBuilder\") .))
+     (#set! injection.language \"regex\"))";
+    Query::new(&language, query).unwrap();
+}

cli/src/tests/tags_test.rs

@@ -401,8 +401,11 @@ fn test_tags_via_c_api() {
 
         let syntax_types = unsafe {
             let mut len = 0;
-            let ptr =
-                c::ts_tagger_syntax_kinds_for_scope_name(tagger, c_scope_name.as_ptr(), &mut len);
+            let ptr = c::ts_tagger_syntax_kinds_for_scope_name(
+                tagger,
+                c_scope_name.as_ptr(),
+                &raw mut len,
+            );
             slice::from_raw_parts(ptr, len as usize)
                 .iter()
                 .map(|i| CStr::from_ptr(*i).to_str().unwrap())

cli/src/wasm.rs

@@ -23,10 +23,18 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)>
 pub fn get_grammar_name(language_dir: &Path) -> Result<String> {
     let src_dir = language_dir.join("src");
     let grammar_json_path = src_dir.join("grammar.json");
-    let grammar_json = fs::read_to_string(&grammar_json_path)
-        .with_context(|| format!("Failed to read grammar file {grammar_json_path:?}"))?;
-    let grammar: GrammarJSON = serde_json::from_str(&grammar_json)
-        .with_context(|| format!("Failed to parse grammar file {grammar_json_path:?}"))?;
+    let grammar_json = fs::read_to_string(&grammar_json_path).with_context(|| {
+        format!(
+            "Failed to read grammar file {}",
+            grammar_json_path.display()
+        )
+    })?;
+    let grammar: GrammarJSON = serde_json::from_str(&grammar_json).with_context(|| {
+        format!(
+            "Failed to parse grammar file {}",
+            grammar_json_path.display()
+        )
+    })?;
     Ok(grammar.name)
 }
 

docs/src/cli/parse.md

@@ -66,7 +66,7 @@ Suppress main output.
 
 ### `--edits <EDITS>...`
 
-Apply edits after parsing the file. Edits are in the form of `row, col delcount insert_text` where row and col are 0-indexed.
+Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, or position are 0-indexed.
 
 ### `--encoding <ENCODING>`
 

lib/CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.25.1"
+        VERSION "0.25.4"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)

lib/binding_rust/build.rs

@@ -112,7 +112,10 @@ fn generate_bindings(out_dir: &std::path::Path) {
         .expect("Failed to generate bindings");
 
     let bindings_rs = out_dir.join("bindings.rs");
-    bindings
-        .write_to_file(&bindings_rs)
-        .unwrap_or_else(|_| panic!("Failed to write bindings into path: {bindings_rs:?}"));
+    bindings.write_to_file(&bindings_rs).unwrap_or_else(|_| {
+        panic!(
+            "Failed to write bindings into path: {}",
+            bindings_rs.display()
+        )
+    });
 }

lib/binding_rust/ffi.rs

@@ -16,6 +16,7 @@ extern "C" {
 }
 
 #[cfg(windows)]
+#[cfg(feature = "std")]
 extern "C" {
     pub(crate) fn _ts_dup(handle: *mut std::os::raw::c_void) -> std::os::raw::c_int;
 }

lib/binding_rust/lib.rs

@@ -1222,7 +1222,7 @@ impl Parser {
             len: u32,
             code_point: *mut i32,
         ) -> u32 {
-            let (c, len) = D::decode(std::slice::from_raw_parts(data, len as usize));
+            let (c, len) = D::decode(core::slice::from_raw_parts(data, len as usize));
             if let Some(code_point) = code_point.as_mut() {
                 *code_point = c;
             }
@@ -1422,7 +1422,7 @@ impl Parser {
         if let Some(flag) = flag {
             ffi::ts_parser_set_cancellation_flag(
                 self.0.as_ptr(),
-                std::ptr::from_ref::<AtomicUsize>(flag).cast::<usize>(),
+                core::ptr::from_ref::<AtomicUsize>(flag).cast::<usize>(),
             );
         } else {
             ffi::ts_parser_set_cancellation_flag(self.0.as_ptr(), ptr::null());
@@ -1432,7 +1432,11 @@ impl Parser {
 
 impl Drop for Parser {
     fn drop(&mut self) {
-        self.stop_printing_dot_graphs();
+        #[cfg(feature = "std")]
+        #[cfg(not(target_os = "wasi"))]
+        {
+            self.stop_printing_dot_graphs();
+        }
         self.set_logger(None);
         unsafe { ffi::ts_parser_delete(self.0.as_ptr()) }
     }
@@ -2087,7 +2091,7 @@ impl<'tree> Node<'tree> {
 
 impl PartialEq for Node<'_> {
     fn eq(&self, other: &Self) -> bool {
-        self.0.id == other.0.id
+        core::ptr::eq(self.0.id, other.0.id)
     }
 }
 

lib/binding_web/lib/tree-sitter.c

@@ -44,17 +44,22 @@ static inline void marshal_node(const void **buffer, TSNode node) {
   buffer[4] = (const void *)node.context[3];
 }
 
-static inline TSNode unmarshal_node(const TSTree *tree) {
+static inline TSNode unmarshal_node_at(const TSTree *tree, uint32_t index) {
   TSNode node;
-  node.id = TRANSFER_BUFFER[0];
-  node.context[0] = code_unit_to_byte((uint32_t)TRANSFER_BUFFER[1]);
-  node.context[1] = (uint32_t)TRANSFER_BUFFER[2];
-  node.context[2] = code_unit_to_byte((uint32_t)TRANSFER_BUFFER[3]);
-  node.context[3] = (uint32_t)TRANSFER_BUFFER[4];
+  const void **buffer = TRANSFER_BUFFER + index * SIZE_OF_NODE;
+  node.id = buffer[0];
+  node.context[0] = code_unit_to_byte((uint32_t)buffer[1]);
+  node.context[1] = (uint32_t)buffer[2];
+  node.context[2] = code_unit_to_byte((uint32_t)buffer[3]);
+  node.context[3] = (uint32_t)buffer[4];
   node.tree = tree;
   return node;
 }
 
+static inline TSNode unmarshal_node(const TSTree *tree) {
+  return unmarshal_node_at(tree, 0);
+}
+
 static inline void marshal_cursor(const TSTreeCursor *cursor) {
   TRANSFER_BUFFER[0] = cursor->id;
   TRANSFER_BUFFER[1] = (const void *)cursor->context[0];
@@ -616,7 +621,7 @@ void ts_node_parent_wasm(const TSTree *tree) {
 
 void ts_node_child_with_descendant_wasm(const TSTree *tree) {
   TSNode node = unmarshal_node(tree);
-  TSNode descendant = unmarshal_node(tree);
+  TSNode descendant = unmarshal_node_at(tree, 1);
   marshal_node(TRANSFER_BUFFER, ts_node_child_with_descendant(node, descendant));
 }
 

lib/binding_web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.25.0",
+  "version": "0.25.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "web-tree-sitter",
-      "version": "0.25.0",
+      "version": "0.25.1",
       "license": "MIT",
       "devDependencies": {
         "@eslint/js": "^9.19.0",

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.25.1",
+  "version": "0.25.4",
   "description": "Tree-sitter bindings for the web",
   "repository": "https://github.com/tree-sitter/tree-sitter",
   "homepage": "https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web",
@@ -19,12 +19,16 @@
   "exports": {
     ".": {
       "import": "./tree-sitter.js",
-      "require": "./tree-sitter.cjs"
+      "require": "./tree-sitter.cjs",
+      "types": "./web-tree-sitter.d.ts"
     },
+    "./web-tree-sitter.wasm": "./web-tree-sitter.wasm",
     "./debug": {
       "import": "./debug/tree-sitter.js",
-      "require": "./debug/tree-sitter.cjs"
-    }
+      "require": "./debug/tree-sitter.cjs",
+      "types": "./web-tree-sitter.d.ts"
+    },
+    "./debug/web-tree-sitter.wasm": "./debug/web-tree-sitter.wasm"
   },
   "types": "web-tree-sitter.d.ts",
   "keywords": [

lib/binding_web/src/language.ts

@@ -261,8 +261,7 @@ export class Language {
     } else {
       // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
       if (globalThis.process?.versions.node) {
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports
-        const fs: typeof import('fs/promises') = require('fs/promises');
+        const fs: typeof import('fs/promises') = await import('fs/promises');
         bytes = fs.readFile(input);
       } else {
         bytes = fetch(input)

lib/binding_web/src/marshal.ts

@@ -34,8 +34,8 @@ export function unmarshalCaptures(
  *
  * Marshals a {@link Node} to the transfer buffer.
  */
-export function marshalNode(node: Node) {
-  let address = TRANSFER_BUFFER;
+export function marshalNode(node: Node, index = 0) {
+  let address = TRANSFER_BUFFER + index * SIZE_OF_NODE;
   C.setValue(address, node.id, 'i32');
   address += SIZE_OF_INT;
   C.setValue(address, node.startIndex, 'i32');

lib/binding_web/src/node.ts

@@ -416,6 +416,11 @@ export class Node {
     // Convert the type strings to numeric type symbols
     const symbols: number[] = [];
     const typesBySymbol = this.tree.language.types;
+    for (const node_type of types) {
+      if (node_type == "ERROR") {
+        symbols.push(65535); // Internally, ts_builtin_sym_error is -1, which is UINT_16MAX
+      }
+    }
     for (let i = 0, n = typesBySymbol.length; i < n; i++) {
       if (types.includes(typesBySymbol[i])) {
         symbols.push(i);
@@ -517,7 +522,7 @@ export class Node {
    */
   childWithDescendant(descendant: Node): Node | null {
     marshalNode(this);
-    marshalNode(descendant);
+    marshalNode(descendant, 1);
     C._ts_node_child_with_descendant_wasm(this.tree[0]);
     return unmarshalNode(this.tree);
   }

lib/binding_web/test/language.test.ts

@@ -89,6 +89,7 @@ describe('Language', () => {
               '_literal_pattern',
               'captured_pattern',
               'const_block',
+              'generic_pattern',
               'identifier',
               'macro_invocation',
               'mut_pattern',

lib/binding_web/test/node.test.ts

@@ -63,7 +63,7 @@ describe('Node', () => {
       tree = parser.parse('x10 + 1000')!;
       expect(tree.rootNode.children).toHaveLength(1);
       const sumNode = tree.rootNode.firstChild!.firstChild!;
-      expect(sumNode.children.map(child => child!.type)).toEqual(['identifier', '+', 'number' ]);
+      expect(sumNode.children.map(child => child!.type)).toEqual(['identifier', '+', 'number']);
     });
   });
 
@@ -189,6 +189,21 @@ describe('Node', () => {
     });
   });
 
+  describe('.childWithDescendant()', () => {
+    it('correctly retrieves immediate children', () => {
+      const sourceCode = 'let x = 1; console.log(x);';
+      tree = parser.parse(sourceCode)!;
+      const root = tree.rootNode
+      const child = root.children[0].children[0]
+      const a = root.childWithDescendant(child)
+      expect(a!.startIndex).toBe(0)
+      const b = a!.childWithDescendant(child)
+      expect(b).toEqual(child)
+      const c = b!.childWithDescendant(child)
+      expect(c).toBeNull()
+    });
+  });
+
   describe('.nextSibling and .previousSibling', () => {
     it('returns the node\'s next and previous sibling', () => {
       tree = parser.parse('x10 + 1000')!;
@@ -449,6 +464,24 @@ describe('Node', () => {
     });
   });
 
+  describe('.descendantsOfType("ERROR")', () => {
+    it('finds all of the descendants of an ERROR node', () => {
+      tree = parser.parse(
+        `if ({a: 'b'} {c: 'd'}) {
+          // ^ ERROR
+          x = function(a) { b; } function(c) { d; }
+        }`
+      )!;
+      const errorNode = tree.rootNode;
+      const descendants = errorNode.descendantsOfType('ERROR');
+      expect(
+        descendants.map((node) => node!.startIndex)
+      ).toEqual(
+        [4]
+      );
+    });
+  });
+
   describe('.descendantsOfType', () => {
     it('finds all descendants of a given type in the given range', () => {
       tree = parser.parse('a + 1 * b * 2 + c + 3')!;

lib/binding_web/test/parser.test.ts

@@ -256,7 +256,7 @@ describe('Parser', () => {
       expect(() => parser.parse({})).toThrow('Argument must be a string or a function');
     });
 
-    it('handles long input strings', { timeout: 5000 }, () => {
+    it('handles long input strings', { timeout: 10000 }, () => {
       const repeatCount = 10000;
       const inputString = `[${Array(repeatCount).fill('0').join(',')}]`;
 

lib/binding_web/test/query.test.ts

@@ -64,7 +64,7 @@ describe('Query', () => {
   });
 
   describe('.matches', () => {
-    it('returns all of the matches for the given query', () => {
+    it('returns all of the matches for the given query', { timeout: 10000 }, () => {
       tree = parser.parse('function one() { two(); function three() {} }')!;
       query = new Query(JavaScript, `
         (function_declaration name: (identifier) @fn-def)
@@ -462,7 +462,7 @@ describe('Query', () => {
   });
 
   describe('Set a timeout', () => {
-    it('returns less than the expected matches', () => {
+    it('returns less than the expected matches', { timeout: 10000 }, () => {
       tree = parser.parse('function foo() while (true) { } }\n'.repeat(1000))!;
       query = new Query(JavaScript, '(function_declaration name: (identifier) @function)');
       const matches = query.matches(tree.rootNode, { timeoutMicros: 1000 });
@@ -538,7 +538,7 @@ describe('Query', () => {
     });
   });
 
-  describe('Executes with a timeout', () => {
+  describe('Executes with a timeout', { timeout: 10000 }, () => {
     it('Returns less than the expected matches', () => {
       tree = parser.parse('function foo() while (true) { } }\n'.repeat(1000))!;
       query = new Query(JavaScript, '(function_declaration) @function');

lib/include/tree_sitter/api.h

@@ -42,7 +42,6 @@ typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
-typedef struct TSLanguageMetadata TSLanguageMetadata;
 typedef struct TSParser TSParser;
 typedef struct TSTree TSTree;
 typedef struct TSQuery TSQuery;

lib/language/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "tree-sitter-language"
 description = "The tree-sitter Language type, used by the library and by language implementations"
-version = "0.1.4"
+version = "0.1.5"
 authors.workspace = true
 edition.workspace = true
-rust-version.workspace = true
+rust-version = "1.76"
 readme = "README.md"
 homepage.workspace = true
 repository.workspace = true

lib/src/get_changed_ranges.c

@@ -108,6 +108,7 @@ typedef struct {
   const TSLanguage *language;
   unsigned visible_depth;
   bool in_padding;
+  Subtree prev_external_token;
 } Iterator;
 
 static Iterator iterator_new(
@@ -127,6 +128,7 @@ static Iterator iterator_new(
     .language = language,
     .visible_depth = 1,
     .in_padding = false,
+    .prev_external_token = NULL_SUBTREE,
   };
 }
 
@@ -244,6 +246,10 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) {
 
       position = child_right;
       if (!ts_subtree_extra(*child)) structural_child_index++;
+      Subtree last_external_token = ts_subtree_last_external_token(*child);
+      if (last_external_token.ptr) {
+        self->prev_external_token = last_external_token;
+      }
     }
   } while (did_descend);
 
@@ -268,6 +274,10 @@ static void iterator_advance(Iterator *self) {
 
     const Subtree *parent = array_back(&self->cursor.stack)->subtree;
     uint32_t child_index = entry.child_index + 1;
+    Subtree last_external_token = ts_subtree_last_external_token(*entry.subtree);
+    if (last_external_token.ptr) {
+      self->prev_external_token = last_external_token;
+    }
     if (ts_subtree_child_count(*parent) > child_index) {
       Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree));
       uint32_t structural_child_index = entry.structural_child_index;
@@ -313,29 +323,41 @@ static IteratorComparison iterator_compare(
   TSSymbol new_alias_symbol = 0;
   iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
   iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);
+  TSSymbol old_symbol = ts_subtree_symbol(old_tree);
+  TSSymbol new_symbol = ts_subtree_symbol(new_tree);
 
   if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches;
   if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers;
+  if (old_alias_symbol != new_alias_symbol || old_symbol != new_symbol) return IteratorDiffers;
+
+  uint32_t old_size = ts_subtree_size(old_tree).bytes;
+  uint32_t new_size = ts_subtree_size(new_tree).bytes;
+  TSStateId old_state = ts_subtree_parse_state(old_tree);
+  TSStateId new_state = ts_subtree_parse_state(new_tree);
+  bool old_has_external_tokens = ts_subtree_has_external_tokens(old_tree);
+  bool new_has_external_tokens = ts_subtree_has_external_tokens(new_tree);
+  uint32_t old_error_cost = ts_subtree_error_cost(old_tree);
+  uint32_t new_error_cost = ts_subtree_error_cost(new_tree);
 
   if (
-    old_alias_symbol == new_alias_symbol &&
-    ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree)
+    old_start != new_start ||
+    old_symbol == ts_builtin_sym_error ||
+    old_size != new_size ||
+    old_state == TS_TREE_STATE_NONE ||
+    new_state == TS_TREE_STATE_NONE ||
+    ((old_state == ERROR_STATE) != (new_state == ERROR_STATE)) ||
+    old_error_cost != new_error_cost ||
+    old_has_external_tokens != new_has_external_tokens ||
+    ts_subtree_has_changes(old_tree) ||
+    (
+      old_has_external_tokens &&
+      !ts_subtree_external_scanner_state_eq(old_iter->prev_external_token, new_iter->prev_external_token)
+    )
   ) {
-    if (old_start == new_start &&
-        !ts_subtree_has_changes(old_tree) &&
-        ts_subtree_symbol(old_tree) != ts_builtin_sym_error &&
-        ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes &&
-        ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE &&
-        ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE &&
-        (ts_subtree_parse_state(old_tree) == ERROR_STATE) ==
-        (ts_subtree_parse_state(new_tree) == ERROR_STATE)) {
-      return IteratorMatches;
-    } else {
-      return IteratorMayDiffer;
-    }
+    return IteratorMayDiffer;
   }
 
-  return IteratorDiffers;
+  return IteratorMatches;
 }
 
 #ifdef DEBUG_GET_CHANGED_RANGES
@@ -348,8 +370,8 @@ static inline void iterator_print_state(Iterator *self) {
     "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])",
     name, self->in_padding ? "(p)" : "   ",
     self->visible_depth,
-    start.row + 1, start.column,
-    end.row + 1, end.column
+    start.row, start.column,
+    end.row, end.column
   );
 }
 #endif
@@ -380,7 +402,7 @@ unsigned ts_subtree_get_changed_ranges(
 
   do {
     #ifdef DEBUG_GET_CHANGED_RANGES
-    printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column);
+    printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column);
     iterator_print_state(&old_iter);
     printf("\tvs\t");
     iterator_print_state(&new_iter);

lib/src/parser.c

@@ -556,27 +556,29 @@ static Subtree ts_parser__lex(
           external_scanner_state_len
         );
 
-        // When recovering from an error, ignore any zero-length external tokens
-        // unless they have changed the external scanner's state. This helps to
-        // avoid infinite loops which could otherwise occur, because the lexer is
-        // looking for any possible token, instead of looking for the specific set of
-        // tokens that are valid in some parse state.
+        // Avoid infinite loops caused by the external scanner returning empty tokens.
+        // Empty tokens are needed in some circumstances, e.g. indent/dedent tokens
+        // in Python. Ignore the following classes of empty tokens:
         //
-        // Note that it's possible that the token end position may be *before* the
-        // original position of the lexer because of the way that tokens are positioned
-        // at included range boundaries: when a token is terminated at the start of
-        // an included range, it is marked as ending at the *end* of the preceding
-        // included range.
+        // * Tokens produced during error recovery. When recovering from an error,
+        //   all tokens are allowed, so it's easy to accidentally return unwanted
+        //   empty tokens.
+        // * Tokens that are marked as 'extra' in the grammar. These don't change
+        //   the parse state, so they would definitely cause an infinite loop.
         if (
           self->lexer.token_end_position.bytes <= current_position.bytes &&
-          (error_mode || !ts_stack_has_advanced_since_error(self->stack, version)) &&
           !external_scanner_state_changed
         ) {
-          LOG(
-            "ignore_empty_external_token symbol:%s",
-            SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol])
-          )
-          found_token = false;
+          TSSymbol symbol = self->language->external_scanner.symbol_map[self->lexer.data.result_symbol];
+          TSStateId next_parse_state = ts_language_next_state(self->language, parse_state, symbol);
+          bool token_is_extra = (next_parse_state == parse_state);
+          if (error_mode || !ts_stack_has_advanced_since_error(self->stack, version) || token_is_extra) {
+            LOG(
+              "ignore_empty_external_token symbol:%s",
+              SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol])
+            );
+            found_token = false;
+          }
         }
       }
 
@@ -947,6 +949,7 @@ static StackVersion ts_parser__reduce(
   // children.
   StackSliceArray pop = ts_stack_pop_count(self->stack, version, count);
   uint32_t removed_version_count = 0;
+  uint32_t halted_version_count = ts_stack_halted_version_count(self->stack);
   for (uint32_t i = 0; i < pop.size; i++) {
     StackSlice slice = pop.contents[i];
     StackVersion slice_version = slice.version - removed_version_count;
@@ -955,11 +958,12 @@ static StackVersion ts_parser__reduce(
     // will all be sorted and truncated at the end of the outer parsing loop.
     // Allow the maximum version count to be temporarily exceeded, but only
     // by a limited threshold.
-    if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW) {
+    if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW + halted_version_count) {
       ts_stack_remove_version(self->stack, slice_version);
       ts_subtree_array_delete(&self->tree_pool, &slice.subtrees);
       removed_version_count++;
       while (i + 1 < pop.size) {
+        LOG("aborting reduce with too many versions")
         StackSlice next_slice = pop.contents[i + 1];
         if (next_slice.version != slice.version) break;
         ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees);
@@ -1316,10 +1320,23 @@ static void ts_parser__recover(
   // and subsequently halted. Remove those versions.
   for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) {
     if (!ts_stack_is_active(self->stack, i)) {
+      LOG("removed paused version:%u", i);
       ts_stack_remove_version(self->stack, i--);
+      LOG_STACK();
     }
   }
 
+  // If the parser is still in the error state at the end of the file, just wrap everything
+  // in an ERROR node and terminate.
+  if (ts_subtree_is_eof(lookahead)) {
+    LOG("recover_eof");
+    SubtreeArray children = array_new();
+    Subtree parent = ts_subtree_new_error_node(&children, false, self->language);
+    ts_stack_push(self->stack, version, parent, false, 1);
+    ts_parser__accept(self, version, lookahead);
+    return;
+  }
+
   // If strategy 1 succeeded, a new stack version will have been created which is able to handle
   // the current lookahead token. Now, in addition, try strategy 2 described above: skip the
   // current lookahead token by wrapping it in an ERROR node.
@@ -1340,17 +1357,6 @@ static void ts_parser__recover(
     return;
   }
 
-  // If the parser is still in the error state at the end of the file, just wrap everything
-  // in an ERROR node and terminate.
-  if (ts_subtree_is_eof(lookahead)) {
-    LOG("recover_eof");
-    SubtreeArray children = array_new();
-    Subtree parent = ts_subtree_new_error_node(&children, false, self->language);
-    ts_stack_push(self->stack, version, parent, false, 1);
-    ts_parser__accept(self, version, lookahead);
-    return;
-  }
-
   // Do not recover if the result would clearly be worse than some existing stack version.
   unsigned new_cost =
     current_error_cost + ERROR_COST_PER_SKIPPED_TREE +
@@ -1534,7 +1540,7 @@ static bool ts_parser__check_progress(TSParser *self, Subtree *lookahead, const
   if (self->operation_count >= OP_COUNT_PER_PARSER_TIMEOUT_CHECK) {
     self->operation_count = 0;
   }
-  if (self->parse_options.progress_callback && position != NULL) {
+  if (position != NULL) {
     self->parse_state.current_byte_offset = *position;
     self->parse_state.has_error = self->has_error;
   }
@@ -1616,6 +1622,7 @@ static bool ts_parser__advance(
     // an ambiguous state. REDUCE actions always create a new stack
     // version, whereas SHIFT actions update the existing stack version
     // and terminate this loop.
+    bool did_reduce = false;
     StackVersion last_reduction_version = STACK_VERSION_NONE;
     for (uint32_t i = 0; i < table_entry.action_count; i++) {
       TSParseAction action = table_entry.actions[i];
@@ -1651,6 +1658,7 @@ static bool ts_parser__advance(
             action.reduce.dynamic_precedence, action.reduce.production_id,
             is_fragile, end_of_non_terminal_extra
           );
+          did_reduce = true;
           if (reduction_version != STACK_VERSION_NONE) {
             last_reduction_version = reduction_version;
           }
@@ -1702,9 +1710,12 @@ static bool ts_parser__advance(
       continue;
     }
 
-    // A non-terminal extra rule was reduced and merged into an existing
-    // stack version. This version can be discarded.
-    if (!lookahead.ptr) {
+    // A reduction was performed, but was merged into an existing stack version.
+    // This version can be discarded.
+    if (did_reduce) {
+      if (lookahead.ptr) {
+        ts_subtree_release(&self->tree_pool, lookahead);
+      }
       ts_stack_halt(self->stack, version);
       return true;
     }
@@ -1753,7 +1764,7 @@ static bool ts_parser__advance(
     // versions that exist. If some other version advances successfully, then
     // this version can simply be removed. But if all versions end up paused,
     // then error recovery is needed.
-    LOG("detect_error");
+    LOG("detect_error lookahead:%s", TREE_NAME(lookahead));
     ts_stack_pause(self->stack, version, lookahead);
     return true;
   }
@@ -1842,6 +1853,7 @@ static unsigned ts_parser__condense_stack(TSParser *self) {
           has_unpaused_version = true;
         } else {
           ts_stack_remove_version(self->stack, i);
+          made_changes = true;
           i--;
           n--;
         }
@@ -2226,6 +2238,8 @@ TSTree *ts_parser_parse_with_options(
   self->parse_options = parse_options;
   self->parse_state.payload = parse_options.payload;
   TSTree *result = ts_parser_parse(self, old_tree, input);
+  // Reset parser options before further parse calls.
+  self->parse_options = (TSParseOptions) {0};
   return result;
 }
 

lib/src/parser.h

@@ -18,7 +18,6 @@ typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
-typedef struct TSLanguageMetadata TSLanguageMetadata;
 typedef struct TSLanguageMetadata {
   uint8_t major_version;
   uint8_t minor_version;

lib/src/portable/endian.h

@@ -18,16 +18,20 @@
 #if defined(HAVE_ENDIAN_H) || \
     defined(__linux__) || \
     defined(__GNU__) || \
+    defined(__NetBSD__) || \
     defined(__OpenBSD__) || \
     defined(__CYGWIN__) || \
     defined(__MSYS__) || \
     defined(__EMSCRIPTEN__)
 
+#if defined(__NetBSD__)
+#define _NETBSD_SOURCE 1
+#endif
+
 # include <endian.h>
 
 #elif defined(HAVE_SYS_ENDIAN_H) || \
     defined(__FreeBSD__) || \
-    defined(__NetBSD__) || \
     defined(__DragonFly__)
 
 # include <sys/endian.h>

lib/src/query.c

@@ -1,3 +1,12 @@
+/*
+ * On NetBSD, defining standard requirements like this removes symbols
+ * from the namespace; however, we need non-standard symbols for
+ * endian.h.
+ */
+#if defined(__NetBSD__) && defined(_POSIX_C_SOURCE)
+#undef _POSIX_C_SOURCE
+#endif
+
 #include "tree_sitter/api.h"
 #include "./alloc.h"
 #include "./array.h"
@@ -2514,6 +2523,9 @@ static TSQueryError ts_query__parse_pattern(
           child_is_immediate,
           &child_capture_quantifiers
         );
+        // In the event we only parsed a predicate, meaning no new steps were added,
+        // then subtract one so we're not indexing past the end of the array
+        if (step_index == self->steps.size) step_index--;
         if (e == PARENT_DONE) {
           if (stream->next == ')') {
             if (child_is_immediate) {
@@ -2522,7 +2534,7 @@ static TSQueryError ts_query__parse_pattern(
                 return TSQueryErrorSyntax;
               }
               // Mark this step *and* its alternatives as the last child of the parent.
-              QueryStep *last_child_step = &self->steps.contents[last_child_step_index];
+              QueryStep *last_child_step = array_get(&self->steps, last_child_step_index);
               last_child_step->is_last_child = true;
               if (
                 last_child_step->alternative_index != NONE &&
@@ -2966,9 +2978,7 @@ const TSQueryPredicateStep *ts_query_predicates_for_pattern(
 ) {
   Slice slice = self->patterns.contents[pattern_index].predicate_steps;
   *step_count = slice.length;
-  if (self->predicate_steps.contents == NULL) {
-    return NULL;
-  }
+  if (slice.length == 0) return NULL;
   return &self->predicate_steps.contents[slice.offset];
 }
 

lib/src/stack.c

@@ -460,6 +460,17 @@ uint32_t ts_stack_version_count(const Stack *self) {
   return self->heads.size;
 }
 
+uint32_t ts_stack_halted_version_count(Stack *self) {
+  uint32_t count = 0;
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    StackHead *head = array_get(&self->heads, i);
+    if (head->status == StackStatusHalted) {
+      count++;
+    }
+  }
+  return count;
+}
+
 TSStateId ts_stack_state(const Stack *self, StackVersion version) {
   return array_get(&self->heads, version)->node->state;
 }
@@ -524,6 +535,7 @@ StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t c
   return stack__iter(self, version, pop_count_callback, &count, (int)count);
 }
 
+
 forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) {
   (void)payload;
   if (iterator->subtree_count >= 1) {

lib/src/stack.h

@@ -36,6 +36,9 @@ void ts_stack_delete(Stack *self);
 // Get the stack's current number of versions.
 uint32_t ts_stack_version_count(const Stack *self);
 
+// Get the stack's current number of halted versions.
+uint32_t ts_stack_halted_version_count(Stack *self);
+
 // Get the state at the top of the given version of the stack. If the stack is
 // empty, this returns the initial state, 0.
 TSStateId ts_stack_state(const Stack *self, StackVersion version);

lib/src/subtree.c

@@ -407,7 +407,12 @@ void ts_subtree_summarize_children(
     self.ptr->dynamic_precedence += ts_subtree_dynamic_precedence(child);
     self.ptr->visible_descendant_count += ts_subtree_visible_descendant_count(child);
 
-    if (alias_sequence && alias_sequence[structural_index] != 0 && !ts_subtree_extra(child)) {
+    if (
+      !ts_subtree_extra(child) &&
+      ts_subtree_symbol(child) != 0 &&
+      alias_sequence &&
+      alias_sequence[structural_index] != 0
+    ) {
       self.ptr->visible_descendant_count++;
       self.ptr->visible_child_count++;
       if (ts_language_symbol_metadata(language, alias_sequence[structural_index]).named) {

lib/src/tree_cursor.c

@@ -304,8 +304,9 @@ int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint go
 }
 
 TreeCursorStep ts_tree_cursor_goto_sibling_internal(
-    TSTreeCursor *_self,
-    bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) {
+  TSTreeCursor *_self,
+  bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)
+) {
   TreeCursor *self = (TreeCursor *)_self;
   uint32_t initial_size = self->stack.size;
 

lib/src/wasm_store.c

@@ -1220,6 +1220,10 @@ const TSLanguage *ts_wasm_store_load_language(
   const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
   memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory));
 
+  bool has_supertypes =
+    wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS &&
+    wasm_language.supertype_count > 0;
+
   int32_t addresses[] = {
     wasm_language.parse_table,
     wasm_language.small_parse_table,
@@ -1239,9 +1243,9 @@ const TSLanguage *ts_wasm_store_load_language(
     wasm_language.primary_state_ids,
     wasm_language.name,
     wasm_language.reserved_words,
-    wasm_language.supertype_symbols,
-    wasm_language.supertype_map_entries,
-    wasm_language.supertype_map_slices,
+    has_supertypes ? wasm_language.supertype_symbols : 0,
+    has_supertypes ? wasm_language.supertype_map_entries : 0,
+    has_supertypes ? wasm_language.supertype_map_slices : 0,
     wasm_language.external_token_count > 0 ? wasm_language.external_scanner.states : 0,
     wasm_language.external_token_count > 0 ? wasm_language.external_scanner.symbol_map : 0,
     wasm_language.external_token_count > 0 ? wasm_language.external_scanner.create : 0,
@@ -1331,7 +1335,7 @@ const TSLanguage *ts_wasm_store_load_language(
     );
   }
 
-  if (language->supertype_count > 0) {
+  if (has_supertypes) {
     language->supertype_symbols = copy(
       &memory[wasm_language.supertype_symbols],
       wasm_language.supertype_count * sizeof(TSSymbol)
@@ -1609,13 +1613,22 @@ static void ts_wasm_store__call(
   }
 }
 
+// The data fields of TSLexer, without the function pointers.
+//
+// This portion of the struct needs to be copied in and out
+// of wasm memory before and after calling a scan function.
+typedef struct {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+} TSLexerDataPrefix;
+
 static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned function_index, TSStateId state) {
   wasmtime_context_t *context = wasmtime_store_context(self->store);
   uint8_t *memory_data = wasmtime_memory_data(context, &self->memory);
   memcpy(
     &memory_data[self->lexer_address],
-    &self->current_lexer->lookahead,
-    sizeof(self->current_lexer->lookahead)
+    self->current_lexer,
+    sizeof(TSLexerDataPrefix)
   );
 
   wasmtime_val_raw_t args[2] = {
@@ -1627,9 +1640,9 @@ static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned functio
   bool result = args[0].i32;
 
   memcpy(
-    &self->current_lexer->lookahead,
+    self->current_lexer,
     &memory_data[self->lexer_address],
-    sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol)
+    sizeof(TSLexerDataPrefix)
   );
   return result;
 }
@@ -1674,8 +1687,8 @@ bool ts_wasm_store_call_scanner_scan(
 
   memcpy(
     &memory_data[self->lexer_address],
-    &self->current_lexer->lookahead,
-    sizeof(self->current_lexer->lookahead)
+    self->current_lexer,
+    sizeof(TSLexerDataPrefix)
   );
 
   uint32_t valid_tokens_address =
@@ -1690,9 +1703,9 @@ bool ts_wasm_store_call_scanner_scan(
   if (self->has_error) return false;
 
   memcpy(
-    &self->current_lexer->lookahead,
+    self->current_lexer,
     &memory_data[self->lexer_address],
-    sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol)
+    sizeof(TSLexerDataPrefix)
   );
   return args[0].i32;
 }

test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A document
+==========================
+
+a b
+
+---
+
+(document)

test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js

@@ -0,0 +1,11 @@
+module.exports = grammar({
+    name: 'epsilon_external_extra_tokens',
+
+    extras: $ => [/\s/, $.comment],
+
+    externals: $ => [$.comment],
+
+    rules: {
+        document: $ => seq('a', 'b'),
+    }
+});

test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c

@@ -0,0 +1,33 @@
+#include "tree_sitter/parser.h"
+
+enum TokenType {
+  COMMENT
+};
+
+void *tree_sitter_epsilon_external_extra_tokens_external_scanner_create(void) {
+  return NULL;
+}
+
+bool tree_sitter_epsilon_external_extra_tokens_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = COMMENT;
+  return true;
+}
+
+unsigned tree_sitter_epsilon_external_extra_tokens_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_epsilon_external_extra_tokens_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_epsilon_external_extra_tokens_external_scanner_destroy(void *payload) {}

xtask/src/bump.rs

@@ -127,6 +127,11 @@ pub fn run(args: BumpVersion) -> Result<()> {
         }
         next_version
     };
+    if next_version <= current_version {
+        return Err(anyhow!(format!(
+            "Next version {next_version} must be greater than current version {current_version}"
+        )));
+    }
 
     println!("Bumping from {current_version} to {next_version}");
     update_crates(&current_version, &next_version)?;

xtask/src/main.rs

@@ -144,7 +144,7 @@ struct Test {
     iterations: Option<u32>,
     /// Set the seed used to control random behavior.
     #[arg(long, short)]
-    seed: Option<u32>,
+    seed: Option<usize>,
     /// Print parsing log to stderr.
     #[arg(long, short)]
     debug: bool,