0.26.5

(cherry picked from commit 7100767f64)

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.26.3"
+        VERSION "0.26.5"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
@@ -81,7 +81,7 @@ set_target_properties(tree-sitter
                       SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
                       DEFINE_SYMBOL "")
 
-target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
 
 include(GNUInstallDirs)
 

Cargo.toml

@@ -14,7 +14,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.26.3"
+version = "0.26.5"
 authors = [
   "Max Brunsfeld <maxbrunsfeld@gmail.com>",
   "Amaan Qureshi <amaanq12@gmail.com>",
@@ -153,11 +153,11 @@ walkdir = "2.5.0"
 wasmparser = "0.243.0"
 webbrowser = "1.0.5"
 
-tree-sitter = { version = "0.26.3", path = "./lib" }
-tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" }
-tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" }
-tree-sitter-config = { version = "0.26.3", path = "./crates/config" }
-tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" }
-tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" }
+tree-sitter = { version = "0.26.5", path = "./lib" }
+tree-sitter-generate = { version = "0.26.5", path = "./crates/generate" }
+tree-sitter-loader = { version = "0.26.5", path = "./crates/loader" }
+tree-sitter-config = { version = "0.26.5", path = "./crates/config" }
+tree-sitter-highlight = { version = "0.26.5", path = "./crates/highlight" }
+tree-sitter-tags = { version = "0.26.5", path = "./crates/tags" }
 
 tree-sitter-language = { version = "0.1", path = "./crates/language" }

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.26.3
+VERSION := 0.26.5
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
 ARFLAGS := rcs
 CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning

Package.swift

@@ -27,6 +27,7 @@ let package = Package(
                         .headerSearchPath("src"),
                         .define("_POSIX_C_SOURCE", to: "200112L"),
                         .define("_DEFAULT_SOURCE"),
+                        .define("_BSD_SOURCE"),
                         .define("_DARWIN_C_SOURCE"),
                 ]),
     ],

build.zig

@@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
 
     lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
     lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
+    lib.root_module.addCMacro("_BSD_SOURCE", "");
     lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
 
     if (wasm) {

build.zig.zon

@@ -1,7 +1,7 @@
 .{
     .name = .tree_sitter,
     .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.26.3",
+    .version = "0.26.5",
     .minimum_zig_version = "0.14.1",
     .paths = .{
         "build.zig",

crates/cli/benches/benchmark.rs

@@ -19,9 +19,7 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
 static EXAMPLE_FILTER: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
 static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
-    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
-        .map(|s| s.parse::<usize>().unwrap())
-        .unwrap_or(5)
+    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
 });
 static TEST_LOADER: LazyLock<Loader> =
     LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));

crates/cli/npm/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.26.3",
+  "version": "0.26.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "tree-sitter-cli",
-      "version": "0.26.3",
+      "version": "0.26.5",
       "hasInstallScript": true,
       "license": "MIT",
       "bin": {

crates/cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.26.3",
+  "version": "0.26.5",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

crates/cli/src/fuzz.rs

@@ -44,11 +44,13 @@ pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
 
 pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
 
+pub const DEFAULT_EDIT_COUNT: usize = 3;
 pub static EDIT_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
+    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(DEFAULT_EDIT_COUNT));
 
+pub const DEFAULT_ITERATION_COUNT: usize = 10;
 pub static ITERATION_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
+    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(DEFAULT_ITERATION_COUNT));
 
 fn int_env_var(name: &'static str) -> Option<usize> {
     env::var(name).ok().and_then(|e| e.parse().ok())
@@ -221,7 +223,7 @@ pub fn fuzz_language_corpus(
                 }
 
                 // Perform a random series of edits and reparse.
-                let edit_count = rand.unsigned(*EDIT_COUNT);
+                let edit_count = rand.unsigned(options.edits);
                 let mut undo_stack = Vec::with_capacity(edit_count);
                 for _ in 0..=edit_count {
                     let edit = get_random_edit(&mut rand, &input);

crates/cli/src/init.rs

@@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
 use crc32fast::hash as crc32;
 use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
 use indoc::{formatdoc, indoc};
-use log::warn;
+use log::info;
 use rand::{thread_rng, Rng};
 use semver::Version;
 use serde::{Deserialize, Serialize};
@@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
 const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
 const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
 
-const TREE_SITTER_JSON_SCHEMA: &str =
+pub const TREE_SITTER_JSON_SCHEMA: &str =
     "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -356,7 +356,7 @@ pub fn generate_grammar_files(
                     "tree-sitter-cli":"#},
                 );
             if !contents.contains("module") {
-                warn!("Updating package.json");
+                info!("Migrating package.json to ESM");
                 contents = contents.replace(
                     r#""repository":"#,
                     indoc! {r#"
@@ -378,6 +378,7 @@ pub fn generate_grammar_files(
             |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if contents.contains("module.exports") {
+                    info!("Migrating grammars.js to ESM");
                     contents = contents.replace("module.exports =", "export default");
                     write_file(path, contents)?;
                 }
@@ -393,10 +394,16 @@ pub fn generate_grammar_files(
         allow_update,
         |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
         |path| {
-            let contents = fs::read_to_string(path)?;
+            let mut contents = fs::read_to_string(path)?;
             if !contents.contains("Zig artifacts") {
-                warn!("Replacing .gitignore");
-                generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
+                info!("Adding zig entries to .gitignore");
+                contents.push('\n');
+                contents.push_str(indoc! {"
+                # Zig artifacts
+                .zig-cache/
+                zig-cache/
+                zig-out/
+                "});
             }
             Ok(())
         },
@@ -409,8 +416,13 @@ pub fn generate_grammar_files(
         |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
         |path| {
             let mut contents = fs::read_to_string(path)?;
-            contents = contents.replace("bindings/c/* ", "bindings/c/** ");
+            let c_bindings_entry = "bindings/c/* ";
+            if contents.contains(c_bindings_entry) {
+                info!("Updating c bindings entry in .gitattributes");
+                contents = contents.replace(c_bindings_entry, "bindings/c/** ");
+            }
             if !contents.contains("Zig bindings") {
+                info!("Adding zig entries to .gitattributes");
                 contents.push('\n');
                 contents.push_str(indoc! {"
                 # Zig bindings
@@ -438,39 +450,40 @@ pub fn generate_grammar_files(
             }, |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if !contents.contains("#[cfg(with_highlights_query)]") {
-                let replacement = indoc! {r#"
-                    #[cfg(with_highlights_query)]
-                    /// The syntax highlighting query for this grammar.
-                    pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
+                    info!("Updating query constants in bindings/rust/lib.rs");
+                    let replacement = indoc! {r#"
+                        #[cfg(with_highlights_query)]
+                        /// The syntax highlighting query for this grammar.
+                        pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
 
-                    #[cfg(with_injections_query)]
-                    /// The language injection query for this grammar.
-                    pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
+                        #[cfg(with_injections_query)]
+                        /// The language injection query for this grammar.
+                        pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
 
-                    #[cfg(with_locals_query)]
-                    /// The local variable query for this grammar.
-                    pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
+                        #[cfg(with_locals_query)]
+                        /// The local variable query for this grammar.
+                        pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
 
-                    #[cfg(with_tags_query)]
-                    /// The symbol tagging query for this grammar.
-                    pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
-                    "#}
-                    .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                    .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                    .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                    .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
-                contents = contents
-                    .replace(
-                        indoc! {r#"
-                        // NOTE: uncomment these to include any queries that this grammar contains:
+                        #[cfg(with_tags_query)]
+                        /// The symbol tagging query for this grammar.
+                        pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
+                        "#}
+                        .replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
+                        .replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
+                        .replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
+                        .replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
+                    contents = contents
+                        .replace(
+                            indoc! {r#"
+                            // NOTE: uncomment these to include any queries that this grammar contains:
 
-                        // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-                        // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-                        // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-                        // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-                        "#},
-                        &replacement,
-                    );
+                            // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
+                            // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
+                            // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
+                            // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
+                            "#},
+                            &replacement,
+                        );
                 }
                 write_file(path, contents)?;
                 Ok(())
@@ -483,6 +496,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("wasm32-unknown-unknown") {
+                        info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
                         let replacement = indoc!{r#"
                             c_config.flag("-utf-8");
 
@@ -503,19 +517,18 @@ pub fn generate_grammar_files(
                                     wasm_src.join("string.c"),
                                 ]);
                             }
-                        "#};
-
-                        let indented_replacement = replacement
+                        "#}
                             .lines()
                             .map(|line| if line.is_empty() { line.to_string() } else { format!("    {line}") })
                             .collect::<Vec<_>>()
                             .join("\n");
 
-                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &indented_replacement);
+                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &replacement);
                     }
 
                     // Introduce configuration variables for dynamic query inclusion
                     if !contents.contains("with_highlights_query") {
+                        info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
                         let replaced = indoc! {r#"
                                 c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
                             }"#}
@@ -542,10 +555,10 @@ pub fn generate_grammar_files(
                                 }
                             }"#}
                             .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
-                            .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                            .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                            .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                            .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
+                            .replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
+                            .replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
+                            .replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
+                            .replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
 
                         contents = contents.replace(
                             &replaced,
@@ -572,6 +585,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("\"LICENSE\"") {
+                        info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
                         write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
                     }
                     Ok(())
@@ -592,7 +606,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("Object.defineProperty") {
-                        warn!("Replacing index.js");
+                        info!("Replacing index.js");
                         generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -606,7 +620,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("export default binding") {
-                        warn!("Replacing index.d.ts");
+                        info!("Replacing index.d.ts");
                         generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -627,7 +641,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("import") {
-                        warn!("Replacing binding_test.js");
+                        info!("Replacing binding_test.js");
                         generate_file(
                             path,
                             BINDING_TEST_JS_TEMPLATE,
@@ -650,6 +664,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("fs.exists(") {
+                        info!("Replacing `fs.exists` calls in binding.gyp");
                         write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
                     }
                     Ok(())
@@ -662,14 +677,17 @@ pub fn generate_grammar_files(
 
     // Generate C bindings
     if tree_sitter_config.bindings.c {
+        let kebab_case_name = language_name.to_kebab_case();
         missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
-            let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
+            let header_name = format!("tree-sitter-{kebab_case_name}.h");
+            let old_file = &path.join(&header_name);
             if allow_update && fs::exists(old_file).unwrap_or(false) {
+                info!("Removing bindings/c/{header_name}");
                 fs::remove_file(old_file)?;
             }
             missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
                 missing_path(
-                    include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
+                    include_path.join(&header_name),
                     |path| {
                         generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
                     },
@@ -678,7 +696,7 @@ pub fn generate_grammar_files(
             })?;
 
             missing_path(
-                path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
+                path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
                 |path| {
                     generate_file(
                         path,
@@ -698,23 +716,27 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
-                        warn!("Replacing Makefile");
+                        info!("Replacing Makefile");
                         generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
                     } else {
-                        contents = contents
-                            .replace(
-                                indoc! {r"
-                                $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
-                                "},
-                                indoc! {r"
-                                $(SRC_DIR)/grammar.json: grammar.js
-                                        $(TS) generate --no-parser $^
+                        let replaced = indoc! {r"
+                            $(PARSER): $(SRC_DIR)/grammar.json
+                                    $(TS) generate $^
+                            "};
+                        if contents.contains(replaced) {
+                            info!("Adding --no-parser target to Makefile");
+                            contents = contents
+                                .replace(
+                                    replaced,
+                                    indoc! {r"
+                                    $(SRC_DIR)/grammar.json: grammar.js
+                                            $(TS) generate --no-parser $^
 
-                                $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
-                                "}
-                            );
+                                    $(PARSER): $(SRC_DIR)/grammar.json
+                                            $(TS) generate $^
+                                    "}
+                                );
+                        }
                         write_file(path, contents)?;
                     }
                     Ok(())
@@ -726,8 +748,8 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let contents = fs::read_to_string(path)?;
+                    let replaced_contents = contents
                         .replace("add_custom_target(test", "add_custom_target(ts-test")
                         .replace(
                             &formatdoc! {r#"
@@ -775,7 +797,10 @@ pub fn generate_grammar_files(
                                                COMMENT "Generating parser.c")
                             "#}
                         );
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating CMakeLists.txt");
+                        write_file(path, replaced_contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -811,7 +836,8 @@ pub fn generate_grammar_files(
     // Generate Python bindings
     if tree_sitter_config.bindings.python {
         missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
-            let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
+            let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
+            let lang_path = path.join(&snake_case_grammar_name);
             missing_path(&lang_path, create_dir)?;
 
             missing_path_else(
@@ -821,6 +847,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("PyModuleDef_Init") {
+                        info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
                         contents = contents
                             .replace("PyModule_Create", "PyModuleDef_Init")
                             .replace(
@@ -862,7 +889,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.py");
+                        info!("Replacing __init__.py");
                         generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -876,9 +903,10 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.pyi");
+                        info!("Replacing __init__.pyi");
                         generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
                     } else if !contents.contains("CapsuleType") {
+                        info!("Updating __init__.pyi");
                         contents = contents
                             .replace(
                                 "from typing import Final",
@@ -910,6 +938,7 @@ pub fn generate_grammar_files(
                     |path| {
                         let mut contents = fs::read_to_string(path)?;
                         if !contents.contains("Parser(Language(") {
+                            info!("Updating Language function in bindings/python/tests/test_binding.py");
                             contents = contents
                                 .replace("tree_sitter.Language(", "Parser(Language(")
                                 .replace(".language())\n", ".language()))\n")
@@ -930,11 +959,19 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let contents = fs::read_to_string(path)?;
+                    let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("build_ext") {
-                        warn!("Replacing setup.py");
+                        info!("Replacing setup.py");
                         generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
+                    if !contents.contains(" and not get_config_var") {
+                        info!("Updating Python free-threading support in setup.py");
+                        contents = contents.replace(
+                            r#"startswith("cp"):"#,
+                            r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
+                        );
+                        write_file(path, contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -953,6 +990,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cp310-*") {
+                        info!("Updating dependencies in pyproject.toml");
                         contents = contents
                             .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
                             .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
@@ -990,15 +1028,18 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let contents = fs::read_to_string(path)?;
+                    let replaced_contents = contents
                         .replace(
                             "https://github.com/ChimeHQ/SwiftTreeSitter",
                             "https://github.com/tree-sitter/swift-tree-sitter",
                         )
                         .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
                         .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating tree-sitter dependency in Package.swift");
+                        write_file(path, contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -1016,7 +1057,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains("b.pkg_hash.len") {
-                    warn!("Replacing build.zig");
+                    info!("Replacing build.zig");
                     generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1031,7 +1072,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains(".name = .tree_sitter_") {
-                    warn!("Replacing build.zig.zon");
+                    info!("Replacing build.zig.zon");
                     generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1047,7 +1088,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("ts.Language") {
-                        warn!("Replacing root.zig");
+                        info!("Replacing root.zig");
                         generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
                     } else {
                         Ok(())
@@ -1189,17 +1230,20 @@ fn generate_file(
         .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name)
         .replace(
             HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.highlights_query_path,
+            &generate_opts.highlights_query_path.replace('\\', "/"),
         )
         .replace(
             INJECTIONS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.injections_query_path,
+            &generate_opts.injections_query_path.replace('\\', "/"),
         )
         .replace(
             LOCALS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.locals_query_path,
+            &generate_opts.locals_query_path.replace('\\', "/"),
         )
-        .replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
+        .replace(
+            TAGS_QUERY_PATH_PLACEHOLDER,
+            &generate_opts.tags_query_path.replace('\\', "/"),
+        );
 
     if let Some(name) = generate_opts.author_name {
         replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);

crates/cli/src/main.rs

@@ -16,11 +16,11 @@ use semver::Version as SemverVersion;
 use tree_sitter::{ffi, Parser, Point};
 use tree_sitter_cli::{
     fuzz::{
-        fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED,
-        LOG_GRAPH_ENABLED, START_SEED,
+        fuzz_language_corpus, FuzzOptions, DEFAULT_EDIT_COUNT, DEFAULT_ITERATION_COUNT, EDIT_COUNT,
+        ITERATION_COUNT, LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED,
     },
     highlight::{self, HighlightOptions},
-    init::{generate_grammar_files, JsonConfigOpts},
+    init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
     input::{get_input, get_tmp_source_file, CliInput},
     logger,
     parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
@@ -391,11 +391,15 @@ struct Fuzz {
     /// library's language function
     #[arg(long)]
     pub lang_name: Option<String>,
-    /// Maximum number of edits to perform per fuzz test
-    #[arg(long)]
+    #[arg(
+        long,
+        help=format!("Maximum number of edits to perform per fuzz test (Default: {DEFAULT_EDIT_COUNT})")
+    )]
     pub edits: Option<usize>,
-    /// Number of fuzzing iterations to run per test
-    #[arg(long)]
+    #[arg(
+        long,
+        help=format!("Number of fuzzing iterations to run per test (Default: {DEFAULT_ITERATION_COUNT})")
+    )]
     pub iterations: Option<usize>,
     /// Only fuzz corpus test cases whose name matches the given regex
     #[arg(long, short)]
@@ -867,10 +871,26 @@ impl Init {
 
             (opts.name.clone(), Some(opts))
         } else {
-            let mut json = serde_json::from_str::<TreeSitterJSON>(
-                &fs::read_to_string(current_dir.join("tree-sitter.json"))
-                    .with_context(|| "Failed to read tree-sitter.json")?,
-            )?;
+            let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
+                .with_context(|| "Failed to read tree-sitter.json")?;
+
+            let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
+            if json.schema.is_none() {
+                json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
+            }
+
+            let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
+            // Write the re-serialized config back, as newly added optional boolean fields
+            // will be included with explicit `false`s rather than implict `null`s
+            if self.update && !old_config.trim().eq(new_config.trim()) {
+                info!("Updating tree-sitter.json");
+                fs::write(
+                    current_dir.join("tree-sitter.json"),
+                    serde_json::to_string_pretty(&json)?,
+                )
+                .with_context(|| "Failed to write tree-sitter.json")?;
+            }
+
             (json.grammars.swap_remove(0).name, None)
         };
 
@@ -955,11 +975,21 @@ impl Build {
         } else {
             let output_path = if let Some(ref path) = self.output {
                 let path = Path::new(path);
-                if path.is_absolute() {
+                let full_path = if path.is_absolute() {
                     path.to_path_buf()
                 } else {
                     current_dir.join(path)
-                }
+                };
+                let parent_path = full_path
+                    .parent()
+                    .context("Output path must have a parent")?;
+                let name = full_path
+                    .file_name()
+                    .context("Ouput path must have a filename")?;
+                fs::create_dir_all(parent_path).context("Failed to create output path")?;
+                let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
+                canon_path.push(name);
+                canon_path
             } else {
                 let file_name = grammar_path
                     .file_stem()
@@ -984,7 +1014,7 @@ impl Build {
 
             loader
                 .compile_parser_at_path(&grammar_path, output_path, flags)
-                .unwrap();
+                .context("Failed to compile parser")?;
         }
         Ok(())
     }
@@ -1622,6 +1652,7 @@ impl Highlight {
         let loader_config = config.get()?;
         loader.find_all_languages(&loader_config)?;
         loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
+        let languages = loader.languages_at_path(current_dir)?;
 
         let cancellation_flag = util::cancel_on_signal();
 
@@ -1702,7 +1733,6 @@ impl Highlight {
             } => {
                 let path = get_tmp_source_file(&contents)?;
 
-                let languages = loader.languages_at_path(current_dir)?;
                 let language = languages
                     .iter()
                     .find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
@@ -1733,7 +1763,6 @@ impl Highlight {
                     if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
                         (l, lc)
                     } else {
-                        let languages = loader.languages_at_path(current_dir)?;
                         let language = languages
                             .first()
                             .map(|(l, _)| l.clone())

crates/cli/src/parse.rs

@@ -515,7 +515,6 @@ pub fn parse_file_at_path(
 
         if opts.output == ParseOutput::Cst {
             render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
-            println!();
         }
 
         if opts.output == ParseOutput::Xml {
@@ -785,7 +784,7 @@ pub fn render_cst<'a, 'b: 'a>(
         .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
         .max()
         .unwrap_or(1);
-    let mut indent_level = 1;
+    let mut indent_level = usize::from(!opts.no_ranges);
     let mut did_visit_children = false;
     let mut in_error = false;
     loop {
@@ -883,35 +882,24 @@ fn write_node_text(
                     0
                 };
             let formatted_line = render_line_feed(line, opts);
-            if !opts.no_ranges {
-                write!(
-                    out,
-                    "{}{}{}{}{}{}",
-                    if multiline { "\n" } else { "" },
-                    if multiline {
-                        render_node_range(opts, cursor, is_named, true, total_width, node_range)
-                    } else {
-                        String::new()
-                    },
-                    if multiline {
-                        "  ".repeat(indent_level + 1)
-                    } else {
-                        String::new()
-                    },
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            } else {
-                write!(
-                    out,
-                    "\n{}{}{}{}",
-                    "  ".repeat(indent_level + 1),
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            }
+            write!(
+                out,
+                "{}{}{}{}{}{}",
+                if multiline { "\n" } else { " " },
+                if multiline && !opts.no_ranges {
+                    render_node_range(opts, cursor, is_named, true, total_width, node_range)
+                } else {
+                    String::new()
+                },
+                if multiline {
+                    "  ".repeat(indent_level + 1)
+                } else {
+                    String::new()
+                },
+                paint(quote_color, &String::from(quote)),
+                paint(color, &render_node_text(&formatted_line)),
+                paint(quote_color, &String::from(quote)),
+            )?;
         }
     }
 
@@ -1011,10 +999,9 @@ fn cst_render_node(
         } else {
             opts.parse_theme.node_kind
         };
-        write!(out, "{}", paint(kind_color, node.kind()),)?;
+        write!(out, "{}", paint(kind_color, node.kind()))?;
 
         if node.child_count() == 0 {
-            write!(out, " ")?;
             // Node text from a pattern or external scanner
             write_node_text(
                 opts,

crates/cli/src/templates/setup.py

@@ -32,7 +32,7 @@ class BuildExt(build_ext):
 class BdistWheel(bdist_wheel):
     def get_tag(self):
         python, abi, platform = super().get_tag()
-        if python.startswith("cp"):
+        if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
             python, abi = "cp310", "abi3"
         return python, abi, platform
 

crates/cli/src/test.rs

@@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
             render_assertion_results("queries", &self.query_results)?;
         }
 
+        write!(f, "{}", self.parse_stats)?;
+
         Ok(())
     }
 }

crates/cli/src/tests/wasm_language_test.rs

@@ -118,6 +118,32 @@ fn test_load_fixture_language_wasm() {
     });
 }
 
+#[test]
+fn test_wasm_realloc_smaller_size() {
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let mut parser = Parser::new();
+        let language = get_test_fixture_language_wasm("wasm_realloc_overflow_heap");
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse("hello", None).unwrap();
+        assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
+    });
+}
+
+#[test]
+fn test_wasm_realloc_clobber_region() {
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let mut parser = Parser::new();
+        let language = get_test_fixture_language_wasm("wasm_realloc_clobber_region");
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse("hello", None).unwrap();
+        assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
+    });
+}
+
 #[test]
 fn test_load_multiple_wasm_languages() {
     allocations::record(|| {

crates/generate/src/generate.rs

@@ -41,7 +41,7 @@ pub use parse_grammar::ParseGrammarError;
 use prepare_grammar::prepare_grammar;
 pub use prepare_grammar::PrepareGrammarError;
 use render::render_c_code;
-pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
+pub use render::{RenderError, ABI_VERSION_MAX, ABI_VERSION_MIN};
 
 static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
     RegexBuilder::new("^\\s*//.*")
@@ -93,6 +93,8 @@ pub enum GenerateError {
     VariableInfo(#[from] VariableInfoError),
     #[error(transparent)]
     BuildTables(#[from] ParseTableBuilderError),
+    #[error(transparent)]
+    Render(#[from] RenderError),
     #[cfg(feature = "load")]
     #[error(transparent)]
     ParseVersion(#[from] ParseVersionError),
@@ -398,7 +400,7 @@ fn generate_parser_for_grammar_with_opts(
         abi_version,
         semantic_version,
         supertype_symbol_map,
-    );
+    )?;
     Ok(GeneratedParser {
         c_code,
         #[cfg(feature = "load")]

crates/generate/src/parse_grammar.rs

@@ -274,9 +274,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
             };
             let matches_empty = match inner_rule {
                 Rule::String(rule_str) => rule_str.is_empty(),
-                Rule::Pattern(ref value, _) => Regex::new(value)
-                    .map(|reg| reg.is_match(""))
-                    .unwrap_or(false),
+                Rule::Pattern(ref value, _) => Regex::new(value).is_ok_and(|reg| reg.is_match("")),
                 _ => false,
             };
             if matches_empty {

crates/generate/src/prepare_grammar/extract_tokens.rs

@@ -21,6 +21,8 @@ unless they are used only as the grammar's start rule.
 "
     )]
     EmptyString(String),
+    #[error("Terminal rule '{0}' cannot be used as a supertype")]
+    SupertypeTerminal(String),
     #[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
     ExternalTokenNonTerminal(String),
     #[error("Non-symbol rules cannot be used as external tokens")]
@@ -128,11 +130,18 @@ pub(super) fn extract_tokens(
         })
         .collect();
 
-    let supertype_symbols = grammar
+    let supertype_symbols: Vec<Symbol> = grammar
         .supertype_symbols
         .into_iter()
         .map(|symbol| symbol_replacer.replace_symbol(symbol))
         .collect();
+    for supertype_symbol in &supertype_symbols {
+        if supertype_symbol.is_terminal() {
+            Err(ExtractTokensError::SupertypeTerminal(
+                lexical_variables[supertype_symbol.index].name.clone(),
+            ))?;
+        }
+    }
 
     let variables_to_inline = grammar
         .variables_to_inline

crates/generate/src/prepare_grammar/process_inlines.rs

@@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
         let production_map = production_indices_by_step_id
             .into_iter()
             .map(|(step_id, production_indices)| {
-                let production = step_id.variable_index.map_or_else(
-                    || &productions[step_id.production_index],
-                    |variable_index| {
-                        &grammar.variables[variable_index].productions[step_id.production_index]
-                    },
-                ) as *const Production;
+                let production =
+                    core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
+                        || &productions[step_id.production_index],
+                        |variable_index| {
+                            &grammar.variables[variable_index].productions[step_id.production_index]
+                        },
+                    ));
                 ((production, step_id.step_index as u32), production_indices)
             })
             .collect();

crates/generate/src/render.rs

@@ -7,6 +7,8 @@ use std::{
 
 use crate::LANGUAGE_VERSION;
 use indoc::indoc;
+use serde::Serialize;
+use thiserror::Error;
 
 use super::{
     build_tables::Tables,
@@ -25,6 +27,16 @@ pub const ABI_VERSION_MIN: usize = 14;
 pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION;
 const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
 
+pub type RenderResult<T> = Result<T, RenderError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum RenderError {
+    #[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
+    ParseTable(usize),
+    #[error("This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {0}")]
+    ABI(usize),
+}
+
 #[clippy::format_args]
 macro_rules! add {
     ($this: tt, $($arg: tt)*) => {{
@@ -104,7 +116,7 @@ struct Metadata {
 }
 
 impl Generator {
-    fn generate(mut self) -> String {
+    fn generate(mut self) -> RenderResult<String> {
         self.init();
         self.add_header();
         self.add_includes();
@@ -161,7 +173,7 @@ impl Generator {
             self.add_reserved_word_sets();
         }
 
-        self.add_parse_table();
+        self.add_parse_table()?;
 
         if !self.syntax_grammar.external_tokens.is_empty() {
             self.add_external_token_enum();
@@ -171,7 +183,7 @@ impl Generator {
 
         self.add_parser_export();
 
-        self.buffer
+        Ok(self.buffer)
     }
 
     fn init(&mut self) {
@@ -1273,7 +1285,7 @@ impl Generator {
         add_line!(self, "");
     }
 
-    fn add_parse_table(&mut self) {
+    fn add_parse_table(&mut self) -> RenderResult<()> {
         let mut parse_table_entries = HashMap::new();
         let mut next_parse_action_list_index = 0;
 
@@ -1443,6 +1455,9 @@ impl Generator {
             add_line!(self, "}};");
             add_line!(self, "");
         }
+        if next_parse_action_list_index >= usize::from(u16::MAX) {
+            Err(RenderError::ParseTable(next_parse_action_list_index))?;
+        }
 
         let mut parse_table_entries = parse_table_entries
             .into_iter()
@@ -1450,6 +1465,8 @@ impl Generator {
             .collect::<Vec<_>>();
         parse_table_entries.sort_by_key(|(index, _)| *index);
         self.add_parse_action_list(parse_table_entries);
+
+        Ok(())
     }
 
     fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
@@ -1942,11 +1959,10 @@ pub fn render_c_code(
     abi_version: usize,
     semantic_version: Option<(u8, u8, u8)>,
     supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
-) -> String {
-    assert!(
-        (ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version),
-        "This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {abi_version}",
-    );
+) -> RenderResult<String> {
+    if !(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version) {
+        Err(RenderError::ABI(abi_version))?;
+    }
 
     Generator {
         language_name: name.to_string(),

crates/generate/src/templates/array.h

@@ -52,67 +52,91 @@ extern "C" {
 
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity) \
-  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity)        \
+  ((self)->contents = _array__reserve(           \
+    (void *)(self)->contents, &(self)->capacity, \
+    array_elem_size(self), new_capacity)         \
+  )
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((Array *)(self))
+#define array_delete(self) _array__delete((self), (void *)(self)->contents, sizeof(*self))
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                            \
-  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
-   (self)->contents[(self)->size++] = (element))
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__grow(                              \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    );                                                            \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count) \
-  do { \
-    if ((count) == 0) break; \
-    _array__grow((Array *)(self), count, array_elem_size(self)); \
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__grow(                                             \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    );                                                                           \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count); \
+    (self)->size += (count);                                                     \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other)                                       \
+#define array_push_all(self, other) \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, contents)                    \
-  _array__splice(                                               \
-    (Array *)(self), array_elem_size(self), (self)->size, \
-    0, count,  contents                                        \
+#define array_extend(self, count, other_contents)                 \
+  (self)->contents = _array__splice(                              \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
   )
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents)  \
-  _array__splice(                                                       \
-    (Array *)(self), array_elem_size(self), _index,                \
-    old_count, new_count, new_contents                                 \
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  (self)->contents = _array__splice(                                   \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
   )
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element) \
-  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+#define array_insert(self, _index, element)                     \
+  (self)->contents = _array__splice(                            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((Array *)(self), array_elem_size(self), _index)
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other) \
-  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+#define array_assign(self, other)                                   \
+  (self)->contents = _array__assign(                                \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )
 
 /// Swap one array with another
-#define array_swap(self, other) \
-  _array__swap((Array *)(self), (Array *)(other))
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    struct Swap swapped_contents = _array__swap(                    \
+      (void *)(self)->contents, &(self)->size, &(self)->capacity,   \
+      (void *)(other)->contents, &(other)->size, &(other)->capacity \
+    );                                                              \
+    (self)->contents = swapped_contents.self_contents;              \
+    (other)->contents = swapped_contents.other_contents;            \
+  } while (0)
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +181,112 @@ extern "C" {
 
 // Private
 
-typedef Array(void) Array;
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
 
 /// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(Array *self) {
-  if (self->contents) {
-    ts_free(self->contents);
-    self->contents = NULL;
-    self->size = 0;
-    self->capacity = 0;
-  }
+static inline void _array__delete(void *self, void *contents, size_t self_size) {
+  if (contents) ts_free(contents);
+  if (self) memset(self, 0, self_size);
 }
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(Array *self, size_t element_size,
-                                uint32_t index) {
-  assert(index < self->size);
-  char *contents = (char *)self->contents;
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  assert(index < *size);
+  char *contents = (char *)self_contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (self->size - index - 1) * element_size);
-  self->size--;
+          (*size - index - 1) * element_size);
+  (*size)--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
-  if (new_capacity > self->capacity) {
-    if (self->contents) {
-      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
     } else {
-      self->contents = ts_malloc(new_capacity * element_size);
+      new_contents = ts_malloc(new_capacity * element_size);
     }
-    self->capacity = new_capacity;
+    *capacity = new_capacity;
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
-  _array__reserve(self, element_size, other->size);
-  self->size = other->size;
-  memcpy(self->contents, other->contents, self->size * element_size);
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
 }
 
+struct Swap {
+  void *self_contents;
+  void *other_contents;
+};
+
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(Array *self, Array *other) {
-  Array swap = *other;
-  *other = *self;
-  *self = swap;
+// static inline void _array__swap(Array *self, Array *other) {
+static inline struct Swap _array__swap(void *self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                               void *other_contents, uint32_t *other_size, uint32_t *other_capacity) {
+  void *new_self_contents = other_contents;
+  uint32_t new_self_size = *other_size;
+  uint32_t new_self_capacity = *other_capacity;
+
+  void *new_other_contents = self_contents;
+  *other_size = *self_size;
+  *other_capacity = *self_capacity;
+
+  *self_size = new_self_size;
+  *self_capacity = new_self_capacity;
+
+  struct Swap out = {
+    .self_contents = new_self_contents,
+    .other_contents = new_other_contents,
+  };
+  return out;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
-  uint32_t new_size = self->size + count;
-  if (new_size > self->capacity) {
-    uint32_t new_capacity = self->capacity * 2;
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    _array__reserve(self, element_size, new_capacity);
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void _array__splice(Array *self, size_t element_size,
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t new_size = *size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  assert(old_end <= self->size);
+  assert(old_end <= *size);
 
-  _array__reserve(self, element_size, new_size);
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
 
-  char *contents = (char *)self->contents;
-  if (self->size > old_end) {
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (self->size - old_end) * element_size
+      (*size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -250,7 +304,9 @@ static inline void _array__splice(Array *self, size_t element_size,
       );
     }
   }
-  self->size += new_count - old_count;
+  *size += new_count - old_count;
+
+  return new_contents;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

crates/language/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-language"
 description = "The tree-sitter Language type, used by the library and by language implementations"
-version = "0.1.6"
+version = "0.1.7"
 authors.workspace = true
 edition.workspace = true
 rust-version = "1.77"

crates/language/wasm/include/stdint.h

@@ -23,9 +23,15 @@ typedef long unsigned int size_t;
 
 typedef long unsigned int uintptr_t;
 
-#define UINT16_MAX 65535
+#define INT8_MAX 127
+#define INT16_MAX 32767
+#define INT32_MAX 2147483647L
+#define INT64_MAX 9223372036854775807LL
 
+#define UINT8_MAX 255
+#define UINT16_MAX 65535
 #define UINT32_MAX 4294967295U
+#define UINT64_MAX 18446744073709551615ULL
 
 #if defined(__wasm32__)
 

crates/language/wasm/include/string.h

@@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
 
 int strncmp(const char *left, const char *right, size_t n);
 
+size_t strlen(const char *str);
+
 #endif // TREE_SITTER_WASM_STRING_H_

crates/language/wasm/src/stdio.c

@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <string.h>
 
 typedef struct {
   bool left_justify;   // -
@@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
   return 2 + len;
 }
 
-size_t strlen(const char *str) {
-  const char *s = str;
-  while (*s) s++;
-  return s - str;
-}
-
 char *strncpy(char *dest, const char *src, size_t n) {
   char *d = dest;
   const char *s = src;

crates/language/wasm/src/stdlib.c

@@ -48,6 +48,19 @@ static int grow_heap(size_t size) {
   return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX;
 }
 
+// Grows the heap if necessary to fit a region at the _end_ of the heap
+// ending at `region_end` by `size` bytes.
+//
+// Returns 0 if the heap could not be grown, 1 otherwise.
+static inline int grow_heap_for_region(Region *region_end, size_t size) {
+  if (region_end > heap_end) {
+    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) return 0;
+    if (!grow_heap(size)) return 0;
+    heap_end = get_heap_end();
+  }
+  return 1;
+}
+
 // Clear out the heap, and move it to the given address.
 void reset_heap(void *new_heap_start) {
   heap_start = new_heap_start;
@@ -76,13 +89,7 @@ void *malloc(size_t size) {
 
   Region *region_end = region_after(next, size);
 
-  if (region_end > heap_end) {
-    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) {
-      return NULL;
-    }
-    if (!grow_heap(size)) return NULL;
-    heap_end = get_heap_end();
-  }
+  if (!grow_heap_for_region(region_end, size)) return NULL;
 
   void *result = &next->data;
   next->size = size;
@@ -109,6 +116,7 @@ void free(void *ptr) {
 
 void *calloc(size_t count, size_t size) {
   void *result = malloc(count * size);
+  if (!result) return NULL;
   memset(result, 0, count * size);
   return result;
 }
@@ -117,19 +125,36 @@ void *realloc(void *ptr, size_t new_size) {
   if (ptr == NULL) {
     return malloc(new_size);
   }
+  if (new_size == 0) {
+    free(ptr);
+    return NULL;
+  }
+
 
   Region *region = region_for_ptr(ptr);
   Region *region_end = region_after(region, region->size);
 
-  // When reallocating the last allocated region, return
-  // the same pointer, and skip copying the data.
+  // When reallocating the last allocated region, resize
+  // in place if possible, return the same pointer, and
+  // skip copying the data.
   if (region_end == next) {
-    next = region;
-    return malloc(new_size);
+    Region *new_region_end = region_after(region, new_size);
+
+    size_t additional_size = (char *)new_region_end - (char *)heap_end;
+    if (!grow_heap_for_region(new_region_end, additional_size)) return NULL;
+
+    region->size = new_size;
+    next = new_region_end;
+    return &region->data;
   }
 
   void *result = malloc(new_size);
-  memcpy(result, &region->data, region->size);
+  if (!result) return NULL;
+
+  size_t copy_size = region->size < new_size ? region->size : new_size;
+  memcpy(result, &region->data, copy_size);
+
+  free(ptr);
   return result;
 }
 

crates/language/wasm/src/string.c

@@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
   }
   return 0;
 }
+
+size_t strlen(const char *str) {
+  const char *s = str;
+  while (*s) s++;
+  return s - str;
+}

crates/loader/src/loader.rs

@@ -1,6 +1,8 @@
 #![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 
+#[cfg(unix)]
+use std::fmt::Write as _;
 #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))]
 use std::ops::Range;
 #[cfg(feature = "tree-sitter-highlight")]
@@ -8,6 +10,7 @@ use std::sync::Mutex;
 use std::{
     collections::HashMap,
     env, fs,
+    hash::{Hash as _, Hasher as _},
     io::{BufRead, BufReader},
     marker::PhantomData,
     mem,
@@ -75,8 +78,6 @@ pub enum LoaderError {
     NoLanguage,
     #[error(transparent)]
     Query(LoaderQueryError),
-    #[error(transparent)]
-    ScannerSymbols(ScannerSymbolError),
     #[error("Failed to load language for scope '{0}':\n{1}")]
     ScopeLoad(String, Box<Self>),
     #[error(transparent)]
@@ -199,28 +200,6 @@ impl std::fmt::Display for SymbolError {
     }
 }
 
-#[derive(Debug, Error)]
-pub struct ScannerSymbolError {
-    pub missing: Vec<String>,
-}
-
-impl std::fmt::Display for ScannerSymbolError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(
-            f,
-            "Missing required functions in the external scanner, parsing won't work without these!\n"
-        )?;
-        for symbol in &self.missing {
-            writeln!(f, "  `{symbol}`")?;
-        }
-        writeln!(
-            f,
-            "You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n"
-        )?;
-        Ok(())
-    }
-}
-
 #[derive(Debug, Error)]
 pub struct WasiSDKClangError {
     pub wasi_sdk_dir: String,
@@ -1025,20 +1004,26 @@ impl Loader {
             return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
         }
 
+        // Create a unique lock path based on the output path hash to prevent
+        // interference when multiple processes build the same grammar (by name)
+        // to different output locations
+        let lock_hash = {
+            let mut hasher = std::hash::DefaultHasher::new();
+            output_path.hash(&mut hasher);
+            format!("{:x}", hasher.finish())
+        };
+
         let lock_path = if env::var("CROSS_RUNNER").is_ok() {
             tempfile::tempdir()
-                .unwrap()
+                .expect("create a temp dir")
                 .path()
-                .join("tree-sitter")
-                .join("lock")
-                .join(format!("{}.lock", config.name))
+                .to_path_buf()
         } else {
-            etcetera::choose_base_strategy()?
-                .cache_dir()
-                .join("tree-sitter")
-                .join("lock")
-                .join(format!("{}.lock", config.name))
-        };
+            etcetera::choose_base_strategy()?.cache_dir()
+        }
+        .join("tree-sitter")
+        .join("lock")
+        .join(format!("{}-{lock_hash}.lock", config.name));
 
         if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
             recompile = false;
@@ -1085,10 +1070,30 @@ impl Loader {
             self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?;
 
             if config.scanner_path.is_some() {
-                self.check_external_scanner(&config.name, &output_path)?;
+                self.check_external_scanner(&output_path)?;
             }
         }
 
+        // Ensure the dynamic library exists before trying to load it. This can
+        // happen in race conditions where we couldn't acquire the lock because
+        // another process was compiling but it still hasn't finished by the
+        // time we reach this point, so the output file still doesn't exist.
+        //
+        // Instead of allowing the `load_language` call below to fail, return a
+        // clearer error to the user here.
+        if !output_path.exists() {
+            let msg = format!(
+                "Dynamic library `{}` not found after build attempt. \
+                Are you running multiple processes building to the same output location?",
+                output_path.display()
+            );
+
+            Err(LoaderError::IO(IoError::new(
+                std::io::Error::new(std::io::ErrorKind::NotFound, msg),
+                Some(output_path.as_path()),
+            )))?;
+        }
+
         Self::load_language(&output_path, &language_fn_name)
     }
 
@@ -1187,6 +1192,7 @@ impl Loader {
                 command.arg("-UTREE_SITTER_REUSE_ALLOCATOR");
             } else {
                 command.arg("-shared");
+                command.arg("-Wl,--no-undefined");
             }
             command.args(cc_config.get_files());
             command.arg("-o").arg(output_path);
@@ -1221,25 +1227,15 @@ impl Loader {
     }
 
     #[cfg(unix)]
-    fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> {
-        let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) {
-            "_"
+    fn check_external_scanner(&self, library_path: &Path) -> LoaderResult<()> {
+        let section = " T ";
+        // Older ppc toolchains incorrectly report functions in the Data section. This bug has been
+        // fixed, but we still need to account for older systems.
+        let old_ppc_section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
+            Some(" D ")
         } else {
-            ""
+            None
         };
-        let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
-            " D "
-        } else {
-            " T "
-        };
-        let mut must_have = vec![
-            format!("{prefix}tree_sitter_{name}_external_scanner_create"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_destroy"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_serialize"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_deserialize"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_scan"),
-        ];
-
         let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned());
         let command = Command::new(nm_cmd)
             .arg("--defined-only")
@@ -1247,54 +1243,41 @@ impl Loader {
             .output();
         if let Ok(output) = command {
             if output.status.success() {
-                let mut found_non_static = false;
+                let mut non_static_symbols = String::new();
                 for line in String::from_utf8_lossy(&output.stdout).lines() {
-                    if line.contains(section) {
+                    if line.contains(section) || old_ppc_section.is_some_and(|s| line.contains(s)) {
                         if let Some(function_name) =
                             line.split_whitespace().collect::<Vec<_>>().get(2)
                         {
                             if !line.contains("tree_sitter_") {
-                                if !found_non_static {
-                                    found_non_static = true;
-                                    warn!("Found non-static non-tree-sitter functions in the external scanner");
-                                }
-                                warn!("  `{function_name}`");
-                            } else {
-                                must_have.retain(|f| f != function_name);
+                                writeln!(&mut non_static_symbols, "  `{function_name}`").unwrap();
                             }
                         }
                     }
                 }
-                if found_non_static {
-                    warn!(concat!(
-                        "Consider making these functions static, they can cause conflicts ",
-                        "when another tree-sitter project uses the same function name."
-                    ));
-                }
-
-                if !must_have.is_empty() {
-                    return Err(LoaderError::ScannerSymbols(ScannerSymbolError {
-                        missing: must_have,
-                    }));
+                if !non_static_symbols.is_empty() {
+                    warn!(
+                        "Found non-static non-tree-sitter functions in the external scanner\n{non_static_symbols}\n{}",
+                        concat!(
+                            "Consider making these functions static, they can cause conflicts ",
+                            "when another tree-sitter project uses the same function name."
+                        )
+                    );
                 }
             }
+        } else {
+            warn!(
+                "Failed to run `nm` to verify symbols in {}",
+                library_path.display()
+            );
         }
 
         Ok(())
     }
 
     #[cfg(windows)]
-    fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> {
+    fn check_external_scanner(&self, _library_path: &Path) -> LoaderResult<()> {
         // TODO: there's no nm command on windows, whoever wants to implement this can and should :)
-
-        // let mut must_have = vec![
-        //     format!("tree_sitter_{name}_external_scanner_create"),
-        //     format!("tree_sitter_{name}_external_scanner_destroy"),
-        //     format!("tree_sitter_{name}_external_scanner_serialize"),
-        //     format!("tree_sitter_{name}_external_scanner_deserialize"),
-        //     format!("tree_sitter_{name}_external_scanner_scan"),
-        // ];
-
         Ok(())
     }
 

crates/xtask/src/build_wasm.rs

@@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
         "-D", "NDEBUG=",
         "-D", "_POSIX_C_SOURCE=200112L",
         "-D", "_DEFAULT_SOURCE=",
+        "-D", "_BSD_SOURCE=",
         "-D", "_DARWIN_C_SOURCE=",
         "-I", "lib/src",
         "-I", "lib/include",

crates/xtask/src/fetch.rs

@@ -1,19 +1,74 @@
-use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION};
+use crate::{bail_on_err, root_dir, EMSCRIPTEN_VERSION};
 use anyhow::Result;
-use std::{fs, process::Command};
+use std::{fs, path::Path, process::Command};
 
-pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
+enum FixtureRef<'a> {
+    Tag(&'a str),
+    Branch(&'a str),
+}
+
+impl<'a> FixtureRef<'a> {
+    #[allow(clippy::use_self)]
+    const fn new(tag: &'a str, branch: Option<&'a str>) -> FixtureRef<'a> {
+        if let Some(b) = branch {
+            Self::Branch(b)
+        } else {
+            Self::Tag(tag)
+        }
+    }
+
+    const fn ref_type(&self) -> &'static str {
+        match self {
+            FixtureRef::Tag(_) => "tag",
+            FixtureRef::Branch(_) => "branch",
+        }
+    }
+}
+
+impl std::fmt::Display for FixtureRef<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FixtureRef::Tag(tag) => write!(f, "{tag}"),
+            FixtureRef::Branch(branch) => write!(f, "{branch}"),
+        }
+    }
+}
+
+fn current_ref_name(grammar_dir: &Path) -> Result<(String, Option<&'static str>)> {
+    let tag_args = ["describe", "--tags", "--exact-match", "HEAD"];
+    let branch_args = ["rev-parse", "--abbrev-ref", "HEAD"];
+
+    for (args, ref_type) in [tag_args.as_ref(), branch_args.as_ref()]
+        .iter()
+        .zip(&["tag", "branch"])
+    {
+        let name_cmd = Command::new("git")
+            .current_dir(grammar_dir)
+            .args(*args)
+            .output()?;
+        let name = String::from_utf8_lossy(&name_cmd.stdout);
+        let name = name.trim();
+        if !name.is_empty() {
+            return Ok((name.to_string(), Some(ref_type)));
+        }
+    }
+
+    Ok(("<unknown>".to_string(), None))
+}
+
+pub fn run_fixtures() -> Result<()> {
     let fixtures_dir = root_dir().join("test").join("fixtures");
     let grammars_dir = fixtures_dir.join("grammars");
     let fixtures_path = fixtures_dir.join("fixtures.json");
 
-    // grammar name, tag
-    let mut fixtures: Vec<(String, String)> =
+    // grammar name, tag, [branch]
+    let fixtures: Vec<(String, String, Option<String>)> =
         serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?;
 
-    for (grammar, tag) in &mut fixtures {
-        let grammar_dir = grammars_dir.join(&grammar);
+    for (grammar, tag, branch) in &fixtures {
+        let grammar_dir = grammars_dir.join(grammar);
         let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}");
+        let target_ref = FixtureRef::new(tag, branch.as_deref());
 
         println!("Fetching the {grammar} grammar...");
 
@@ -24,7 +79,7 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 "--depth",
                 "1",
                 "--branch",
-                tag,
+                &target_ref.to_string(),
                 &grammar_url,
                 &grammar_dir.to_string_lossy(),
             ]);
@@ -33,31 +88,71 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 &format!("Failed to clone the {grammar} grammar"),
             )?;
         } else {
-            let mut describe_command = Command::new("git");
-            describe_command.current_dir(&grammar_dir).args([
-                "describe",
-                "--tags",
-                "--exact-match",
-                "HEAD",
-            ]);
+            let (current_ref, current_ref_type) = current_ref_name(&grammar_dir)?;
+            if current_ref != target_ref.to_string() {
+                println!(
+                    "Updating {grammar} grammar from {} {current_ref} to {} {target_ref}...",
+                    current_ref_type.unwrap_or("<unknown>"),
+                    target_ref.ref_type(),
+                );
 
-            let output = describe_command.output()?;
-            let current_tag = String::from_utf8_lossy(&output.stdout);
-            let current_tag = current_tag.trim();
-
-            if current_tag != tag {
-                println!("Updating {grammar} grammar from {current_tag} to {tag}...");
-
-                let mut fetch_command = Command::new("git");
-                fetch_command.current_dir(&grammar_dir).args([
-                    "fetch",
-                    "origin",
-                    &format!("refs/tags/{tag}:refs/tags/{tag}"),
-                ]);
-                bail_on_err(
-                    &fetch_command.spawn()?.wait_with_output()?,
-                    &format!("Failed to fetch tag {tag} for {grammar} grammar"),
-                )?;
+                match target_ref {
+                    FixtureRef::Branch(branch) => {
+                        let mut fetch_cmd = Command::new("git");
+                        fetch_cmd.current_dir(&grammar_dir).args([
+                            "fetch",
+                            "--update-shallow",
+                            "origin",
+                            &format!("+refs/heads/{branch}:refs/remotes/origin/{branch}"),
+                        ]);
+                        bail_on_err(
+                            &fetch_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to fetch branch {branch}"),
+                        )?;
+                        let mut switch_cmd = Command::new("git");
+                        switch_cmd
+                            .current_dir(&grammar_dir)
+                            .args(["switch", branch]);
+                        bail_on_err(
+                            &switch_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to checkout branch {branch}"),
+                        )?;
+                        let mut set_upstream_cmd = Command::new("git");
+                        set_upstream_cmd.current_dir(&grammar_dir).args([
+                            "branch",
+                            "--set-upstream-to",
+                            &format!("origin/{branch}"),
+                            branch,
+                        ]);
+                        bail_on_err(
+                            &set_upstream_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to set upstream for branch {branch}"),
+                        )?;
+                        let mut pull_cmd = Command::new("git");
+                        pull_cmd
+                            .current_dir(&grammar_dir)
+                            .args(["pull", "origin", branch]);
+                        bail_on_err(
+                            &pull_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to pull latest from branch {branch}"),
+                        )?;
+                    }
+                    FixtureRef::Tag(tag) => {
+                        let mut fetch_command = Command::new("git");
+                        fetch_command.current_dir(&grammar_dir).args([
+                            "fetch",
+                            "origin",
+                            &format!("refs/tags/{tag}:refs/tags/{tag}"),
+                        ]);
+                        bail_on_err(
+                            &fetch_command.spawn()?.wait_with_output()?,
+                            &format!(
+                                "Failed to fetch {} {target_ref} for {grammar} grammar",
+                                target_ref.ref_type()
+                            ),
+                        )?;
+                    }
+                }
 
                 let mut reset_command = Command::new("git");
                 reset_command
@@ -71,29 +166,23 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 let mut checkout_command = Command::new("git");
                 checkout_command
                     .current_dir(&grammar_dir)
-                    .args(["checkout", tag]);
+                    .args(["checkout", &target_ref.to_string()]);
                 bail_on_err(
                     &checkout_command.spawn()?.wait_with_output()?,
-                    &format!("Failed to checkout tag {tag} for {grammar} grammar"),
+                    &format!(
+                        "Failed to checkout {} {target_ref} for {grammar} grammar",
+                        target_ref.ref_type()
+                    ),
                 )?;
             } else {
-                println!("{grammar} grammar is already at tag {tag}");
+                println!(
+                    "{grammar} grammar is already at {} {target_ref}",
+                    target_ref.ref_type()
+                );
             }
         }
     }
 
-    if args.update {
-        println!("Updating the fixtures lock file");
-        fs::write(
-            &fixtures_path,
-            // format the JSON without extra newlines
-            serde_json::to_string(&fixtures)?
-                .replace("[[", "[\n  [")
-                .replace("],", "],\n  ")
-                .replace("]]", "]\n]"),
-        )?;
-    }
-
     Ok(())
 }
 

crates/xtask/src/main.rs

@@ -36,7 +36,7 @@ enum Commands {
     /// Fetches emscripten.
     FetchEmscripten,
     /// Fetches the fixtures for testing tree-sitter.
-    FetchFixtures(FetchFixtures),
+    FetchFixtures,
     /// Generate the Rust bindings from the C library.
     GenerateBindings,
     /// Generates the fixtures for testing tree-sitter.
@@ -118,13 +118,6 @@ struct Clippy {
     package: Option<String>,
 }
 
-#[derive(Args)]
-struct FetchFixtures {
-    /// Update all fixtures to the latest tag
-    #[arg(long, short)]
-    update: bool,
-}
-
 #[derive(Args)]
 struct GenerateFixtures {
     /// Generates the parser to Wasm
@@ -232,8 +225,8 @@ fn run() -> Result<()> {
         Commands::CheckWasmExports(check_options) => check_wasm_exports::run(&check_options)?,
         Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?,
         Commands::FetchEmscripten => fetch::run_emscripten()?,
-        Commands::FetchFixtures(fetch_fixture_options) => {
-            fetch::run_fixtures(&fetch_fixture_options)?;
+        Commands::FetchFixtures => {
+            fetch::run_fixtures()?;
         }
         Commands::GenerateBindings => generate::run_bindings()?,
         Commands::GenerateFixtures(generate_fixtures_options) => {

docs/src/assets/js/playground.js

@@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
 }
 
 window.initializePlayground = async (opts) => {
-  const { Parser, Language } = window.TreeSitter;
+  const { Parser, Language, Query } = window.TreeSitter;
 
   const { local } = opts;
   if (local) {
@@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
       marks.forEach((m) => m.clear());
 
       if (tree && query) {
-        const captures = query.captures(
-          tree.rootNode,
-          { row: startRow, column: 0 },
-          { row: endRow, column: 0 },
-        );
+        const captures = query.captures(tree.rootNode, {
+          startPosition: { row: startRow, column: 0 },
+          endPosition: { row: endRow, column: 0 },
+        });
         let lastNodeId;
         for (const { name, node } of captures) {
           if (node.id === lastNodeId) continue;
@@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
       const queryText = queryEditor.getValue();
 
       try {
-        query = parser.language.query(queryText);
+        query = new Query(parser.language, queryText);
         let match;
 
         let row = 0;

docs/src/cli/fuzz.md

@@ -31,11 +31,13 @@ If `--lib-path` is used, the name of the language used to extract the library's
 
 ### `--edits <EDITS>`
 
-The maximum number of edits to perform. The default is 3.
+The maximum number of edits to perform. The default is 3. This value can also be set via the `TREE_SITTER_EDITS` environment
+variable.
 
 ### `--iterations <ITERATIONS>`
 
-The number of iterations to run. The default is 10.
+The number of iterations to run. The default is 10. This value can also be set via the `TREE_SITTER_ITERATIONS` environment
+variable.
 
 ### `-i/--include <INCLUDE>`
 

flake.nix

@@ -17,7 +17,7 @@
       eachSystem = lib.genAttrs systems;
       pkgsFor = inputs.nixpkgs.legacyPackages;
 
-      version = "0.26.3";
+      version = "0.26.5";
 
       fs = lib.fileset;
       src = fs.toSource {

lib/binding_rust/build.rs

@@ -49,6 +49,7 @@ fn main() {
         .include(&include_path)
         .define("_POSIX_C_SOURCE", "200112L")
         .define("_DEFAULT_SOURCE", None)
+        .define("_BSD_SOURCE", None)
         .define("_DARWIN_C_SOURCE", None)
         .warnings(false)
         .file(src_path.join("lib.c"))

lib/binding_web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.26.3",
+  "version": "0.26.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "web-tree-sitter",
-      "version": "0.26.3",
+      "version": "0.26.5",
       "license": "MIT",
       "devDependencies": {
         "@eslint/js": "^9.39.1",

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.26.3",
+  "version": "0.26.5",
   "description": "Tree-sitter bindings for the web",
   "repository": {
     "type": "git",

lib/src/array.h

@@ -52,67 +52,91 @@ extern "C" {
 
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity) \
-  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity)        \
+  ((self)->contents = _array__reserve(           \
+    (void *)(self)->contents, &(self)->capacity, \
+    array_elem_size(self), new_capacity)         \
+  )
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((Array *)(self))
+#define array_delete(self) _array__delete((self), (void *)(self)->contents, sizeof(*self))
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                            \
-  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
-   (self)->contents[(self)->size++] = (element))
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__grow(                              \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    );                                                            \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count) \
-  do { \
-    if ((count) == 0) break; \
-    _array__grow((Array *)(self), count, array_elem_size(self)); \
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__grow(                                             \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    );                                                                           \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count); \
+    (self)->size += (count);                                                     \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other)                                       \
+#define array_push_all(self, other) \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, contents)                    \
-  _array__splice(                                               \
-    (Array *)(self), array_elem_size(self), (self)->size, \
-    0, count,  contents                                        \
+#define array_extend(self, count, other_contents)                 \
+  (self)->contents = _array__splice(                              \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
   )
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents)  \
-  _array__splice(                                                       \
-    (Array *)(self), array_elem_size(self), _index,                \
-    old_count, new_count, new_contents                                 \
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  (self)->contents = _array__splice(                                   \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
   )
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element) \
-  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+#define array_insert(self, _index, element)                     \
+  (self)->contents = _array__splice(                            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((Array *)(self), array_elem_size(self), _index)
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other) \
-  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+#define array_assign(self, other)                                   \
+  (self)->contents = _array__assign(                                \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )
 
 /// Swap one array with another
-#define array_swap(self, other) \
-  _array__swap((Array *)(self), (Array *)(other))
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    struct Swap swapped_contents = _array__swap(                    \
+      (void *)(self)->contents, &(self)->size, &(self)->capacity,   \
+      (void *)(other)->contents, &(other)->size, &(other)->capacity \
+    );                                                              \
+    (self)->contents = swapped_contents.self_contents;              \
+    (other)->contents = swapped_contents.other_contents;            \
+  } while (0)
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +181,112 @@ extern "C" {
 
 // Private
 
-typedef Array(void) Array;
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
 
 /// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(Array *self) {
-  if (self->contents) {
-    ts_free(self->contents);
-    self->contents = NULL;
-    self->size = 0;
-    self->capacity = 0;
-  }
+static inline void _array__delete(void *self, void *contents, size_t self_size) {
+  if (contents) ts_free(contents);
+  if (self) memset(self, 0, self_size);
 }
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(Array *self, size_t element_size,
-                                uint32_t index) {
-  ts_assert(index < self->size);
-  char *contents = (char *)self->contents;
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  ts_assert(index < *size);
+  char *contents = (char *)self_contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (self->size - index - 1) * element_size);
-  self->size--;
+          (*size - index - 1) * element_size);
+  (*size)--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
-  if (new_capacity > self->capacity) {
-    if (self->contents) {
-      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
     } else {
-      self->contents = ts_malloc(new_capacity * element_size);
+      new_contents = ts_malloc(new_capacity * element_size);
     }
-    self->capacity = new_capacity;
+    *capacity = new_capacity;
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
-  _array__reserve(self, element_size, other->size);
-  self->size = other->size;
-  memcpy(self->contents, other->contents, self->size * element_size);
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
 }
 
+struct Swap {
+  void *self_contents;
+  void *other_contents;
+};
+
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(Array *self, Array *other) {
-  Array swap = *other;
-  *other = *self;
-  *self = swap;
+// static inline void _array__swap(Array *self, Array *other) {
+static inline struct Swap _array__swap(void *self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                               void *other_contents, uint32_t *other_size, uint32_t *other_capacity) {
+  void *new_self_contents = other_contents;
+  uint32_t new_self_size = *other_size;
+  uint32_t new_self_capacity = *other_capacity;
+
+  void *new_other_contents = self_contents;
+  *other_size = *self_size;
+  *other_capacity = *self_capacity;
+
+  *self_size = new_self_size;
+  *self_capacity = new_self_capacity;
+
+  struct Swap out = {
+    .self_contents = new_self_contents,
+    .other_contents = new_other_contents,
+  };
+  return out;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
-  uint32_t new_size = self->size + count;
-  if (new_size > self->capacity) {
-    uint32_t new_capacity = self->capacity * 2;
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    _array__reserve(self, element_size, new_capacity);
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void _array__splice(Array *self, size_t element_size,
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t new_size = *size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  ts_assert(old_end <= self->size);
+  ts_assert(old_end <= *size);
 
-  _array__reserve(self, element_size, new_size);
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
 
-  char *contents = (char *)self->contents;
-  if (self->size > old_end) {
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (self->size - old_end) * element_size
+      (*size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -250,7 +304,9 @@ static inline void _array__splice(Array *self, size_t element_size,
       );
     }
   }
-  self->size += new_count - old_count;
+  *size += new_count - old_count;
+
+  return new_contents;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

lib/src/wasm_store.c

@@ -255,7 +255,7 @@ static bool wasm_dylink_info__parse(
  * Native callbacks exposed to Wasm modules
  *******************************************/
 
- static wasm_trap_t *callback__abort(
+static wasm_trap_t *callback__abort(
   void *env,
   wasmtime_caller_t* caller,
   wasmtime_val_raw_t *args_and_results,

test/fixtures/fixtures.json

@@ -1,17 +1,17 @@
 [
-  ["bash","v0.25.0"],
-  ["c","v0.24.1"],
-  ["cpp","v0.23.4"],
-  ["embedded-template","v0.25.0"],
-  ["go","v0.25.0"],
-  ["html","v0.23.2"],
-  ["java","v0.23.5"],
-  ["javascript","v0.25.0"],
-  ["jsdoc","v0.23.2"],
-  ["json","v0.24.8"],
-  ["php","v0.24.2"],
-  ["python","v0.23.6"],
-  ["ruby","v0.23.1"],
-  ["rust","v0.24.0"],
-  ["typescript","v0.23.2"]
-]
+  ["bash","v0.25.0", null],
+  ["c","v0.24.1", null],
+  ["cpp","v0.23.4", null],
+  ["embedded-template","v0.25.0", null],
+  ["go","v0.25.0", null],
+  ["html","v0.23.2", null],
+  ["java","v0.23.5", null],
+  ["javascript","v0.25.0", null],
+  ["jsdoc","v0.23.2", null],
+  ["json","v0.24.8", null],
+  ["php","v0.24.2", "upstream_test_fixture"],
+  ["python","v0.23.6", null],
+  ["ruby","v0.23.1", null],
+  ["rust","v0.24.0", null],
+  ["typescript","v0.23.2", null]
+]

test/fixtures/test_grammars/terminal_supertype/expected_error.txt

@@ -0,0 +1 @@
+Terminal rule 'A' cannot be used as a supertype

test/fixtures/test_grammars/terminal_supertype/grammar.js

@@ -0,0 +1,18 @@
+module.exports = grammar({
+  name: 'terminal_supertype',
+
+  supertypes: $ => [$.A],
+  extras: $ => [$.lineComment, $.lineComment2],
+
+  rules: {
+    source_file: $ => choice(
+      $.B,
+      $.A,
+    ),
+    lineComment: $ => /;;.*/,
+    lineComment2: $ => /##.*/,
+    A: $ => /xyz/,
+    C: $ => "xyz",
+    B: $ => seq('abc', $.C),
+  }
+});

test/fixtures/test_grammars/wasm_realloc_clobber_region/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A leading zero-width token
+==========================
+
+hello
+
+---
+
+(document (zero_width))

test/fixtures/test_grammars/wasm_realloc_clobber_region/grammar.js

@@ -0,0 +1,10 @@
+export default grammar({
+    name: 'wasm_realloc_clobber_region',
+
+    extras: _ => [/\s/],
+    externals: $ => [$.zero_width],
+
+    rules: {
+        document: $ => seq($.zero_width, 'hello'),
+    }
+});

test/fixtures/test_grammars/wasm_realloc_clobber_region/scanner.c

@@ -0,0 +1,68 @@
+#include "tree_sitter/parser.h"
+#include <string.h>
+
+#ifdef __wasm__
+  #define panic(x) __builtin_trap()
+#else
+  #include <assert.h>
+  #define panic(x) assert(0)
+#endif
+
+enum TokenType {
+  ZERO_WIDTH_TOKEN
+};
+
+void *tree_sitter_wasm_realloc_clobber_region_external_scanner_create(void) {
+  size_t a_count = 32;
+  void *a = malloc(a_count);
+  memset(a, 0x41, a_count); // fill `a` with 'A's
+  for (int i = 0; i < 32; i++) {
+    // This condition is unreachable, but it prevents wasi-sdk clang from optimizing
+    // out the allocation to `a`.
+    if (((char *)a)[i] != 0x41) panic();
+  }
+
+  size_t b_count = 16;
+  void *b = malloc(b_count);
+  memset(b, 0x42, b_count); // fill `b` with 'B's
+
+  // `a` is now first on the free list
+  free(a);
+  // We're re-allocating the last region, so `realloc`'s optimization to re-use the region
+  // by altering its metadata's size (thus avoiding a memcpy) should be used.
+  void *c = realloc(b, 24); // realloc `b` should still contain 'B's
+
+  // ensure the contents of `b` were not clobbered. A previous bug caused `a`s region
+  // to be used for `c`.
+  char *p = (char *)c;
+  for (int i = 0; i < (int)b_count; i++) {
+    if (p[i] != 0x42) panic();
+  }
+
+  free(c);
+  return NULL;
+}
+
+bool tree_sitter_wasm_realloc_clobber_region_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = ZERO_WIDTH_TOKEN;
+  return true;
+}
+
+unsigned tree_sitter_wasm_realloc_clobber_region_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_wasm_realloc_clobber_region_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_wasm_realloc_clobber_region_external_scanner_destroy(void *payload) {}

test/fixtures/test_grammars/wasm_realloc_overflow_heap/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A leading zero-width token
+==========================
+
+hello
+
+---
+
+(document (zero_width))

test/fixtures/test_grammars/wasm_realloc_overflow_heap/grammar.js

@@ -0,0 +1,10 @@
+export default grammar({
+    name: 'wasm_realloc_overflow_heap',
+
+    extras: _ => [/\s/],
+    externals: $ => [$.zero_width],
+
+    rules: {
+        document: $ => seq($.zero_width, 'hello'),
+    }
+});

test/fixtures/test_grammars/wasm_realloc_overflow_heap/scanner.c

@@ -0,0 +1,43 @@
+#include "tree_sitter/parser.h"
+
+// Constant copied from `crates/language/wasm/src/stdlib.c`,
+// Must be kept in sync for a reliable repro. 
+#define MAX_HEAP_SIZE (4 * 1024 * 1024)
+
+enum TokenType {
+  ZERO_WIDTH_TOKEN
+};
+
+void *tree_sitter_wasm_realloc_overflow_heap_external_scanner_create(void) {
+  size_t large_size = MAX_HEAP_SIZE - 64;
+  void *p = malloc(large_size);
+  void *q = malloc(4);
+  p = realloc(p, 4);
+  free(p);
+  free(q);
+  return NULL;
+}
+
+bool tree_sitter_wasm_realloc_overflow_heap_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = ZERO_WIDTH_TOKEN;
+  return true;
+}
+
+unsigned tree_sitter_wasm_realloc_overflow_heap_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_wasm_realloc_overflow_heap_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_wasm_realloc_overflow_heap_external_scanner_destroy(void *payload) {}