release v0.26.10

(cherry picked from commit 41d3e16eac)

.github/actions/cache/action.yml

@@ -10,7 +10,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       id: cache
       with:
         path: |
@@ -22,4 +22,5 @@ runs:
           'lib/src/array.h',
           'lib/src/alloc.h',
           'test/fixtures/grammars/*/**/src/*.c',
+          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}

.github/workflows/build.yml

@@ -32,6 +32,7 @@ jobs:
           # When adding a new `target`:
           # 1. Define a new platform alias above
           # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
+          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
           - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
           - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
           - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
@@ -46,8 +47,8 @@ jobs:
 
           # Extra features
           - { platform: linux-arm64     , features: wasm }
-          - { platform: linux-x64       , features: wasm }
-          - { platform: macos-arm64     , features: wasm }
+          - { platform: linux-x64       , features: wasm , run-wasm-test: true }
+          - { platform: macos-arm64     , features: wasm , run-wasm-test: true }
           - { platform: macos-x64       , features: wasm }
 
           # Cross-compilation
@@ -85,14 +86,37 @@ jobs:
         } >> $GITHUB_ENV
 
     - name: Get emscripten version
-      if: contains(matrix.features, 'wasm')
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
 
-    - name: Install Emscripten
-      if: contains(matrix.features, 'wasm')
-      uses: mymindstorm/setup-emsdk@v14
+    - name: Cache Emscripten SDK
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/cache@v4
       with:
-        version: ${{ env.EMSCRIPTEN_VERSION }}
+        path: emsdk
+        key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
+
+    - name: Install Emscripten
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      run: |
+        if [[ ! -d emsdk ]]; then
+          git clone --depth 1 https://github.com/emscripten-core/emsdk.git
+        fi
+        cd emsdk
+        ./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
+        ./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
+        echo "$PWD" >> "$GITHUB_PATH"
+        echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
+        echo "EMSDK=$PWD" >> "$GITHUB_ENV"
+        echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
+
+    - name: Set up Node.js
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/setup-node@v4
+      with:
+        node-version: 24
+        cache: npm
+        cache-dependency-path: lib/binding_web/package-lock.json
 
     - name: Set up Rust
       uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -117,34 +141,12 @@ jobs:
       if: matrix.platform == 'windows-x64'
       uses: msys2/setup-msys2@v2
       with:
-        update: true
         install: |
           mingw-w64-x86_64-toolchain
           mingw-w64-x86_64-clang
           mingw-w64-x86_64-make
           mingw-w64-x86_64-cmake
 
-    # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
-    # the `mismatched-lifetime-syntaxes` lint
-    - name: Build wasmtime library (Windows x64 MSYS2)
-      if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64'
-      run: |
-        mkdir -p target
-        WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
-                           jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
-        cd target/wasmtime-${WASMTIME_VERSION}
-        cmake -S crates/c-api -B target/c-api \
-          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
-          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
-          -DWASMTIME_FEATURE_CRANELIFT=ON \
-          -DWASMTIME_TARGET='x86_64-pc-windows-gnu'
-        cmake --build target/c-api && cmake --install target/c-api
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
-      env:
-        WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
-
     - name: Build C library (Windows x64 MSYS2 CMake)
       if: matrix.platform == 'windows-x64'
       shell: msys2 {0}
@@ -171,26 +173,23 @@ jobs:
       env:
         WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
 
-    # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
-    # the `mismatched-lifetime-syntaxes` lint
-    - name: Build wasmtime library
-      if: contains(matrix.features, 'wasm')
+    - name: Download wasmtime C API
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       run: |
         mkdir -p target
         WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
                            jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
-        cd target/wasmtime-${WASMTIME_VERSION}
-        cmake -S crates/c-api -B target/c-api \
-          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
-          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
-          -DWASMTIME_FEATURE_CRANELIFT=ON \
-          -DWASMTIME_TARGET='${{ matrix.target }}'
-        cmake --build target/c-api && cmake --install target/c-api
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
+        case '${{ matrix.target }}' in
+          x86_64-unknown-linux-gnu)  WT_TARGET=x86_64-linux ;;
+          aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
+          x86_64-apple-darwin)       WT_TARGET=x86_64-macos ;;
+          aarch64-apple-darwin)      WT_TARGET=aarch64-macos ;;
+        esac
+        curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
+          | tar xJf - -C target
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
       env:
         WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
 
     - name: Build C library (make)
       if: runner.os != 'Windows'
@@ -207,7 +206,7 @@ jobs:
         make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
       env:
         PLATFORM: ${{ matrix.platform }}
-        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
     - name: Build C library (CMake)
       if: "!matrix.cross"
@@ -227,10 +226,10 @@ jobs:
         cmake --build build/shared --verbose
       env:
         CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
-        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
+        WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
 
     - name: Build Wasm library
-      if: contains(matrix.features, 'wasm')
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       shell: bash
       run: |
         cd lib/binding_web
@@ -247,7 +246,8 @@ jobs:
       run: cargo check --no-default-features --target='${{ matrix.target }}'
 
     - name: Build target
-      run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE
+      if: "!inputs.run-test"
+      run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
       env:
         PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
 
@@ -265,20 +265,20 @@ jobs:
       run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
 
     - name: Generate Wasm fixtures
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true'
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
       run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
 
     - name: Run main tests
       if: inputs.run-test && !matrix.no-run
-      run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}'
+      run: cargo test --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
 
     - name: Run Wasm tests
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm')
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
       run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
 
     - name: Upload CLI artifact
-      if: "!matrix.no-run"
-      uses: actions/upload-artifact@v5
+      if: "!inputs.run-test && !matrix.no-run"
+      uses: actions/upload-artifact@v6
       with:
         name: tree-sitter.${{ matrix.platform }}
         path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
@@ -286,8 +286,8 @@ jobs:
         retention-days: 7
 
     - name: Upload Wasm artifacts
-      if: matrix.platform == 'linux-x64'
-      uses: actions/upload-artifact@v5
+      if: "!inputs.run-test && matrix.platform == 'linux-x64'"
+      uses: actions/upload-artifact@v6
       with:
         name: tree-sitter.wasm
         path: |

.github/workflows/crate_versions.yml

@@ -0,0 +1,22 @@
+name: Crate Versions Check
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+  workflow_dispatch:
+
+jobs:
+  check-crates:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Check crates against crates.io
+        uses: katyo/publish-crates@v2
+        with:
+          dry-run: true

.github/workflows/release.yml

@@ -45,6 +45,7 @@ jobs:
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
+            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
@@ -54,12 +55,14 @@ jobs:
         with:
           subject-path: |
             target/tree-sitter-*.gz
+            target/tree-sitter-cli-*.zip
             target/web-tree-sitter.tar.gz
 
       - name: Create release
         run: |-
           gh release create $GITHUB_REF_NAME \
             target/tree-sitter-*.gz \
+            target/tree-sitter-cli-*.zip \
             target/web-tree-sitter.tar.gz
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/wasm_exports.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Build C library (make)
         run: make -j CFLAGS="$CFLAGS"
         env:
-          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
       - name: Build Wasm Library
         working-directory: lib/binding_web

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.26.3"
+        VERSION "0.26.10"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
@@ -33,7 +33,8 @@ if(MSVC)
 else()
   target_compile_options(tree-sitter PRIVATE
                          -Wall -Wextra -Wshadow -Wpedantic
-                         -Werror=incompatible-pointer-types)
+                         -Werror=incompatible-pointer-types
+                         -Werror=strict-aliasing -Wstrict-aliasing=2)
 endif()
 
 if(TREE_SITTER_FEATURE_WASM)
@@ -81,7 +82,7 @@ set_target_properties(tree-sitter
                       SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
                       DEFINE_SYMBOL "")
 
-target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
 
 include(GNUInstallDirs)
 

Cargo.toml

@@ -14,7 +14,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.26.3"
+version = "0.26.10"
 authors = [
   "Max Brunsfeld <maxbrunsfeld@gmail.com>",
   "Amaan Qureshi <amaanq12@gmail.com>",
@@ -153,11 +153,11 @@ walkdir = "2.5.0"
 wasmparser = "0.243.0"
 webbrowser = "1.0.5"
 
-tree-sitter = { version = "0.26.3", path = "./lib" }
-tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" }
-tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" }
-tree-sitter-config = { version = "0.26.3", path = "./crates/config" }
-tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" }
-tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" }
+tree-sitter = { version = "0.26.10", path = "./lib" }
+tree-sitter-generate = { version = "0.26.10", path = "./crates/generate", default-features = false }
+tree-sitter-loader = { version = "0.26.10", path = "./crates/loader" }
+tree-sitter-config = { version = "0.26.10", path = "./crates/config" }
+tree-sitter-highlight = { version = "0.26.10", path = "./crates/highlight" }
+tree-sitter-tags = { version = "0.26.10", path = "./crates/tags" }
 
 tree-sitter-language = { version = "0.1", path = "./crates/language" }

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.26.3
+VERSION := 0.26.10
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -22,9 +22,9 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -122,7 +122,6 @@ test-wasm:
 
 lint:
 	cargo update --workspace --locked --quiet
-	cargo check --workspace --all-targets
 	cargo fmt --all --check
 	cargo clippy --workspace --all-targets -- -D warnings
 

Package.swift

@@ -27,6 +27,7 @@ let package = Package(
                         .headerSearchPath("src"),
                         .define("_POSIX_C_SOURCE", to: "200112L"),
                         .define("_DEFAULT_SOURCE"),
+                        .define("_BSD_SOURCE"),
                         .define("_DARWIN_C_SOURCE"),
                 ]),
     ],

build.zig

@@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
 
     lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
     lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
+    lib.root_module.addCMacro("_BSD_SOURCE", "");
     lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
 
     if (wasm) {

build.zig.zon

@@ -1,7 +1,7 @@
 .{
     .name = .tree_sitter,
     .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.26.3",
+    .version = "0.26.10",
     .minimum_zig_version = "0.14.1",
     .paths = .{
         "build.zig",
@@ -13,83 +13,83 @@
     },
     .dependencies = .{
         .wasmtime_c_api_aarch64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIfPIgdw2YnV3QyiFQ2NHdrxrXzzCdjYJyxJDOta",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-aarch64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAF_TGAScX_LFOvFsd_2DCTFyBbQkTXp-1NDJEWw2",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAIt97QZi7Pf7nNJ2mVY6uxA80Klyuvvtop3pLMRK",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-aarch64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAFFxGgSHQS9Q-eRPQ4K_LaVi24HTgcSKnN-p3f1b",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAAO48QQf91w9RmmUDHTja8DrXZA1n6Bmc8waW3qe",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-aarch64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAKeC3gKf_ZY5DgFk2AKLLlU1giW3wumLG8nFE-OU",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAI196wa9pwADoA2RbCDp5F7bKQg1iOPq6gIh8-FH",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-aarch64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AACtSGARdJhgFq_MqHMOXiB8SFg_qBxtxqofjBN4t",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-windows-c-api.zip",
-            .hash = "N-V-__8AAC9u4wXfqd1Q6XyQaC8_DbQZClXux60Vu5743N05",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-aarch64-windows-c-api.zip",
+            .hash = "",
             .lazy = true,
         },
         .wasmtime_c_api_armv7_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-armv7-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAHXe8gWs3s83Cc5G6SIq0_jWxj8fGTT5xG4vb6-x",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-armv7-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAPdLawNEZYAVbrh4_2oOmRy-g-Tnijz67BQ67eIF",
             .lazy = true,
         },
         .wasmtime_c_api_i686_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAN2pzgUUfulRCYnipSfis9IIYHoTHVlieLRmKuct",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-i686-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAPEKpwMIY5e-o452Vdo3zLOpktSb19weYx9X5W3K",
             .lazy = true,
         },
         .wasmtime_c_api_i686_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-windows-c-api.zip",
-            .hash = "N-V-__8AAJu0YAUUTFBLxFIOi-MSQVezA6MMkpoFtuaf2Quf",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-i686-windows-c-api.zip",
+            .hash = "",
             .lazy = true,
         },
         .wasmtime_c_api_riscv64gc_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-riscv64gc-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAG8m-gc3E3AIImtTZ3l1c7HC6HUWazQ9OH5KACX4",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-riscv64gc-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAPevUwWzGGoZ-oxbgIzEcc6u-YZST7vNMwqrlw8u",
             .lazy = true,
         },
         .wasmtime_c_api_s390x_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-s390x-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAH314gd-gE4IBp2uvAL3gHeuW1uUZjMiLLeUdXL_",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-s390x-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAONHVARoH020TxBULSeOlz7VdXeGhS3O3LCkq1hF",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIPNRwfNkznebrcGb0IKUe7f35bkuZEYOjcx6q3f",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-x86_64-android-c-api.tar.xz",
+            .hash = "N-V-__8AACtumgRV4GtWxYYrpB4ok-bJa5cOsrSr21vWy5is",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAI8EDwcyTtk_Afhk47SEaqfpoRqGkJeZpGs69ChF",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-x86_64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAOc4mQRf7b8HzWmIoMnz-5Tni3Zd-j-EwUeZ3gb2",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAGtGNgVaOpHSxC22IjrampbRIy6lLwscdcAE8nG1",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-x86_64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAPdQVQMFmy3q3cQM-SLUXcqego0xSVT9X58XJsIP",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_mingw = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-mingw-c-api.zip",
-            .hash = "N-V-__8AAPS2PAbVix50L6lnddlgazCPTz3whLUFk1qnRtnZ",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-x86_64-mingw-c-api.zip",
+            .hash = "",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAF-WEQe0nzvi09PgusM5i46FIuCKJmIDWUleWgQ3",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-x86_64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAGFHnATdrqVxCCogyib2Q9CxwDZdjp2GyLcDLM78",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-windows-c-api.zip",
-            .hash = "N-V-__8AAKGNXwbpJQsn0_6kwSIVDDWifSg8cBzf7T2RzsC9",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.12/wasmtime-v36.0.12-x86_64-windows-c-api.zip",
+            .hash = "",
             .lazy = true,
         },
     },

crates/cli/Cargo.toml

@@ -67,7 +67,7 @@ wasmparser.workspace = true
 webbrowser.workspace = true
 
 tree-sitter.workspace = true
-tree-sitter-generate.workspace = true
+tree-sitter-generate = { workspace = true, features = ["load"] }
 tree-sitter-config.workspace = true
 tree-sitter-highlight.workspace = true
 tree-sitter-loader.workspace = true
@@ -81,3 +81,36 @@ tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter
 tempfile.workspace = true
 pretty_assertions.workspace = true
 unindent.workspace = true
+
+[package.metadata.binstall]
+pkg-fmt = "zip"
+
+[package.metadata.binstall.overrides.aarch64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.armv7-unknown-linux-gnueabihf]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.powerpc64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-powerpc64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-x64{ archive-suffix }"

crates/cli/README.md

@@ -11,16 +11,15 @@ The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars fr
 
 ### Installation
 
-You can install the `tree-sitter-cli` with `cargo`:
+You can install the `tree-sitter-cli` with `cargo-binstall`:
 
 ```sh
-cargo install --locked tree-sitter-cli
+cargo binstall tree-sitter-cli
 ```
 
-or with `npm`:
-
+or you can build it from source:
 ```sh
-npm install tree-sitter-cli
+cargo install --locked tree-sitter-cli
 ```
 
 You can also download a pre-built binary for your platform from [the releases page].

crates/cli/benches/benchmark.rs

@@ -19,9 +19,7 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
 static EXAMPLE_FILTER: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
 static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
-    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
-        .map(|s| s.parse::<usize>().unwrap())
-        .unwrap_or(5)
+    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
 });
 static TEST_LOADER: LazyLock<Loader> =
     LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));

crates/cli/npm/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.26.3",
+  "version": "0.26.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "tree-sitter-cli",
-      "version": "0.26.3",
+      "version": "0.26.10",
       "hasInstallScript": true,
       "license": "MIT",
       "bin": {

crates/cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.26.3",
+  "version": "0.26.10",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

crates/cli/src/fuzz.rs

@@ -44,11 +44,13 @@ pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
 
 pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
 
+pub const DEFAULT_EDIT_COUNT: usize = 3;
 pub static EDIT_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
+    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(DEFAULT_EDIT_COUNT));
 
+pub const DEFAULT_ITERATION_COUNT: usize = 10;
 pub static ITERATION_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
+    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(DEFAULT_ITERATION_COUNT));
 
 fn int_env_var(name: &'static str) -> Option<usize> {
     env::var(name).ok().and_then(|e| e.parse().ok())
@@ -221,7 +223,7 @@ pub fn fuzz_language_corpus(
                 }
 
                 // Perform a random series of edits and reparse.
-                let edit_count = rand.unsigned(*EDIT_COUNT);
+                let edit_count = rand.unsigned(options.edits);
                 let mut undo_stack = Vec::with_capacity(edit_count);
                 for _ in 0..=edit_count {
                     let edit = get_random_edit(&mut rand, &input);
@@ -253,7 +255,7 @@ pub fn fuzz_language_corpus(
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
+                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 

crates/cli/src/highlight.rs

@@ -190,20 +190,14 @@ fn parse_style(style: &mut Style, json: Value) {
     if let Value::Object(entries) = json {
         for (property_name, value) in entries {
             match property_name.as_str() {
-                "bold" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.bold();
-                    }
+                "bold" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.bold();
                 }
-                "italic" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.italic();
-                    }
+                "italic" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.italic();
                 }
-                "underline" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.underline();
-                    }
+                "underline" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.underline();
                 }
                 "color" => {
                     if let Some(color) = parse_color(value) {

crates/cli/src/init.rs

@@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
 use crc32fast::hash as crc32;
 use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
 use indoc::{formatdoc, indoc};
-use log::warn;
+use log::info;
 use rand::{thread_rng, Rng};
 use semver::Version;
 use serde::{Deserialize, Serialize};
@@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
 const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
 const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
 
-const TREE_SITTER_JSON_SCHEMA: &str =
+pub const TREE_SITTER_JSON_SCHEMA: &str =
     "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -356,7 +356,7 @@ pub fn generate_grammar_files(
                     "tree-sitter-cli":"#},
                 );
             if !contents.contains("module") {
-                warn!("Updating package.json");
+                info!("Migrating package.json to ESM");
                 contents = contents.replace(
                     r#""repository":"#,
                     indoc! {r#"
@@ -378,6 +378,7 @@ pub fn generate_grammar_files(
             |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if contents.contains("module.exports") {
+                    info!("Migrating grammars.js to ESM");
                     contents = contents.replace("module.exports =", "export default");
                     write_file(path, contents)?;
                 }
@@ -393,10 +394,16 @@ pub fn generate_grammar_files(
         allow_update,
         |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
         |path| {
-            let contents = fs::read_to_string(path)?;
+            let mut contents = fs::read_to_string(path)?;
             if !contents.contains("Zig artifacts") {
-                warn!("Replacing .gitignore");
-                generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
+                info!("Adding zig entries to .gitignore");
+                contents.push('\n');
+                contents.push_str(indoc! {"
+                # Zig artifacts
+                .zig-cache/
+                zig-cache/
+                zig-out/
+                "});
             }
             Ok(())
         },
@@ -409,8 +416,13 @@ pub fn generate_grammar_files(
         |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
         |path| {
             let mut contents = fs::read_to_string(path)?;
-            contents = contents.replace("bindings/c/* ", "bindings/c/** ");
+            let c_bindings_entry = "bindings/c/* ";
+            if contents.contains(c_bindings_entry) {
+                info!("Updating c bindings entry in .gitattributes");
+                contents = contents.replace(c_bindings_entry, "bindings/c/** ");
+            }
             if !contents.contains("Zig bindings") {
+                info!("Adding zig entries to .gitattributes");
                 contents.push('\n');
                 contents.push_str(indoc! {"
                 # Zig bindings
@@ -438,39 +450,40 @@ pub fn generate_grammar_files(
             }, |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if !contents.contains("#[cfg(with_highlights_query)]") {
-                let replacement = indoc! {r#"
-                    #[cfg(with_highlights_query)]
-                    /// The syntax highlighting query for this grammar.
-                    pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
+                    info!("Updating query constants in bindings/rust/lib.rs");
+                    let replacement = indoc! {r#"
+                        #[cfg(with_highlights_query)]
+                        /// The syntax highlighting query for this grammar.
+                        pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
 
-                    #[cfg(with_injections_query)]
-                    /// The language injection query for this grammar.
-                    pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
+                        #[cfg(with_injections_query)]
+                        /// The language injection query for this grammar.
+                        pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
 
-                    #[cfg(with_locals_query)]
-                    /// The local variable query for this grammar.
-                    pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
+                        #[cfg(with_locals_query)]
+                        /// The local variable query for this grammar.
+                        pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
 
-                    #[cfg(with_tags_query)]
-                    /// The symbol tagging query for this grammar.
-                    pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
-                    "#}
-                    .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                    .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                    .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                    .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
-                contents = contents
-                    .replace(
-                        indoc! {r#"
-                        // NOTE: uncomment these to include any queries that this grammar contains:
+                        #[cfg(with_tags_query)]
+                        /// The symbol tagging query for this grammar.
+                        pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
+                        "#}
+                        .replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
+                        .replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
+                        .replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
+                        .replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
+                    contents = contents
+                        .replace(
+                            indoc! {r#"
+                            // NOTE: uncomment these to include any queries that this grammar contains:
 
-                        // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-                        // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-                        // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-                        // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-                        "#},
-                        &replacement,
-                    );
+                            // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
+                            // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
+                            // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
+                            // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
+                            "#},
+                            &replacement,
+                        );
                 }
                 write_file(path, contents)?;
                 Ok(())
@@ -483,6 +496,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("wasm32-unknown-unknown") {
+                        info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
                         let replacement = indoc!{r#"
                             c_config.flag("-utf-8");
 
@@ -503,19 +517,18 @@ pub fn generate_grammar_files(
                                     wasm_src.join("string.c"),
                                 ]);
                             }
-                        "#};
-
-                        let indented_replacement = replacement
+                        "#}
                             .lines()
                             .map(|line| if line.is_empty() { line.to_string() } else { format!("    {line}") })
                             .collect::<Vec<_>>()
                             .join("\n");
 
-                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &indented_replacement);
+                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &replacement);
                     }
 
                     // Introduce configuration variables for dynamic query inclusion
                     if !contents.contains("with_highlights_query") {
+                        info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
                         let replaced = indoc! {r#"
                                 c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
                             }"#}
@@ -542,10 +555,10 @@ pub fn generate_grammar_files(
                                 }
                             }"#}
                             .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
-                            .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                            .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                            .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                            .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
+                            .replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
+                            .replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
+                            .replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
+                            .replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
 
                         contents = contents.replace(
                             &replaced,
@@ -572,6 +585,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("\"LICENSE\"") {
+                        info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
                         write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
                     }
                     Ok(())
@@ -592,7 +606,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("Object.defineProperty") {
-                        warn!("Replacing index.js");
+                        info!("Replacing index.js");
                         generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -606,7 +620,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("export default binding") {
-                        warn!("Replacing index.d.ts");
+                        info!("Replacing index.d.ts");
                         generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -627,7 +641,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("import") {
-                        warn!("Replacing binding_test.js");
+                        info!("Replacing binding_test.js");
                         generate_file(
                             path,
                             BINDING_TEST_JS_TEMPLATE,
@@ -650,6 +664,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("fs.exists(") {
+                        info!("Replacing `fs.exists` calls in binding.gyp");
                         write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
                     }
                     Ok(())
@@ -662,14 +677,17 @@ pub fn generate_grammar_files(
 
     // Generate C bindings
     if tree_sitter_config.bindings.c {
+        let kebab_case_name = language_name.to_kebab_case();
         missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
-            let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
+            let header_name = format!("tree-sitter-{kebab_case_name}.h");
+            let old_file = &path.join(&header_name);
             if allow_update && fs::exists(old_file).unwrap_or(false) {
+                info!("Removing bindings/c/{header_name}");
                 fs::remove_file(old_file)?;
             }
             missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
                 missing_path(
-                    include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
+                    include_path.join(&header_name),
                     |path| {
                         generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
                     },
@@ -678,7 +696,7 @@ pub fn generate_grammar_files(
             })?;
 
             missing_path(
-                path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
+                path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
                 |path| {
                     generate_file(
                         path,
@@ -698,23 +716,40 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
-                        warn!("Replacing Makefile");
+                        info!("Replacing Makefile");
                         generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
                     } else {
-                        contents = contents
-                            .replace(
-                                indoc! {r"
-                                $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
-                                "},
-                                indoc! {r"
-                                $(SRC_DIR)/grammar.json: grammar.js
-                                        $(TS) generate --no-parser $^
+                        let replaced = indoc! {r"
+                            $(PARSER): $(SRC_DIR)/grammar.json
+                                    $(TS) generate $^
+                            "};
+                        if contents.contains(replaced) {
+                            info!("Adding --no-parser target to Makefile");
+                            contents = contents
+                                .replace(
+                                    replaced,
+                                    indoc! {r"
+                                    $(SRC_DIR)/grammar.json: grammar.js
+                                            $(TS) generate --no-parser $^
 
-                                $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
-                                "}
-                            );
+                                    $(PARSER): $(SRC_DIR)/grammar.json
+                                            $(TS) generate $^
+                                    "}
+                                );
+                        }
+                        if !contents.contains("\nDESCRIPTION :=") {
+                            if let Some(version_line) = contents.lines().find(|l| l.starts_with("VERSION := ")) {
+                                info!("Adding DESCRIPTION to Makefile");
+                                let description = generate_opts.description.map_or_else(
+                                    || format!("{} grammar for tree-sitter", generate_opts.camel_parser_name),
+                                    str::to_string,
+                                );
+                                contents = contents.replace(
+                                    version_line,
+                                    &format!("{version_line}\nDESCRIPTION := {description}"),
+                                );
+                            }
+                        }
                         write_file(path, contents)?;
                     }
                     Ok(())
@@ -726,8 +761,8 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let contents = fs::read_to_string(path)?;
+                    let replaced_contents = contents
                         .replace("add_custom_target(test", "add_custom_target(ts-test")
                         .replace(
                             &formatdoc! {r#"
@@ -775,7 +810,10 @@ pub fn generate_grammar_files(
                                                COMMENT "Generating parser.c")
                             "#}
                         );
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating CMakeLists.txt");
+                        write_file(path, replaced_contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -811,7 +849,8 @@ pub fn generate_grammar_files(
     // Generate Python bindings
     if tree_sitter_config.bindings.python {
         missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
-            let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
+            let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
+            let lang_path = path.join(&snake_case_grammar_name);
             missing_path(&lang_path, create_dir)?;
 
             missing_path_else(
@@ -821,6 +860,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("PyModuleDef_Init") {
+                        info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
                         contents = contents
                             .replace("PyModule_Create", "PyModuleDef_Init")
                             .replace(
@@ -861,8 +901,8 @@ pub fn generate_grammar_files(
                 },
                 |path| {
                     let contents = fs::read_to_string(path)?;
-                    if !contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.py");
+                    if contents.contains("uncomment these to include any queries") {
+                        info!("Replacing __init__.py");
                         generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -876,9 +916,10 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.pyi");
+                        info!("Replacing __init__.pyi");
                         generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
                     } else if !contents.contains("CapsuleType") {
+                        info!("Updating __init__.pyi");
                         contents = contents
                             .replace(
                                 "from typing import Final",
@@ -910,6 +951,7 @@ pub fn generate_grammar_files(
                     |path| {
                         let mut contents = fs::read_to_string(path)?;
                         if !contents.contains("Parser(Language(") {
+                            info!("Updating Language function in bindings/python/tests/test_binding.py");
                             contents = contents
                                 .replace("tree_sitter.Language(", "Parser(Language(")
                                 .replace(".language())\n", ".language()))\n")
@@ -930,11 +972,19 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let contents = fs::read_to_string(path)?;
+                    let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("build_ext") {
-                        warn!("Replacing setup.py");
+                        info!("Replacing setup.py");
                         generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
+                    if !contents.contains(" and not get_config_var") {
+                        info!("Updating Python free-threading support in setup.py");
+                        contents = contents.replace(
+                            r#"startswith("cp"):"#,
+                            r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
+                        );
+                        write_file(path, contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -953,6 +1003,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cp310-*") {
+                        info!("Updating dependencies in pyproject.toml");
                         contents = contents
                             .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
                             .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
@@ -990,15 +1041,18 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let contents = fs::read_to_string(path)?;
+                    let replaced_contents = contents
                         .replace(
                             "https://github.com/ChimeHQ/SwiftTreeSitter",
                             "https://github.com/tree-sitter/swift-tree-sitter",
                         )
                         .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
                         .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating tree-sitter dependency in Package.swift");
+                        write_file(path, replaced_contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -1016,7 +1070,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains("b.pkg_hash.len") {
-                    warn!("Replacing build.zig");
+                    info!("Replacing build.zig");
                     generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1031,7 +1085,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains(".name = .tree_sitter_") {
-                    warn!("Replacing build.zig.zon");
+                    info!("Replacing build.zig.zon");
                     generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1047,7 +1101,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("ts.Language") {
-                        warn!("Replacing root.zig");
+                        info!("Replacing root.zig");
                         generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
                     } else {
                         Ok(())
@@ -1189,17 +1243,20 @@ fn generate_file(
         .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name)
         .replace(
             HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.highlights_query_path,
+            &generate_opts.highlights_query_path.replace('\\', "/"),
         )
         .replace(
             INJECTIONS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.injections_query_path,
+            &generate_opts.injections_query_path.replace('\\', "/"),
         )
         .replace(
             LOCALS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.locals_query_path,
+            &generate_opts.locals_query_path.replace('\\', "/"),
         )
-        .replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
+        .replace(
+            TAGS_QUERY_PATH_PLACEHOLDER,
+            &generate_opts.tags_query_path.replace('\\', "/"),
+        );
 
     if let Some(name) = generate_opts.author_name {
         replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);

crates/cli/src/main.rs

@@ -16,11 +16,11 @@ use semver::Version as SemverVersion;
 use tree_sitter::{ffi, Parser, Point};
 use tree_sitter_cli::{
     fuzz::{
-        fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED,
-        LOG_GRAPH_ENABLED, START_SEED,
+        fuzz_language_corpus, FuzzOptions, DEFAULT_EDIT_COUNT, DEFAULT_ITERATION_COUNT, EDIT_COUNT,
+        ITERATION_COUNT, LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED,
     },
     highlight::{self, HighlightOptions},
-    init::{generate_grammar_files, JsonConfigOpts},
+    init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
     input::{get_input, get_tmp_source_file, CliInput},
     logger,
     parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
@@ -234,7 +234,7 @@ struct Parse {
     /// Output the parse data in a pretty-printed CST format
     #[arg(long = "cst", short = 'c')]
     pub output_cst: bool,
-    /// Show parsing statistic
+    /// Show parsing statistics
     #[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")]
     pub stat: bool,
     /// Interrupt the parsing process by timeout (µs)
@@ -391,11 +391,15 @@ struct Fuzz {
     /// library's language function
     #[arg(long)]
     pub lang_name: Option<String>,
-    /// Maximum number of edits to perform per fuzz test
-    #[arg(long)]
+    #[arg(
+        long,
+        help=format!("Maximum number of edits to perform per fuzz test (Default: {DEFAULT_EDIT_COUNT})")
+    )]
     pub edits: Option<usize>,
-    /// Number of fuzzing iterations to run per test
-    #[arg(long)]
+    #[arg(
+        long,
+        help=format!("Number of fuzzing iterations to run per test (Default: {DEFAULT_ITERATION_COUNT})")
+    )]
     pub iterations: Option<usize>,
     /// Only fuzz corpus test cases whose name matches the given regex
     #[arg(long, short)]
@@ -867,10 +871,26 @@ impl Init {
 
             (opts.name.clone(), Some(opts))
         } else {
-            let mut json = serde_json::from_str::<TreeSitterJSON>(
-                &fs::read_to_string(current_dir.join("tree-sitter.json"))
-                    .with_context(|| "Failed to read tree-sitter.json")?,
-            )?;
+            let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
+                .with_context(|| "Failed to read tree-sitter.json")?;
+
+            let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
+            if json.schema.is_none() {
+                json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
+            }
+
+            let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
+            // Write the re-serialized config back, as newly added optional boolean fields
+            // will be included with explicit `false`s rather than implict `null`s
+            if self.update && !old_config.trim().eq(new_config.trim()) {
+                info!("Updating tree-sitter.json");
+                fs::write(
+                    current_dir.join("tree-sitter.json"),
+                    serde_json::to_string_pretty(&json)?,
+                )
+                .with_context(|| "Failed to write tree-sitter.json")?;
+            }
+
             (json.grammars.swap_remove(0).name, None)
         };
 
@@ -955,11 +975,21 @@ impl Build {
         } else {
             let output_path = if let Some(ref path) = self.output {
                 let path = Path::new(path);
-                if path.is_absolute() {
+                let full_path = if path.is_absolute() {
                     path.to_path_buf()
                 } else {
                     current_dir.join(path)
-                }
+                };
+                let parent_path = full_path
+                    .parent()
+                    .context("Output path must have a parent")?;
+                let name = full_path
+                    .file_name()
+                    .context("Ouput path must have a filename")?;
+                fs::create_dir_all(parent_path).context("Failed to create output path")?;
+                let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
+                canon_path.push(name);
+                canon_path
             } else {
                 let file_name = grammar_path
                     .file_stem()
@@ -984,7 +1014,7 @@ impl Build {
 
             loader
                 .compile_parser_at_path(&grammar_path, output_path, flags)
-                .unwrap();
+                .context("Failed to compile parser")?;
         }
         Ok(())
     }
@@ -1323,16 +1353,20 @@ impl Test {
                 self.json_summary,
             )?;
             test_summary.test_num = 1;
+        } else {
+            warn!("Test corpus not found at {}", test_corpus_dir.display());
         }
 
         // Check that all of the queries are valid.
         let query_dir = current_dir.join("queries");
-        check_test(
-            test::check_queries_at_path(language, &query_dir),
-            &test_summary,
-            self.json_summary,
-        )?;
-        test_summary.test_num = 1;
+        if query_dir.is_dir() {
+            check_test(
+                test::check_queries_at_path(language, &query_dir),
+                &test_summary,
+                self.json_summary,
+            )?;
+            test_summary.test_num = 1;
+        }
 
         // Run the syntax highlighting tests.
         let test_highlight_dir = test_dir.join("highlight");
@@ -1622,6 +1656,7 @@ impl Highlight {
         let loader_config = config.get()?;
         loader.find_all_languages(&loader_config)?;
         loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
+        let languages = loader.languages_at_path(current_dir)?;
 
         let cancellation_flag = util::cancel_on_signal();
 
@@ -1702,7 +1737,6 @@ impl Highlight {
             } => {
                 let path = get_tmp_source_file(&contents)?;
 
-                let languages = loader.languages_at_path(current_dir)?;
                 let language = languages
                     .iter()
                     .find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
@@ -1733,7 +1767,6 @@ impl Highlight {
                     if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
                         (l, lc)
                     } else {
-                        let languages = loader.languages_at_path(current_dir)?;
                         let language = languages
                             .first()
                             .map(|(l, _)| l.clone())

crates/cli/src/parse.rs

@@ -306,13 +306,13 @@ pub fn parse_file_at_path(
                 }
                 writeln!(&mut io::stderr(), "{message}").unwrap();
             } else {
+                #[rustfmt::skip]
                 let colors = &[
-                    AnsiColor::White,
-                    AnsiColor::Red,
-                    AnsiColor::Blue,
-                    AnsiColor::Green,
-                    AnsiColor::Cyan,
-                    AnsiColor::Yellow,
+                    AnsiColor::White, AnsiColor::Red, AnsiColor::Blue, AnsiColor::Green,
+                    AnsiColor::Cyan, AnsiColor::Yellow, AnsiColor::Magenta,
+                    AnsiColor::BrightWhite, AnsiColor::BrightRed, AnsiColor::BrightBlue,
+                    AnsiColor::BrightGreen, AnsiColor::BrightCyan, AnsiColor::BrightYellow,
+                    AnsiColor::BrightMagenta,
                 ];
                 if message.starts_with("process version:") {
                     let comma_idx = message.find(',').unwrap();
@@ -321,7 +321,7 @@ pub fn parse_file_at_path(
                         .unwrap();
                 }
                 let color = if use_color {
-                    Some(colors[curr_version])
+                    Some(colors[curr_version % colors.len()])
                 } else {
                     None
                 };
@@ -433,7 +433,7 @@ pub fn parse_file_at_path(
     let parse_duration = parse_time.elapsed();
 
     let stdout = io::stdout();
-    let mut stdout = stdout.lock();
+    let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
 
     if let Some(mut tree) = tree {
         if opts.debug_graph && !opts.edits.is_empty() {
@@ -510,12 +510,11 @@ pub fn parse_file_at_path(
                 }
             }
             cursor.reset(tree.root_node());
-            println!();
+            writeln!(&mut stdout)?;
         }
 
         if opts.output == ParseOutput::Cst {
             render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
-            println!();
         }
 
         if opts.output == ParseOutput::Xml {
@@ -582,11 +581,11 @@ pub fn parse_file_at_path(
                         }
                         let start = node.start_position();
                         let end = node.end_position();
-                        write!(&mut stdout, " srow=\"{}\"", start.row)?;
-                        write!(&mut stdout, " scol=\"{}\"", start.column)?;
-                        write!(&mut stdout, " erow=\"{}\"", end.row)?;
-                        write!(&mut stdout, " ecol=\"{}\"", end.column)?;
-                        write!(&mut stdout, ">")?;
+                        write!(
+                            &mut stdout,
+                            " srow=\"{}\" scol=\"{}\" erow=\"{}\" ecol=\"{}\">",
+                            start.row, start.column, end.row, end.column
+                        )?;
                         tags.push(node.kind());
                         needs_newline = true;
                     }
@@ -782,10 +781,14 @@ pub fn render_cst<'a, 'b: 'a>(
     let total_width = lossy_source_code
         .lines()
         .enumerate()
-        .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
+        .map(|(row, col)| {
+            row.checked_ilog10().unwrap_or(0) as usize
+                + col.len().checked_ilog10().unwrap_or(0) as usize
+                + 1
+        })
         .max()
         .unwrap_or(1);
-    let mut indent_level = 1;
+    let mut indent_level = usize::from(!opts.no_ranges);
     let mut did_visit_children = false;
     let mut in_error = false;
     loop {
@@ -883,35 +886,24 @@ fn write_node_text(
                     0
                 };
             let formatted_line = render_line_feed(line, opts);
-            if !opts.no_ranges {
-                write!(
-                    out,
-                    "{}{}{}{}{}{}",
-                    if multiline { "\n" } else { "" },
-                    if multiline {
-                        render_node_range(opts, cursor, is_named, true, total_width, node_range)
-                    } else {
-                        String::new()
-                    },
-                    if multiline {
-                        "  ".repeat(indent_level + 1)
-                    } else {
-                        String::new()
-                    },
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            } else {
-                write!(
-                    out,
-                    "\n{}{}{}{}",
-                    "  ".repeat(indent_level + 1),
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            }
+            write!(
+                out,
+                "{}{}{}{}{}{}",
+                if multiline { "\n" } else { " " },
+                if multiline && !opts.no_ranges {
+                    render_node_range(opts, cursor, is_named, true, total_width, node_range)
+                } else {
+                    String::new()
+                },
+                if multiline {
+                    "  ".repeat(indent_level + 1)
+                } else {
+                    String::new()
+                },
+                paint(quote_color, &String::from(quote)),
+                paint(color, &render_node_text(&formatted_line)),
+                paint(quote_color, &String::from(quote)),
+            )?;
         }
     }
 
@@ -935,30 +927,27 @@ fn render_node_range(
     range: Range,
 ) -> String {
     let has_field_name = cursor.field_name().is_some();
+    let start = range.start_point;
+    let end = range.end_point;
     let range_color = if is_named && !is_multiline && !has_field_name {
         opts.parse_theme.row_color_named
     } else {
         opts.parse_theme.row_color
     };
 
-    let remaining_width_start = (total_width
-        - (range.start_point.row as f64).log10() as usize
-        - (range.start_point.column as f64).log10() as usize)
-        .max(1);
-    let remaining_width_end = (total_width
-        - (range.end_point.row as f64).log10() as usize
-        - (range.end_point.column as f64).log10() as usize)
-        .max(1);
+    let remaining_width = |row: usize, col: usize| {
+        (total_width
+            .saturating_sub(row.checked_ilog10().unwrap_or(0) as usize)
+            .saturating_sub(col.checked_ilog10().unwrap_or(0) as usize))
+        .max(1)
+    };
+    let remaining_width_start = remaining_width(start.row, start.column);
+    let remaining_width_end = remaining_width(end.row, end.column);
     paint(
         range_color,
         &format!(
             "{}:{}{:remaining_width_start$}- {}:{}{:remaining_width_end$}",
-            range.start_point.row,
-            range.start_point.column,
-            ' ',
-            range.end_point.row,
-            range.end_point.column,
-            ' ',
+            start.row, start.column, ' ', end.row, end.column, ' ',
         ),
     )
 }
@@ -1011,10 +1000,9 @@ fn cst_render_node(
         } else {
             opts.parse_theme.node_kind
         };
-        write!(out, "{}", paint(kind_color, node.kind()),)?;
+        write!(out, "{}", paint(kind_color, node.kind()))?;
 
         if node.child_count() == 0 {
-            write!(out, " ")?;
             // Node text from a pattern or external scanner
             write_node_text(
                 opts,

crates/cli/src/query.rs

@@ -37,7 +37,7 @@ pub fn query_file_at_path(
     test_summary: Option<&mut TestSummary>,
 ) -> Result<()> {
     let stdout = io::stdout();
-    let mut stdout = stdout.lock();
+    let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
 
     let query_source = fs::read_to_string(query_path)
         .with_context(|| format!("Error reading query file {}", query_path.display()))?;

crates/cli/src/templates/makefile

@@ -1,6 +1,7 @@
 LANGUAGE_NAME := tree-sitter-KEBAB_PARSER_NAME
 HOMEPAGE_URL := PARSER_URL
 VERSION := PARSER_VERSION
+DESCRIPTION := PARSER_DESCRIPTION
 
 # repository
 SRC_DIR := src

crates/cli/src/templates/setup.py

@@ -32,7 +32,7 @@ class BuildExt(build_ext):
 class BdistWheel(bdist_wheel):
     def get_tag(self):
         python, abi, platform = super().get_tag()
-        if python.startswith("cp"):
+        if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
             python, abi = "cp310", "abi3"
         return python, abi, platform
 

crates/cli/src/test.rs

@@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
             render_assertion_results("queries", &self.query_results)?;
         }
 
+        write!(f, "{}", self.parse_stats)?;
+
         Ok(())
     }
 }
@@ -605,11 +607,13 @@ pub fn run_tests_at_path(
     test_summary: &mut TestSummary,
 ) -> Result<()> {
     let test_entry = parse_tests(&opts.path)?;
-    let mut _log_session = None;
 
-    if opts.debug_graph {
-        _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?);
-    } else if opts.debug {
+    let _log_session = if opts.debug_graph {
+        Some(util::log_graphs(parser, "log.html", opts.open_log)?)
+    } else {
+        None
+    };
+    if opts.debug {
         parser.set_logger(Some(Box::new(|log_type, message| {
             if log_type == LogType::Lex {
                 io::stderr().write_all(b"  ").unwrap();
@@ -642,22 +646,20 @@ pub fn run_tests_at_path(
 }
 
 pub fn check_queries_at_path(language: &Language, path: &Path) -> Result<()> {
-    if path.exists() {
-        for entry in WalkDir::new(path)
-            .into_iter()
-            .filter_map(std::result::Result::ok)
-            .filter(|e| {
-                e.file_type().is_file()
-                    && e.path().extension().and_then(OsStr::to_str) == Some("scm")
-                    && !e.path().starts_with(".")
-            })
-        {
-            let filepath = entry.file_name().to_str().unwrap_or("");
-            let content = fs::read_to_string(entry.path())
-                .with_context(|| format!("Error reading query file {filepath:?}"))?;
-            Query::new(language, &content)
-                .with_context(|| format!("Error in query file {filepath:?}"))?;
-        }
+    for entry in WalkDir::new(path)
+        .into_iter()
+        .filter_map(std::result::Result::ok)
+        .filter(|e| {
+            e.file_type().is_file()
+                && e.path().extension().and_then(OsStr::to_str) == Some("scm")
+                && !e.path().starts_with(".")
+        })
+    {
+        let filepath = entry.file_name().to_str().unwrap_or("");
+        let content = fs::read_to_string(entry.path())
+            .with_context(|| format!("Error reading query file {filepath:?}"))?;
+        Query::new(language, &content)
+            .with_context(|| format!("Error in query file {filepath:?}"))?;
     }
     Ok(())
 }
@@ -867,13 +869,14 @@ fn run_tests(
                 let tree = parser.parse(&input, None).unwrap();
                 let parse_rate = {
                     let parse_time = start.elapsed();
-                    let true_parse_rate = tree.root_node().byte_range().len() as f64
-                        / (parse_time.as_nanos() as f64 / 1_000_000.0);
+                    let byte_len = tree.root_node().byte_range().len();
+                    let true_parse_rate =
+                        byte_len as f64 / (parse_time.as_nanos() as f64 / 1_000_000.0);
                     let adj_parse_rate = adjusted_parse_rate(&tree, parse_time);
 
                     test_summary.parse_stats.total_parses += 1;
                     test_summary.parse_stats.total_duration += parse_time;
-                    test_summary.parse_stats.total_bytes += tree.root_node().byte_range().len();
+                    test_summary.parse_stats.total_bytes += byte_len;
 
                     Some((true_parse_rate, adj_parse_rate))
                 };

crates/cli/src/test_highlight.rs

@@ -13,10 +13,10 @@ use crate::{
 
 #[derive(Debug)]
 pub struct Failure {
-    row: usize,
-    column: usize,
-    expected_highlight: String,
-    actual_highlights: Vec<String>,
+    pub(crate) row: usize,
+    pub(crate) column: usize,
+    pub(crate) expected_highlight: String,
+    pub(crate) actual_highlights: Vec<String>,
 }
 
 impl std::error::Error for Failure {}
@@ -126,6 +126,7 @@ pub fn test_highlights(
         Ok(())
     }
 }
+
 pub fn iterate_assertions(
     assertions: &[Assertion],
     highlights: &[(Utf8Point, Utf8Point, Highlight)],
@@ -142,49 +143,48 @@ pub fn iterate_assertions(
         expected_capture_name: expected_highlight,
     } in assertions
     {
-        let mut passed = false;
-        let mut end_column = position.column + length - 1;
+        // Iterate through all of the highlights that start at or before this assertion's
+        // position, looking for one that matches the assertion.
         actual_highlights.clear();
-
-        // The assertions are ordered by position, so skip past all of the highlights that
-        // end at or before this assertion's position.
-        'highlight_loop: while let Some(highlight) = highlights.get(i) {
+        let mut passed = false;
+        let end_column = position.column + length - 1;
+        for highlight in &highlights[i..] {
+            // The assertions are ordered by position, so skip past all of the highlights that
+            // end at or before this assertion's position.
             if highlight.1 <= *position {
                 i += 1;
                 continue;
             }
+            if (highlight.0.row > position.row)
+                || (highlight.0.row == position.row && highlight.0.column > end_column)
+            {
+                break;
+            }
 
-            // Iterate through all of the highlights that start at or before this assertion's
-            // position, looking for one that matches the assertion.
-            let mut j = i;
-            while let (false, Some(highlight)) = (passed, highlights.get(j)) {
-                end_column = position.column + length - 1;
-                if highlight.0.row >= position.row && highlight.0.column > end_column {
-                    break 'highlight_loop;
-                }
-
-                // If the highlight matches the assertion, or if the highlight doesn't
-                // match the assertion but it's negative, this test passes. Otherwise,
-                // add this highlight to the list of actual highlights that span the
-                // assertion's position, in order to generate an error message in the event
-                // of a failure.
-                let highlight_name = &highlight_names[(highlight.2).0];
-                if (*highlight_name == *expected_highlight) == *negative {
-                    actual_highlights.push(highlight_name);
-                } else {
-                    passed = true;
-                    break 'highlight_loop;
-                }
-
-                j += 1;
+            // If the highlight matches the assertion, or if the highlight doesn't
+            // match the assertion but it's negative, this test passes. Otherwise,
+            // add this highlight to the list of actual highlights that span the
+            // assertion's position, in order to generate an error message in the event
+            // of a failure.
+            let highlight_name = &highlight_names[(highlight.2).0];
+            if (*highlight_name == *expected_highlight) == *negative {
+                actual_highlights.push(highlight_name);
+            } else {
+                passed = true;
+                break;
             }
         }
 
         if !passed {
+            let mut expected = String::with_capacity(expected_highlight.len() + 1);
+            if *negative {
+                expected.push('!');
+            }
+            expected.push_str(expected_highlight);
             return Err(Failure {
                 row: position.row,
                 column: end_column,
-                expected_highlight: expected_highlight.clone(),
+                expected_highlight: expected,
                 actual_highlights: actual_highlights.into_iter().cloned().collect(),
             }
             .into());

crates/cli/src/tests/corpus_test.rs

@@ -274,7 +274,7 @@ pub fn test_language_corpus(
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
+                    println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 
@@ -393,12 +393,12 @@ fn test_feature_corpus_files() {
                     failure_count += 1;
                 }
             } else {
-                eprintln!("Expected error message but got none for test grammar '{language_name}'",);
+                eprintln!("Expected error message but got none for test grammar '{language_name}'");
                 failure_count += 1;
             }
         } else {
             if let Err(e) = &generate_result {
-                eprintln!("Unexpected error for test grammar '{language_name}':\n{e}",);
+                eprintln!("Unexpected error for test grammar '{language_name}':\n{e}");
                 failure_count += 1;
                 continue;
             }

crates/cli/src/tests/detect_language.rs

@@ -127,6 +127,52 @@ fn detect_language_by_double_barrel_file_extension() {
     );
 }
 
+#[test]
+fn detect_language_with_dots_in_filename() {
+    let blade_dir = tree_sitter_dir(
+        r#"{
+  "grammars": [
+    {
+      "name": "blade_dots",
+      "path": ".",
+      "scope": "source.blade",
+      "file-types": [
+        "blade.php"
+      ]
+    },
+    {
+      "name": "php_dots",
+      "path": ".",
+      "scope": "source.php",
+      "file-types": [
+        "php"
+      ]
+    }
+  ],
+  "metadata": {
+    "version": "0.0.1"
+  }
+}
+"#,
+        "blade_dots",
+    );
+
+    let mut loader = Loader::with_parser_lib_path(scratch_dir().to_path_buf());
+    let config = loader
+        .find_language_configurations_at_path(blade_dir.path(), false)
+        .unwrap();
+
+    // this is just to validate that we can read the tree-sitter.json correctly
+    assert_eq!(config[0].scope.as_ref().unwrap(), "source.blade");
+
+    let file_name = blade_dir.path().join("foo.bar.baz.blade.php");
+    fs::write(&file_name, "").unwrap();
+    assert_eq!(
+        get_lang_scope(&loader, &file_name),
+        Some("source.blade".into())
+    );
+}
+
 #[test]
 fn detect_language_without_filename() {
     let gitignore_dir = tree_sitter_dir(

crates/cli/src/tests/parser_test.rs

@@ -249,6 +249,64 @@ fn test_parsing_with_custom_utf16_be_input() {
     assert_eq!(root.child(0).unwrap().kind(), "function_item");
 }
 
+#[test]
+fn test_utf16_decode_does_not_read_oob() {
+    // Test for a buffer over-read in ts_decode_utf16_le/be when a lead surrogate
+    // is the last code unit in a chunk. The test grammar's external scanner
+    // distinguishes surrogate code points from supplementary-plane characters,
+    // making the over-read directly observable in the parse tree.
+    //
+    // Buffer layout:
+    //   buf[0] = 0xD83E  (lead surrogate)
+    //   buf[1] = 0xDD8B  (POISON: fake trail surrogate, adjacent in memory)
+    //
+    // The callback returns only buf[0..1] (one code unit = 2 bytes).
+    //
+    // When functioning correctly, this test passes a length of 2 bytes, which is
+    // interpreted as 2/2 = 1 code unit, and thus doesn't over-read into the "poison"
+    // fake trail surrogate. If an over-read does occur, the scanner sees a
+    // supplementary token.
+    let mut parser = Parser::new();
+    let language = get_test_fixture_language("utf16_surrogate_oob");
+    parser.set_language(&language).unwrap();
+
+    let buf = vec![
+        0xD83E, // lead surrogate (the only "visible" code unit)
+        0xDD8B, // POISON: adjacent in Vec memory, past the chunk
+    ];
+    assert_eq!("🦋", String::from_utf16(&buf).unwrap());
+
+    let mut callback = |offset: usize, _position: Point| -> &[u16] {
+        // only expose buf[0], never buf[1]
+        if offset >= 1 {
+            return [].as_slice();
+        }
+        &buf[0..1]
+    };
+
+    // Use the parse function matching the host endianness, since the
+    // buffer contains native u16 values.
+    #[cfg(target_endian = "little")]
+    let tree = parser
+        .parse_utf16_le_with_options(&mut callback, None, None)
+        .unwrap();
+    #[cfg(target_endian = "big")]
+    let tree = parser
+        .parse_utf16_be_with_options(&mut callback, None, None)
+        .unwrap();
+
+    let root = tree.root_node();
+
+    // Correct: scanner sees raw surrogate (0xD83E) -> `surrogate` node
+    // Incorrect: scanner sees supplementary (U+1F98B, aka 🦋) -> `supplementary` node
+    assert_eq!(
+        root.to_sexp(),
+        "(program (surrogate))",
+        "buffer over-read: decoder read past chunk boundary and formed a \
+         supplementary character from OOB adjacent memory"
+    );
+}
+
 #[test]
 fn test_parsing_with_callback_returning_owned_strings() {
     let mut parser = Parser::new();

crates/cli/src/tests/query_test.rs

@@ -1592,6 +1592,148 @@ fn test_query_matches_with_leading_zero_or_more_repeated_leaf_nodes() {
     });
 }
 
+#[test]
+fn test_matches_with_anchor_sibling_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[(0, vec![("name", "a")]), (0, vec![("name", "b")])],
+        );
+    });
+}
+
+#[test]
+fn test_matches_with_anchor_sibling_with_quantifier_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)+
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[(0, vec![("name", "a")]), (0, vec![("name", "b")])],
+        );
+    });
+}
+
+#[test]
+fn test_matches_with_anchor_sibling_with_quantifier_captured_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)+ @doc
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[
+                (0, vec![("doc", "// A"), ("name", "a")]),
+                (0, vec![("doc", "// B"), ("name", "b")]),
+            ],
+        );
+    });
+}
+
+#[test]
+fn test_matches_anchored_quantified_sibling_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(translation_unit (comment)* @comment . (declaration) @decl)",
+        )
+        .unwrap();
+        assert_query_matches(
+            &language,
+            &query,
+            "
+void foo() {}
+
+// this one has
+// two comments
+extern int baz;
+
+// this one has a comment
+extern int bar;
+",
+            &[
+                (
+                    0,
+                    vec![
+                        ("comment", "// this one has"),
+                        ("comment", "// two comments"),
+                        ("decl", "extern int baz;"),
+                    ],
+                ),
+                (
+                    0,
+                    vec![
+                        ("comment", "// this one has a comment"),
+                        ("decl", "extern int bar;"),
+                    ],
+                ),
+            ],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_trailing_optional_nodes() {
     allocations::record(|| {
@@ -3075,6 +3217,74 @@ fn test_query_matches_with_deeply_nested_patterns_with_fields() {
     });
 }
 
+#[test]
+fn test_query_alternation_with_inner_quantifier() {
+    let language = get_language("c");
+    let source_code = "#include <foo>
+#include <bar>
+#include <baz>
+
+// comment";
+    let matches = &[
+        (
+            0,
+            vec![
+                ("capture", "#include <foo>\n"),
+                ("capture", "#include <bar>\n"),
+                ("capture", "#include <baz>\n"),
+            ],
+        ),
+        (0, vec![("capture", "// comment")]),
+    ];
+
+    let query = "[
+       (preproc_include)+
+       (comment)
+    ] @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+
+    let query = "[
+       (comment)
+       (preproc_include)+
+    ] @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+}
+
+#[test]
+fn test_query_alternation_with_outer_quantifier() {
+    let language = get_language("c");
+    let source_code = "#include <foo>
+#include <bar>
+#include <baz>
+
+// comment";
+    let matches = &[(
+        0,
+        vec![
+            ("capture", "#include <foo>\n"),
+            ("capture", "#include <bar>\n"),
+            ("capture", "#include <baz>\n"),
+            ("capture", "// comment"),
+        ],
+    )];
+
+    let query = "[
+        (preproc_include)
+        (comment)
+    ]+ @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+
+    let query = "([
+        (preproc_include)
+        (comment)
+    ] (_)?)+ @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+}
+
 #[test]
 fn test_query_matches_with_alternations_and_predicates() {
     allocations::record(|| {

crates/cli/src/tests/test_highlight_test.rs

@@ -4,7 +4,7 @@ use tree_sitter_highlight::{Highlight, Highlighter};
 use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
 use crate::{
     query_testing::{parse_position_comments, Assertion, Utf8Point},
-    test_highlight::get_highlight_positions,
+    test_highlight::{get_highlight_positions, iterate_assertions, Failure},
 };
 
 #[test]
@@ -68,3 +68,195 @@ fn test_highlight_test_with_basic_test() {
         ]
     );
 }
+
+#[test]
+fn test_assertion_with_non_matching_highlight_at_same_position() {
+    // Test that an assertion fails when the highlight at the position does not match
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![Assertion::new(1, 0, 1, false, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(1, 0), Utf8Point::new(1, 5), Highlight(1)), // "variable" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 1);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, vec!["variable".to_string()]);
+}
+
+#[test]
+fn test_assertion_with_exact_matching_highlight() {
+    // Test exact match: assertion and highlight have same start and end
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 5, 3, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_contained_within_highlight() {
+    // Test where assertion is fully contained within a larger highlight
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 3, 2, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 0), Utf8Point::new(0, 10), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_overlapping_highlight_start() {
+    // Test where assertion starts before highlight but overlaps with it
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 3, 4, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 5), Utf8Point::new(0, 10), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_with_no_highlights() {
+    // Test that an assertion fails when there are no highlights at all
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, false, String::from("keyword"))];
+    let highlights = vec![];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}
+
+#[test]
+fn test_assertion_with_highlight_ending_before() {
+    // Test where highlight ends before the assertion starts
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 10, 1, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 10);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}
+
+#[test]
+fn test_negative_assertion_with_non_matching_highlight() {
+    // Test that a negative assertion passes when the specified highlight is NOT present
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, true, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(1)), // "variable" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_negative_assertion_with_matching_highlight() {
+    // Test that a negative assertion fails when the specified highlight IS present
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, true, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(0)), // "keyword" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "!keyword");
+    assert_eq!(err.actual_highlights, vec!["keyword".to_string()]);
+}
+
+#[test]
+fn test_multiple_assertions_sequential() {
+    // Test multiple assertions in sequence with non-overlapping highlights
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![
+        Assertion::new(0, 0, 3, false, String::from("keyword")),
+        Assertion::new(0, 10, 1, false, String::from("variable")),
+    ];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 3), Highlight(0)), // "keyword"
+        (Utf8Point::new(0, 10), Utf8Point::new(0, 11), Highlight(1)), // "variable"
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 2);
+}
+
+#[test]
+fn test_multiple_highlights_at_same_position() {
+    // Test where multiple highlights overlap at the assertion position
+    let highlight_names = vec![
+        "keyword".to_string(),
+        "variable".to_string(),
+        "function".to_string(),
+    ];
+    let assertions = vec![Assertion::new(0, 5, 1, false, String::from("variable"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 10), Highlight(0)), // "keyword" spans entire range
+        (Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(1)), // "variable" at assertion position
+        (Utf8Point::new(0, 7), Utf8Point::new(0, 12), Highlight(2)), // "function" overlaps
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertions_across_multiple_rows() {
+    // Test assertions on different rows
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![
+        Assertion::new(0, 5, 3, false, String::from("keyword")),
+        Assertion::new(2, 10, 1, false, String::from("variable")),
+    ];
+    let highlights = vec![
+        (Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(0)), // "keyword" on row 0
+        (Utf8Point::new(2, 10), Utf8Point::new(2, 11), Highlight(1)), // "variable" on row 2
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 2);
+}
+
+#[test]
+fn test_assertion_should_not_match_highlight_on_later_row() {
+    // Test logic for early exit when highlight is on a later row than the assertion
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 5, 3, false, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(1, 0), Utf8Point::new(1, 5), Highlight(0)), // wrong row
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 7); // end_column
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}

crates/cli/src/tests/tree_test.rs

@@ -795,3 +795,44 @@ fn get_changed_ranges(
     *tree = new_tree;
     result
 }
+
+// Regression test for an incremental reparse bug where an external
+// scanner's choice depends on lexer->eof() (and thus on the parser's
+// current included ranges). The cached token at byte 0 was emitted when
+// the included range stopped just after the opener. Widening the range
+// to include a later matching delimiter must invalidate that cached
+// token so the scanner re-runs and emits the open form instead of the
+// unclosed form.
+#[test]
+fn test_reuse_invalidates_scanner_token_when_included_range_expands() {
+    let language = get_test_fixture_language("external_lookahead_eof_boundary");
+    let mut parser = Parser::new();
+    parser.set_language(&language).unwrap();
+
+    let source = "``";
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 1,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 1),
+        }])
+        .unwrap();
+    let tree1 = parser.parse(source, None).unwrap();
+    assert_eq!(tree1.root_node().to_sexp(), "(document (unclosed_delim))");
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 2,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 2),
+        }])
+        .unwrap();
+    let tree2 = parser.parse(source, Some(&tree1)).unwrap();
+    assert_eq!(
+        tree2.root_node().to_sexp(),
+        "(document (span (open_delim) (close_delim)))"
+    );
+}

crates/cli/src/tests/wasm_language_test.rs

@@ -118,6 +118,32 @@ fn test_load_fixture_language_wasm() {
     });
 }
 
+#[test]
+fn test_wasm_realloc_smaller_size() {
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let mut parser = Parser::new();
+        let language = get_test_fixture_language_wasm("wasm_realloc_overflow_heap");
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse("hello", None).unwrap();
+        assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
+    });
+}
+
+#[test]
+fn test_wasm_realloc_clobber_region() {
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let mut parser = Parser::new();
+        let language = get_test_fixture_language_wasm("wasm_realloc_clobber_region");
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse("hello", None).unwrap();
+        assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
+    });
+}
+
 #[test]
 fn test_load_multiple_wasm_languages() {
     allocations::record(|| {
@@ -273,6 +299,55 @@ fn test_load_wasm_errors() {
     });
 }
 
+#[test]
+fn test_load_wasm_language_with_reserved_words() {
+    // This test exercises a grammar with multiple reserved word sets loaded via WASM.
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let language = get_test_fixture_language_wasm("reserved_words");
+
+        let mut parser = Parser::new();
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+
+        // "if" and "while" are globally reserved, so using them as identifiers
+        // should produce an error recovery.
+        let tree = parser
+            .parse("var a =\n\nif (something) {\n  c();\n}", None)
+            .unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            concat!(
+                "(program ",
+                "(ERROR (identifier)) ",
+                "(if_statement (parenthesized_expression (identifier)) ",
+                "(block (expression_statement (call_expression (identifier))))))",
+            )
+        );
+
+        // "if" and "while" are NOT reserved in the 'property' context, so they
+        // can appear as object keys without error.
+        let tree = parser
+            .parse("var x = {\n  if: a,\n  while: b,\n};", None)
+            .unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            concat!(
+                "(program (var_declaration (identifier) (object ",
+                "(pair (identifier) (identifier)) (pair (identifier) (identifier)))))"
+            )
+        );
+
+        // "var" IS reserved in the 'property' context, so using it as a property
+        // key triggers error recovery.
+        let tree = parser.parse("var x = {\nvar y = z;", None).unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            "(program (ERROR (identifier)) (var_declaration (identifier) (identifier)))"
+        );
+    });
+}
+
 #[test]
 fn test_wasm_oom() {
     allocations::record(|| {

crates/cli/src/version.rs

@@ -292,6 +292,9 @@ impl Version {
         } else {
             self.current_dir.join("Makefile")
         };
+        if !makefile_path.exists() {
+            return Ok(());
+        }
 
         self.update_file_with(&makefile_path, |content| {
             content

crates/cli/src/wasm.rs

@@ -15,7 +15,7 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)>
         .unwrap();
     let wasm_filename = format!("tree-sitter-{grammar_name}.wasm");
     let contents = fs::read(language_dir.join(&wasm_filename)).with_context(|| {
-        format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.",)
+        format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.")
     })?;
     Ok((grammar_name, contents))
 }

crates/generate/src/build_tables/minimize_parse_table.rs

@@ -349,7 +349,7 @@ impl Minimizer<'_> {
         new_token: Symbol,
     ) -> bool {
         if new_token == Symbol::end_of_nonterminal_extra() {
-            debug!("split states {left_id} {right_id} - end of non-terminal extra",);
+            debug!("split states {left_id} {right_id} - end of non-terminal extra");
             return true;
         }
 

crates/generate/src/dsl.js

@@ -34,6 +34,9 @@ function blank() {
 }
 
 function field(name, rule) {
+  if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`);
+  }
   return {
     type: "FIELD",
     name,

crates/generate/src/generate.rs

@@ -8,6 +8,7 @@ use std::{
 };
 
 use bitflags::bitflags;
+#[cfg(feature = "load")]
 use log::warn;
 use node_types::VariableInfo;
 use regex::{Regex, RegexBuilder};
@@ -41,7 +42,7 @@ pub use parse_grammar::ParseGrammarError;
 use prepare_grammar::prepare_grammar;
 pub use prepare_grammar::PrepareGrammarError;
 use render::render_c_code;
-pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
+pub use render::{RenderError, ABI_VERSION_MAX, ABI_VERSION_MIN};
 
 static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
     RegexBuilder::new("^\\s*//.*")
@@ -93,6 +94,8 @@ pub enum GenerateError {
     VariableInfo(#[from] VariableInfoError),
     #[error(transparent)]
     BuildTables(#[from] ParseTableBuilderError),
+    #[error(transparent)]
+    Render(#[from] RenderError),
     #[cfg(feature = "load")]
     #[error(transparent)]
     ParseVersion(#[from] ParseVersionError),
@@ -106,6 +109,7 @@ pub struct IoError {
     pub path: Option<String>,
 }
 
+#[cfg(feature = "load")]
 impl IoError {
     fn new(error: &std::io::Error, path: Option<&Path>) -> Self {
         Self {
@@ -333,7 +337,7 @@ pub fn generate_parser_for_grammar(
         LANGUAGE_VERSION,
         semantic_version,
         None,
-        OptLevel::empty(),
+        OptLevel::default(),
     )?;
     Ok((input_grammar.name, parser.c_code))
 }
@@ -398,7 +402,7 @@ fn generate_parser_for_grammar_with_opts(
         abi_version,
         semantic_version,
         supertype_symbol_map,
-    );
+    )?;
     Ok(GeneratedParser {
         c_code,
         #[cfg(feature = "load")]

crates/generate/src/node_types.rs

@@ -1,4 +1,6 @@
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+#[cfg(feature = "load")]
+use std::collections::HashSet;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 
 use serde::Serialize;
 use thiserror::Error;
@@ -584,7 +586,13 @@ pub fn generate_node_types_json(
                 kind: node_type_json.kind.clone(),
                 named: true,
             };
-            subtype_map.push((supertype, subtypes.clone()));
+
+            // We only add to the subtype map if there are visible subtypes.
+            // A supertype may have zero subtypes if its children are all
+            // hidden (e.g., wrapping a hidden external token).
+            if !subtypes.is_empty() {
+                subtype_map.push((supertype, subtypes.clone()));
+            }
             node_type_json.subtypes = Some(subtypes);
         } else if !syntax_grammar.variables_to_inline.contains(&symbol) {
             // If a rule is aliased under multiple names, then its information
@@ -1256,6 +1264,49 @@ mod tests {
         );
     }
 
+    /// A supertype whose only child is a hidden external token
+    /// xgust not cause generation to panic. The subtype map must
+    /// skip entries with empty subtypes to avoid a lookup failure
+    /// in the topological sort.
+    #[test]
+    fn test_node_types_supertype_with_only_hidden_child() {
+        let node_types = get_node_types(&InputGrammar {
+            supertype_symbols: vec!["_type_a".to_string(), "_type_b".to_string()],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::seq(vec![Rule::named("_type_a"), Rule::named("_type_b")]),
+                },
+                // Supertype A: a normal choice of named subtypes
+                Variable {
+                    name: "_type_a".to_string(),
+                    kind: VariableType::Hidden,
+                    rule: Rule::choice(vec![Rule::named("v2"), Rule::named("v3")]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                Variable {
+                    name: "v3".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("y"),
+                },
+                // Supertype B: a hidden external token with no subtypes
+                Variable {
+                    name: "_type_b".to_string(),
+                    kind: VariableType::Hidden,
+                    rule: Rule::external(0),
+                },
+            ],
+            external_tokens: vec![Rule::named("_hidden_ext")],
+            ..Default::default()
+        });
+        assert!(node_types.is_ok());
+    }
+
     #[test]
     fn test_node_types_for_children_without_fields() {
         let node_types = get_node_types(&InputGrammar {

crates/generate/src/parse_grammar.rs

@@ -1,7 +1,6 @@
-use std::collections::HashSet;
-
 use log::warn;
 use regex::Regex;
+use rustc_hash::{FxHashMap, FxHashSet};
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 use thiserror::Error;
@@ -149,51 +148,150 @@ fn rule_is_referenced(rule: &Rule, target: &str, is_external: bool) -> bool {
     }
 }
 
-fn variable_is_used(
-    grammar_rules: &[(String, Rule)],
-    extras: &[Rule],
-    externals: &[Rule],
-    target_name: &str,
-    in_progress: &mut HashSet<String>,
-) -> bool {
-    let root = &grammar_rules.first().unwrap().0;
-    if target_name == root {
-        return true;
-    }
-
-    if extras
-        .iter()
-        .any(|rule| rule_is_referenced(rule, target_name, false))
-    {
-        return true;
-    }
-
-    if externals
-        .iter()
-        .any(|rule| rule_is_referenced(rule, target_name, true))
-    {
-        return true;
-    }
-
-    in_progress.insert(target_name.to_string());
-    let result = grammar_rules
-        .iter()
-        .filter(|(key, _)| *key != target_name)
-        .any(|(name, rule)| {
-            if !rule_is_referenced(rule, target_name, false) || in_progress.contains(name) {
-                return false;
+impl InputGrammar {
+    /// Strip unused rules from the grammar and clean up references to them
+    /// in the surrounding config. (conflicts, supertypes, inline, extras,
+    /// externals, precedences).
+    ///
+    /// A variable is "used" if it is the start rule, the word token, named in
+    /// `extras`/`externals`, or transitively reachable via rule references from
+    /// any of the above.
+    fn normalize(mut self) -> Self {
+        // Compute the used set via forward DFS from the implicit roots
+        // (start rule, word_token, refs in extras and externals).
+        //
+        // Extras count their top-level `NamedSymbol` as a use (so naming
+        // a rule directly in `extras` keeps it), but externals do not (the
+        // external entry is the rule itself, not a reference to one).
+        let used: FxHashSet<String> = {
+            let by_name: FxHashMap<&str, &Rule> = self
+                .variables
+                .iter()
+                .map(|v| (v.name.as_str(), &v.rule))
+                .collect();
+            let mut visited: FxHashSet<&str> = FxHashSet::default();
+            let mut stack: Vec<&str> = Vec::new();
+            if let Some(first) = self.variables.first() {
+                stack.push(first.name.as_str());
             }
-            variable_is_used(grammar_rules, extras, externals, name, in_progress)
-        });
-    in_progress.remove(target_name);
+            if let Some(word) = self.word_token.as_deref() {
+                stack.push(word);
+            }
+            for rule in &self.extra_symbols {
+                collect_referenced_names(rule, false, &mut stack);
+            }
+            for rule in &self.external_tokens {
+                collect_referenced_names(rule, true, &mut stack);
+            }
+            // Reserved-word entries are uses of the named rule (the entry
+            // names a token to reserve in some context). Top-level
+            // `NamedSymbol` counts, same as for extras.
+            for ctx in &self.reserved_words {
+                for rule in &ctx.reserved_words {
+                    collect_referenced_names(rule, false, &mut stack);
+                }
+            }
+            while let Some(name) = stack.pop() {
+                if !visited.insert(name) {
+                    continue;
+                }
+                if let Some(rule) = by_name.get(name) {
+                    collect_referenced_names(rule, false, &mut stack);
+                }
+            }
+            visited.into_iter().map(String::from).collect()
+        };
 
-    result
+        for v in &self.variables {
+            if !used.contains(v.name.as_str()) {
+                continue;
+            }
+            if !self
+                .extra_symbols
+                .iter()
+                .any(|r| rule_is_referenced(r, &v.name, false))
+            {
+                continue;
+            }
+            let inner_rule = match &v.rule {
+                Rule::Metadata { rule, .. } => rule.as_ref(),
+                other => other,
+            };
+            let matches_empty = match inner_rule {
+                Rule::String(s) => s.is_empty(),
+                Rule::Pattern(value, _) => Regex::new(value).is_ok_and(|reg| reg.is_match("")),
+                _ => false,
+            };
+            if matches_empty {
+                warn!(
+                    "Named extra rule `{name}` matches the empty string. \
+                     Inline this to avoid infinite loops while parsing.",
+                    name = v.name,
+                );
+            }
+        }
+
+        // Drop unused variables and clean up any references to them in the
+        // surrounding grammar config.
+        let dropped: Vec<String> = self
+            .variables
+            .iter()
+            .filter(|v| !used.contains(v.name.as_str()))
+            .map(|v| v.name.clone())
+            .collect();
+        self.variables.retain(|v| used.contains(v.name.as_str()));
+        for name in &dropped {
+            self.expected_conflicts.retain(|r| !r.contains(name));
+            self.supertype_symbols.retain(|r| r != name);
+            self.variables_to_inline.retain(|r| r != name);
+            self.extra_symbols
+                .retain(|r| !rule_is_referenced(r, name, true));
+            self.external_tokens
+                .retain(|r| !rule_is_referenced(r, name, true));
+            self.precedence_orderings.retain(|r| {
+                !r.iter()
+                    .any(|e| matches!(e, PrecedenceEntry::Symbol(s) if s == name))
+            });
+            // Prune entries but keep the context: an intentionally-empty
+            // reserved-word context is a meaningful marker that rule bodies
+            // may reference by name.
+            for ctx in &mut self.reserved_words {
+                ctx.reserved_words
+                    .retain(|r| !rule_is_referenced(r, name, false));
+            }
+        }
+
+        self
+    }
+}
+
+/// Append every `NamedSymbol` name reachable in `rule` to `out`. If
+/// `skip_top_level` is true, a `NamedSymbol` at the root of `rule` is
+/// ignored (used for externals entries, which name themselves).
+fn collect_referenced_names<'a>(rule: &'a Rule, skip_top_level: bool, out: &mut Vec<&'a str>) {
+    match rule {
+        Rule::NamedSymbol(name) => {
+            if !skip_top_level {
+                out.push(name.as_str());
+            }
+        }
+        Rule::Choice(rules) | Rule::Seq(rules) => {
+            for r in rules {
+                collect_referenced_names(r, false, out);
+            }
+        }
+        Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
+            collect_referenced_names(rule, skip_top_level, out);
+        }
+        Rule::Repeat(inner) => collect_referenced_names(inner, false, out),
+        Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => {}
+    }
 }
 
 pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
-    let mut grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
+    let grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
 
-    let mut extra_symbols =
+    let extra_symbols =
         grammar_json
             .extras
             .into_iter()
@@ -208,7 +306,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
                 ParseGrammarResult::Ok(acc)
             })?;
 
-    let mut external_tokens = grammar_json
+    let external_tokens = grammar_json
         .externals
         .into_iter()
         .map(|e| parse_rule(e, false))
@@ -227,74 +325,17 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         precedence_orderings.push(ordering);
     }
 
-    let mut variables = Vec::with_capacity(grammar_json.rules.len());
-
-    let rules = grammar_json
+    let variables = grammar_json
         .rules
         .into_iter()
-        .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?, false)?)))
-        .collect::<ParseGrammarResult<Vec<_>>>()?;
-
-    let mut in_progress = HashSet::new();
-
-    for (name, rule) in &rules {
-        if grammar_json.word.as_ref().is_none_or(|w| w != name)
-            && !variable_is_used(
-                &rules,
-                &extra_symbols,
-                &external_tokens,
+        .map(|(name, r)| {
+            Ok(Variable {
                 name,
-                &mut in_progress,
-            )
-        {
-            grammar_json.conflicts.retain(|r| !r.contains(name));
-            grammar_json.supertypes.retain(|r| r != name);
-            grammar_json.inline.retain(|r| r != name);
-            extra_symbols.retain(|r| !rule_is_referenced(r, name, true));
-            external_tokens.retain(|r| !rule_is_referenced(r, name, true));
-            precedence_orderings.retain(|r| {
-                !r.iter().any(|e| {
-                    let PrecedenceEntry::Symbol(s) = e else {
-                        return false;
-                    };
-                    s == name
-                })
-            });
-            continue;
-        }
-
-        if extra_symbols
-            .iter()
-            .any(|r| rule_is_referenced(r, name, false))
-        {
-            let inner_rule = if let Rule::Metadata { rule, .. } = rule {
-                rule
-            } else {
-                rule
-            };
-            let matches_empty = match inner_rule {
-                Rule::String(rule_str) => rule_str.is_empty(),
-                Rule::Pattern(ref value, _) => Regex::new(value)
-                    .map(|reg| reg.is_match(""))
-                    .unwrap_or(false),
-                _ => false,
-            };
-            if matches_empty {
-                warn!(
-                    concat!(
-                        "Named extra rule `{}` matches the empty string. ",
-                        "Inline this to avoid infinite loops while parsing."
-                    ),
-                    name
-                );
-            }
-        }
-        variables.push(Variable {
-            name: name.clone(),
-            kind: VariableType::Named,
-            rule: rule.clone(),
-        });
-    }
+                kind: VariableType::Named,
+                rule: parse_rule(serde_json::from_value(r)?, false)?,
+            })
+        })
+        .collect::<ParseGrammarResult<Vec<_>>>()?;
 
     let reserved_words = grammar_json
         .reserved
@@ -315,7 +356,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         })
         .collect::<ParseGrammarResult<Vec<_>>>()?;
 
-    Ok(InputGrammar {
+    let grammar = InputGrammar {
         name: grammar_json.name,
         word_token: grammar_json.word,
         expected_conflicts: grammar_json.conflicts,
@@ -326,7 +367,9 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         extra_symbols,
         external_tokens,
         reserved_words,
-    })
+    }
+    .normalize();
+    Ok(grammar)
 }
 
 fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult<Rule> {
@@ -400,7 +443,7 @@ fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult<Rule> {
         }),
         RuleJSON::TOKEN { content } => parse_rule(*content, true).map(Rule::token),
         RuleJSON::IMMEDIATE_TOKEN { content } => {
-            parse_rule(*content, is_token).map(Rule::immediate_token)
+            parse_rule(*content, true).map(Rule::immediate_token)
         }
     }
 }

crates/generate/src/prepare_grammar/extract_tokens.rs

@@ -21,6 +21,8 @@ unless they are used only as the grammar's start rule.
 "
     )]
     EmptyString(String),
+    #[error("Terminal rule '{0}' cannot be used as a supertype")]
+    SupertypeTerminal(String),
     #[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
     ExternalTokenNonTerminal(String),
     #[error("Non-symbol rules cannot be used as external tokens")]
@@ -128,11 +130,18 @@ pub(super) fn extract_tokens(
         })
         .collect();
 
-    let supertype_symbols = grammar
+    let supertype_symbols: Vec<Symbol> = grammar
         .supertype_symbols
         .into_iter()
         .map(|symbol| symbol_replacer.replace_symbol(symbol))
         .collect();
+    for supertype_symbol in &supertype_symbols {
+        if supertype_symbol.is_terminal() {
+            Err(ExtractTokensError::SupertypeTerminal(
+                lexical_variables[supertype_symbol.index].name.clone(),
+            ))?;
+        }
+    }
 
     let variables_to_inline = grammar
         .variables_to_inline

crates/generate/src/prepare_grammar/process_inlines.rs

@@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
         let production_map = production_indices_by_step_id
             .into_iter()
             .map(|(step_id, production_indices)| {
-                let production = step_id.variable_index.map_or_else(
-                    || &productions[step_id.production_index],
-                    |variable_index| {
-                        &grammar.variables[variable_index].productions[step_id.production_index]
-                    },
-                ) as *const Production;
+                let production =
+                    core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
+                        || &productions[step_id.production_index],
+                        |variable_index| {
+                            &grammar.variables[variable_index].productions[step_id.production_index]
+                        },
+                    ));
                 ((production, step_id.step_index as u32), production_indices)
             })
             .collect();

crates/generate/src/render.rs

@@ -7,6 +7,8 @@ use std::{
 
 use crate::LANGUAGE_VERSION;
 use indoc::indoc;
+use serde::Serialize;
+use thiserror::Error;
 
 use super::{
     build_tables::Tables,
@@ -25,6 +27,16 @@ pub const ABI_VERSION_MIN: usize = 14;
 pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION;
 const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
 
+pub type RenderResult<T> = Result<T, RenderError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum RenderError {
+    #[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
+    ParseTable(usize),
+    #[error("This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {0}")]
+    ABI(usize),
+}
+
 #[clippy::format_args]
 macro_rules! add {
     ($this: tt, $($arg: tt)*) => {{
@@ -104,7 +116,7 @@ struct Metadata {
 }
 
 impl Generator {
-    fn generate(mut self) -> String {
+    fn generate(mut self) -> RenderResult<String> {
         self.init();
         self.add_header();
         self.add_includes();
@@ -161,7 +173,7 @@ impl Generator {
             self.add_reserved_word_sets();
         }
 
-        self.add_parse_table();
+        self.add_parse_table()?;
 
         if !self.syntax_grammar.external_tokens.is_empty() {
             self.add_external_token_enum();
@@ -171,7 +183,7 @@ impl Generator {
 
         self.add_parser_export();
 
-        self.buffer
+        Ok(self.buffer)
     }
 
     fn init(&mut self) {
@@ -325,7 +337,7 @@ impl Generator {
     }
 
     fn add_header(&mut self) {
-        add_line!(self, "/* Automatically @generated by tree-sitter */",);
+        add_line!(self, "/* Automatically @generated by tree-sitter */");
         add_line!(self, "");
     }
 
@@ -1273,7 +1285,7 @@ impl Generator {
         add_line!(self, "");
     }
 
-    fn add_parse_table(&mut self) {
+    fn add_parse_table(&mut self) -> RenderResult<()> {
         let mut parse_table_entries = HashMap::new();
         let mut next_parse_action_list_index = 0;
 
@@ -1443,6 +1455,9 @@ impl Generator {
             add_line!(self, "}};");
             add_line!(self, "");
         }
+        if next_parse_action_list_index >= usize::from(u16::MAX) {
+            Err(RenderError::ParseTable(next_parse_action_list_index))?;
+        }
 
         let mut parse_table_entries = parse_table_entries
             .into_iter()
@@ -1450,6 +1465,8 @@ impl Generator {
             .collect::<Vec<_>>();
         parse_table_entries.sort_by_key(|(index, _)| *index);
         self.add_parse_action_list(parse_table_entries);
+
+        Ok(())
     }
 
     fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
@@ -1942,11 +1959,10 @@ pub fn render_c_code(
     abi_version: usize,
     semantic_version: Option<(u8, u8, u8)>,
     supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
-) -> String {
-    assert!(
-        (ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version),
-        "This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {abi_version}",
-    );
+) -> RenderResult<String> {
+    if !(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version) {
+        Err(RenderError::ABI(abi_version))?;
+    }
 
     Generator {
         language_name: name.to_string(),

crates/generate/src/templates/array.h

@@ -52,67 +52,96 @@ extern "C" {
 
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity) \
-  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity)        \
+  ((self)->contents = _array__reserve(           \
+    (void *)(self)->contents, &(self)->capacity, \
+    array_elem_size(self), new_capacity)         \
+  )
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((Array *)(self))
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                            \
-  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
-   (self)->contents[(self)->size++] = (element))
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__grow(                              \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    );                                                            \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count) \
-  do { \
-    if ((count) == 0) break; \
-    _array__grow((Array *)(self), count, array_elem_size(self)); \
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__grow(                                             \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    );                                                                           \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count); \
+    (self)->size += (count);                                                     \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other)                                       \
+#define array_push_all(self, other) \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, contents)                    \
-  _array__splice(                                               \
-    (Array *)(self), array_elem_size(self), (self)->size, \
-    0, count,  contents                                        \
+#define array_extend(self, count, other_contents)                 \
+  (self)->contents = _array__splice(                              \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
   )
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents)  \
-  _array__splice(                                                       \
-    (Array *)(self), array_elem_size(self), _index,                \
-    old_count, new_count, new_contents                                 \
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  (self)->contents = _array__splice(                                   \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
   )
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element) \
-  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+#define array_insert(self, _index, element)                     \
+  (self)->contents = _array__splice(                            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((Array *)(self), array_elem_size(self), _index)
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other) \
-  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+#define array_assign(self, other)                                   \
+  (self)->contents = _array__assign(                                \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )
 
 /// Swap one array with another
-#define array_swap(self, other) \
-  _array__swap((Array *)(self), (Array *)(other))
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array_swap_tmp;                            \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
+  } while (0)
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +186,90 @@ extern "C" {
 
 // Private
 
-typedef Array(void) Array;
-
-/// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(Array *self) {
-  if (self->contents) {
-    ts_free(self->contents);
-    self->contents = NULL;
-    self->size = 0;
-    self->capacity = 0;
-  }
-}
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(Array *self, size_t element_size,
-                                uint32_t index) {
-  assert(index < self->size);
-  char *contents = (char *)self->contents;
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  assert(index < *size);
+  char *contents = (char *)self_contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (self->size - index - 1) * element_size);
-  self->size--;
+          (*size - index - 1) * element_size);
+  (*size)--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
-  if (new_capacity > self->capacity) {
-    if (self->contents) {
-      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
     } else {
-      self->contents = ts_malloc(new_capacity * element_size);
+      new_contents = ts_malloc(new_capacity * element_size);
     }
-    self->capacity = new_capacity;
+    *capacity = new_capacity;
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
-  _array__reserve(self, element_size, other->size);
-  self->size = other->size;
-  memcpy(self->contents, other->contents, self->size * element_size);
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(Array *self, Array *other) {
-  Array swap = *other;
-  *other = *self;
-  *self = swap;
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
-  uint32_t new_size = self->size + count;
-  if (new_size > self->capacity) {
-    uint32_t new_capacity = self->capacity * 2;
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    _array__reserve(self, element_size, new_capacity);
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void _array__splice(Array *self, size_t element_size,
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t new_size = *size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  assert(old_end <= self->size);
+  assert(old_end <= *size);
 
-  _array__reserve(self, element_size, new_size);
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
 
-  char *contents = (char *)self->contents;
-  if (self->size > old_end) {
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (self->size - old_end) * element_size
+      (*size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
       );
     }
   }
-  self->size += new_count - old_count;
+  *size += new_count - old_count;
+
+  return new_contents;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

crates/highlight/src/highlight.rs

@@ -1274,28 +1274,22 @@ fn injection_for_match<'a>(
             // In addition to specifying the language name via the text of a
             // captured node, it can also be hard-coded via a `#set!` predicate
             // that sets the injection.language key.
-            "injection.language" => {
-                if language_name.is_none() {
-                    language_name = prop.value.as_ref().map(std::convert::AsRef::as_ref);
-                }
+            "injection.language" if language_name.is_none() => {
+                language_name = prop.value.as_ref().map(std::convert::AsRef::as_ref);
             }
 
             // Setting the `injection.self` key can be used to specify that the
             // language name should be the same as the language of the current
             // layer.
-            "injection.self" => {
-                if language_name.is_none() {
-                    language_name = Some(config.language_name.as_str());
-                }
+            "injection.self" if language_name.is_none() => {
+                language_name = Some(config.language_name.as_str());
             }
 
             // Setting the `injection.parent` key can be used to specify that
             // the language name should be the same as the language of the
             // parent layer
-            "injection.parent" => {
-                if language_name.is_none() {
-                    language_name = parent_name;
-                }
+            "injection.parent" if language_name.is_none() => {
+                language_name = parent_name;
             }
 
             // By default, injections do not include the *children* of an

crates/language/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-language"
 description = "The tree-sitter Language type, used by the library and by language implementations"
-version = "0.1.6"
+version = "0.1.7"
 authors.workspace = true
 edition.workspace = true
 rust-version = "1.77"

crates/language/wasm/include/stdint.h

@@ -23,9 +23,15 @@ typedef long unsigned int size_t;
 
 typedef long unsigned int uintptr_t;
 
-#define UINT16_MAX 65535
+#define INT8_MAX 127
+#define INT16_MAX 32767
+#define INT32_MAX 2147483647L
+#define INT64_MAX 9223372036854775807LL
 
+#define UINT8_MAX 255
+#define UINT16_MAX 65535
 #define UINT32_MAX 4294967295U
+#define UINT64_MAX 18446744073709551615ULL
 
 #if defined(__wasm32__)
 

crates/language/wasm/include/string.h

@@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
 
 int strncmp(const char *left, const char *right, size_t n);
 
+size_t strlen(const char *str);
+
 #endif // TREE_SITTER_WASM_STRING_H_

crates/language/wasm/src/stdio.c

@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <string.h>
 
 typedef struct {
   bool left_justify;   // -
@@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
   return 2 + len;
 }
 
-size_t strlen(const char *str) {
-  const char *s = str;
-  while (*s) s++;
-  return s - str;
-}
-
 char *strncpy(char *dest, const char *src, size_t n) {
   char *d = dest;
   const char *s = src;

crates/language/wasm/src/stdlib.c

@@ -48,6 +48,19 @@ static int grow_heap(size_t size) {
   return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX;
 }
 
+// Grows the heap if necessary to fit a region at the _end_ of the heap
+// ending at `region_end` by `size` bytes.
+//
+// Returns 0 if the heap could not be grown, 1 otherwise.
+static inline int grow_heap_for_region(Region *region_end, size_t size) {
+  if (region_end > heap_end) {
+    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) return 0;
+    if (!grow_heap(size)) return 0;
+    heap_end = get_heap_end();
+  }
+  return 1;
+}
+
 // Clear out the heap, and move it to the given address.
 void reset_heap(void *new_heap_start) {
   heap_start = new_heap_start;
@@ -76,13 +89,7 @@ void *malloc(size_t size) {
 
   Region *region_end = region_after(next, size);
 
-  if (region_end > heap_end) {
-    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) {
-      return NULL;
-    }
-    if (!grow_heap(size)) return NULL;
-    heap_end = get_heap_end();
-  }
+  if (!grow_heap_for_region(region_end, size)) return NULL;
 
   void *result = &next->data;
   next->size = size;
@@ -109,6 +116,7 @@ void free(void *ptr) {
 
 void *calloc(size_t count, size_t size) {
   void *result = malloc(count * size);
+  if (!result) return NULL;
   memset(result, 0, count * size);
   return result;
 }
@@ -117,19 +125,36 @@ void *realloc(void *ptr, size_t new_size) {
   if (ptr == NULL) {
     return malloc(new_size);
   }
+  if (new_size == 0) {
+    free(ptr);
+    return NULL;
+  }
+
 
   Region *region = region_for_ptr(ptr);
   Region *region_end = region_after(region, region->size);
 
-  // When reallocating the last allocated region, return
-  // the same pointer, and skip copying the data.
+  // When reallocating the last allocated region, resize
+  // in place if possible, return the same pointer, and
+  // skip copying the data.
   if (region_end == next) {
-    next = region;
-    return malloc(new_size);
+    Region *new_region_end = region_after(region, new_size);
+
+    size_t additional_size = (char *)new_region_end - (char *)heap_end;
+    if (!grow_heap_for_region(new_region_end, additional_size)) return NULL;
+
+    region->size = new_size;
+    next = new_region_end;
+    return &region->data;
   }
 
   void *result = malloc(new_size);
-  memcpy(result, &region->data, region->size);
+  if (!result) return NULL;
+
+  size_t copy_size = region->size < new_size ? region->size : new_size;
+  memcpy(result, &region->data, copy_size);
+
+  free(ptr);
   return result;
 }
 

crates/language/wasm/src/string.c

@@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
   }
   return 0;
 }
+
+size_t strlen(const char *str) {
+  const char *s = str;
+  while (*s) s++;
+  return s - str;
+}

crates/loader/src/loader.rs

@@ -1,6 +1,8 @@
 #![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 
+#[cfg(unix)]
+use std::fmt::Write as _;
 #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))]
 use std::ops::Range;
 #[cfg(feature = "tree-sitter-highlight")]
@@ -8,6 +10,7 @@ use std::sync::Mutex;
 use std::{
     collections::HashMap,
     env, fs,
+    hash::{Hash as _, Hasher as _},
     io::{BufRead, BufReader},
     marker::PhantomData,
     mem,
@@ -75,8 +78,6 @@ pub enum LoaderError {
     NoLanguage,
     #[error(transparent)]
     Query(LoaderQueryError),
-    #[error(transparent)]
-    ScannerSymbols(ScannerSymbolError),
     #[error("Failed to load language for scope '{0}':\n{1}")]
     ScopeLoad(String, Box<Self>),
     #[error(transparent)]
@@ -199,28 +200,6 @@ impl std::fmt::Display for SymbolError {
     }
 }
 
-#[derive(Debug, Error)]
-pub struct ScannerSymbolError {
-    pub missing: Vec<String>,
-}
-
-impl std::fmt::Display for ScannerSymbolError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(
-            f,
-            "Missing required functions in the external scanner, parsing won't work without these!\n"
-        )?;
-        for symbol in &self.missing {
-            writeln!(f, "  `{symbol}`")?;
-        }
-        writeln!(
-            f,
-            "You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n"
-        )?;
-        Ok(())
-    }
-}
-
 #[derive(Debug, Error)]
 pub struct WasiSDKClangError {
     pub wasi_sdk_dir: String,
@@ -831,8 +810,11 @@ impl Loader {
                     path = PathBuf::from(path.file_stem()?.to_os_string());
                 }
                 extensions.reverse();
-                self.language_configuration_ids_by_file_type
-                    .get(&extensions.join("."))
+                // Try longest extension suffixs first (e.g. "foo.bar.baz"->"bar.baz"->"baz"),
+                // stopping at the first match.
+                (0..extensions.len())
+                    .map(|i| extensions[i..].join("."))
+                    .find_map(|key| self.language_configuration_ids_by_file_type.get(&key))
             });
 
         if let Some(configuration_ids) = configuration_ids {
@@ -1025,20 +1007,26 @@ impl Loader {
             return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
         }
 
+        // Create a unique lock path based on the output path hash to prevent
+        // interference when multiple processes build the same grammar (by name)
+        // to different output locations
+        let lock_hash = {
+            let mut hasher = std::hash::DefaultHasher::new();
+            output_path.hash(&mut hasher);
+            format!("{:x}", hasher.finish())
+        };
+
         let lock_path = if env::var("CROSS_RUNNER").is_ok() {
             tempfile::tempdir()
-                .unwrap()
+                .expect("create a temp dir")
                 .path()
-                .join("tree-sitter")
-                .join("lock")
-                .join(format!("{}.lock", config.name))
+                .to_path_buf()
         } else {
-            etcetera::choose_base_strategy()?
-                .cache_dir()
-                .join("tree-sitter")
-                .join("lock")
-                .join(format!("{}.lock", config.name))
-        };
+            etcetera::choose_base_strategy()?.cache_dir()
+        }
+        .join("tree-sitter")
+        .join("lock")
+        .join(format!("{}-{lock_hash}.lock", config.name));
 
         if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
             recompile = false;
@@ -1085,10 +1073,30 @@ impl Loader {
             self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?;
 
             if config.scanner_path.is_some() {
-                self.check_external_scanner(&config.name, &output_path)?;
+                self.check_external_scanner(&output_path)?;
             }
         }
 
+        // Ensure the dynamic library exists before trying to load it. This can
+        // happen in race conditions where we couldn't acquire the lock because
+        // another process was compiling but it still hasn't finished by the
+        // time we reach this point, so the output file still doesn't exist.
+        //
+        // Instead of allowing the `load_language` call below to fail, return a
+        // clearer error to the user here.
+        if !output_path.exists() {
+            let msg = format!(
+                "Dynamic library `{}` not found after build attempt. \
+                Are you running multiple processes building to the same output location?",
+                output_path.display()
+            );
+
+            Err(LoaderError::IO(IoError::new(
+                std::io::Error::new(std::io::ErrorKind::NotFound, msg),
+                Some(output_path.as_path()),
+            )))?;
+        }
+
         Self::load_language(&output_path, &language_fn_name)
     }
 
@@ -1187,6 +1195,9 @@ impl Loader {
                 command.arg("-UTREE_SITTER_REUSE_ALLOCATOR");
             } else {
                 command.arg("-shared");
+                command.arg("-Wl,--no-undefined");
+                #[cfg(target_os = "openbsd")]
+                command.arg("-lc");
             }
             command.args(cc_config.get_files());
             command.arg("-o").arg(output_path);
@@ -1221,25 +1232,15 @@ impl Loader {
     }
 
     #[cfg(unix)]
-    fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> {
-        let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) {
-            "_"
+    fn check_external_scanner(&self, library_path: &Path) -> LoaderResult<()> {
+        let section = " T ";
+        // Older ppc toolchains incorrectly report functions in the Data section. This bug has been
+        // fixed, but we still need to account for older systems.
+        let old_ppc_section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
+            Some(" D ")
         } else {
-            ""
+            None
         };
-        let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
-            " D "
-        } else {
-            " T "
-        };
-        let mut must_have = vec![
-            format!("{prefix}tree_sitter_{name}_external_scanner_create"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_destroy"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_serialize"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_deserialize"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_scan"),
-        ];
-
         let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned());
         let command = Command::new(nm_cmd)
             .arg("--defined-only")
@@ -1247,54 +1248,41 @@ impl Loader {
             .output();
         if let Ok(output) = command {
             if output.status.success() {
-                let mut found_non_static = false;
+                let mut non_static_symbols = String::new();
                 for line in String::from_utf8_lossy(&output.stdout).lines() {
-                    if line.contains(section) {
+                    if line.contains(section) || old_ppc_section.is_some_and(|s| line.contains(s)) {
                         if let Some(function_name) =
                             line.split_whitespace().collect::<Vec<_>>().get(2)
                         {
                             if !line.contains("tree_sitter_") {
-                                if !found_non_static {
-                                    found_non_static = true;
-                                    warn!("Found non-static non-tree-sitter functions in the external scanner");
-                                }
-                                warn!("  `{function_name}`");
-                            } else {
-                                must_have.retain(|f| f != function_name);
+                                writeln!(&mut non_static_symbols, "  `{function_name}`").unwrap();
                             }
                         }
                     }
                 }
-                if found_non_static {
-                    warn!(concat!(
-                        "Consider making these functions static, they can cause conflicts ",
-                        "when another tree-sitter project uses the same function name."
-                    ));
-                }
-
-                if !must_have.is_empty() {
-                    return Err(LoaderError::ScannerSymbols(ScannerSymbolError {
-                        missing: must_have,
-                    }));
+                if !non_static_symbols.is_empty() {
+                    warn!(
+                        "Found non-static non-tree-sitter functions in the external scanner\n{non_static_symbols}\n{}",
+                        concat!(
+                            "Consider making these functions static, they can cause conflicts ",
+                            "when another tree-sitter project uses the same function name."
+                        )
+                    );
                 }
             }
+        } else {
+            warn!(
+                "Failed to run `nm` to verify symbols in {}",
+                library_path.display()
+            );
         }
 
         Ok(())
     }
 
     #[cfg(windows)]
-    fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> {
+    fn check_external_scanner(&self, _library_path: &Path) -> LoaderResult<()> {
         // TODO: there's no nm command on windows, whoever wants to implement this can and should :)
-
-        // let mut must_have = vec![
-        //     format!("tree_sitter_{name}_external_scanner_create"),
-        //     format!("tree_sitter_{name}_external_scanner_destroy"),
-        //     format!("tree_sitter_{name}_external_scanner_serialize"),
-        //     format!("tree_sitter_{name}_external_scanner_deserialize"),
-        //     format!("tree_sitter_{name}_external_scanner_scan"),
-        // ];
-
         Ok(())
     }
 
@@ -1309,6 +1297,7 @@ impl Loader {
 
         let mut command = Command::new(&clang_executable);
         command.current_dir(src_path).args([
+            "--target=wasm32-unknown-wasi",
             "-o",
             output_path.to_str().unwrap(),
             "-fPIC",
@@ -1487,7 +1476,7 @@ impl Loader {
     ) -> Option<&'a HighlightConfiguration> {
         match self.language_configuration_for_injection_string(string) {
             Err(e) => {
-                error!("Failed to load language for injection string '{string}': {e}",);
+                error!("Failed to load language for injection string '{string}': {e}");
                 None
             }
             Ok(None) => None,

crates/xtask/src/build_wasm.rs

@@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
         "-D", "NDEBUG=",
         "-D", "_POSIX_C_SOURCE=200112L",
         "-D", "_DEFAULT_SOURCE=",
+        "-D", "_BSD_SOURCE=",
         "-D", "_DARWIN_C_SOURCE=",
         "-I", "lib/src",
         "-I", "lib/include",

crates/xtask/src/fetch.rs

@@ -1,19 +1,74 @@
-use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION};
+use crate::{bail_on_err, root_dir, EMSCRIPTEN_VERSION};
 use anyhow::Result;
-use std::{fs, process::Command};
+use std::{fs, path::Path, process::Command};
 
-pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
+enum FixtureRef<'a> {
+    Tag(&'a str),
+    Branch(&'a str),
+}
+
+impl<'a> FixtureRef<'a> {
+    #[allow(clippy::use_self)]
+    const fn new(tag: &'a str, branch: Option<&'a str>) -> FixtureRef<'a> {
+        if let Some(b) = branch {
+            Self::Branch(b)
+        } else {
+            Self::Tag(tag)
+        }
+    }
+
+    const fn ref_type(&self) -> &'static str {
+        match self {
+            FixtureRef::Tag(_) => "tag",
+            FixtureRef::Branch(_) => "branch",
+        }
+    }
+}
+
+impl std::fmt::Display for FixtureRef<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FixtureRef::Tag(tag) => write!(f, "{tag}"),
+            FixtureRef::Branch(branch) => write!(f, "{branch}"),
+        }
+    }
+}
+
+fn current_ref_name(grammar_dir: &Path) -> Result<(String, Option<&'static str>)> {
+    let tag_args = ["describe", "--tags", "--exact-match", "HEAD"];
+    let branch_args = ["rev-parse", "--abbrev-ref", "HEAD"];
+
+    for (args, ref_type) in [tag_args.as_ref(), branch_args.as_ref()]
+        .iter()
+        .zip(&["tag", "branch"])
+    {
+        let name_cmd = Command::new("git")
+            .current_dir(grammar_dir)
+            .args(*args)
+            .output()?;
+        let name = String::from_utf8_lossy(&name_cmd.stdout);
+        let name = name.trim();
+        if !name.is_empty() {
+            return Ok((name.to_string(), Some(ref_type)));
+        }
+    }
+
+    Ok(("<unknown>".to_string(), None))
+}
+
+pub fn run_fixtures() -> Result<()> {
     let fixtures_dir = root_dir().join("test").join("fixtures");
     let grammars_dir = fixtures_dir.join("grammars");
     let fixtures_path = fixtures_dir.join("fixtures.json");
 
-    // grammar name, tag
-    let mut fixtures: Vec<(String, String)> =
+    // grammar name, tag, [branch]
+    let fixtures: Vec<(String, String, Option<String>)> =
         serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?;
 
-    for (grammar, tag) in &mut fixtures {
-        let grammar_dir = grammars_dir.join(&grammar);
+    for (grammar, tag, branch) in &fixtures {
+        let grammar_dir = grammars_dir.join(grammar);
         let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}");
+        let target_ref = FixtureRef::new(tag, branch.as_deref());
 
         println!("Fetching the {grammar} grammar...");
 
@@ -24,7 +79,7 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 "--depth",
                 "1",
                 "--branch",
-                tag,
+                &target_ref.to_string(),
                 &grammar_url,
                 &grammar_dir.to_string_lossy(),
             ]);
@@ -33,31 +88,71 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 &format!("Failed to clone the {grammar} grammar"),
             )?;
         } else {
-            let mut describe_command = Command::new("git");
-            describe_command.current_dir(&grammar_dir).args([
-                "describe",
-                "--tags",
-                "--exact-match",
-                "HEAD",
-            ]);
+            let (current_ref, current_ref_type) = current_ref_name(&grammar_dir)?;
+            if current_ref != target_ref.to_string() {
+                println!(
+                    "Updating {grammar} grammar from {} {current_ref} to {} {target_ref}...",
+                    current_ref_type.unwrap_or("<unknown>"),
+                    target_ref.ref_type(),
+                );
 
-            let output = describe_command.output()?;
-            let current_tag = String::from_utf8_lossy(&output.stdout);
-            let current_tag = current_tag.trim();
-
-            if current_tag != tag {
-                println!("Updating {grammar} grammar from {current_tag} to {tag}...");
-
-                let mut fetch_command = Command::new("git");
-                fetch_command.current_dir(&grammar_dir).args([
-                    "fetch",
-                    "origin",
-                    &format!("refs/tags/{tag}:refs/tags/{tag}"),
-                ]);
-                bail_on_err(
-                    &fetch_command.spawn()?.wait_with_output()?,
-                    &format!("Failed to fetch tag {tag} for {grammar} grammar"),
-                )?;
+                match target_ref {
+                    FixtureRef::Branch(branch) => {
+                        let mut fetch_cmd = Command::new("git");
+                        fetch_cmd.current_dir(&grammar_dir).args([
+                            "fetch",
+                            "--update-shallow",
+                            "origin",
+                            &format!("+refs/heads/{branch}:refs/remotes/origin/{branch}"),
+                        ]);
+                        bail_on_err(
+                            &fetch_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to fetch branch {branch}"),
+                        )?;
+                        let mut switch_cmd = Command::new("git");
+                        switch_cmd
+                            .current_dir(&grammar_dir)
+                            .args(["switch", branch]);
+                        bail_on_err(
+                            &switch_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to checkout branch {branch}"),
+                        )?;
+                        let mut set_upstream_cmd = Command::new("git");
+                        set_upstream_cmd.current_dir(&grammar_dir).args([
+                            "branch",
+                            "--set-upstream-to",
+                            &format!("origin/{branch}"),
+                            branch,
+                        ]);
+                        bail_on_err(
+                            &set_upstream_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to set upstream for branch {branch}"),
+                        )?;
+                        let mut pull_cmd = Command::new("git");
+                        pull_cmd
+                            .current_dir(&grammar_dir)
+                            .args(["pull", "origin", branch]);
+                        bail_on_err(
+                            &pull_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to pull latest from branch {branch}"),
+                        )?;
+                    }
+                    FixtureRef::Tag(tag) => {
+                        let mut fetch_command = Command::new("git");
+                        fetch_command.current_dir(&grammar_dir).args([
+                            "fetch",
+                            "origin",
+                            &format!("refs/tags/{tag}:refs/tags/{tag}"),
+                        ]);
+                        bail_on_err(
+                            &fetch_command.spawn()?.wait_with_output()?,
+                            &format!(
+                                "Failed to fetch {} {target_ref} for {grammar} grammar",
+                                target_ref.ref_type()
+                            ),
+                        )?;
+                    }
+                }
 
                 let mut reset_command = Command::new("git");
                 reset_command
@@ -71,29 +166,23 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 let mut checkout_command = Command::new("git");
                 checkout_command
                     .current_dir(&grammar_dir)
-                    .args(["checkout", tag]);
+                    .args(["checkout", &target_ref.to_string()]);
                 bail_on_err(
                     &checkout_command.spawn()?.wait_with_output()?,
-                    &format!("Failed to checkout tag {tag} for {grammar} grammar"),
+                    &format!(
+                        "Failed to checkout {} {target_ref} for {grammar} grammar",
+                        target_ref.ref_type()
+                    ),
                 )?;
             } else {
-                println!("{grammar} grammar is already at tag {tag}");
+                println!(
+                    "{grammar} grammar is already at {} {target_ref}",
+                    target_ref.ref_type()
+                );
             }
         }
     }
 
-    if args.update {
-        println!("Updating the fixtures lock file");
-        fs::write(
-            &fixtures_path,
-            // format the JSON without extra newlines
-            serde_json::to_string(&fixtures)?
-                .replace("[[", "[\n  [")
-                .replace("],", "],\n  ")
-                .replace("]]", "]\n]"),
-        )?;
-    }
-
     Ok(())
 }
 

crates/xtask/src/main.rs

@@ -36,7 +36,7 @@ enum Commands {
     /// Fetches emscripten.
     FetchEmscripten,
     /// Fetches the fixtures for testing tree-sitter.
-    FetchFixtures(FetchFixtures),
+    FetchFixtures,
     /// Generate the Rust bindings from the C library.
     GenerateBindings,
     /// Generates the fixtures for testing tree-sitter.
@@ -118,13 +118,6 @@ struct Clippy {
     package: Option<String>,
 }
 
-#[derive(Args)]
-struct FetchFixtures {
-    /// Update all fixtures to the latest tag
-    #[arg(long, short)]
-    update: bool,
-}
-
 #[derive(Args)]
 struct GenerateFixtures {
     /// Generates the parser to Wasm
@@ -232,8 +225,8 @@ fn run() -> Result<()> {
         Commands::CheckWasmExports(check_options) => check_wasm_exports::run(&check_options)?,
         Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?,
         Commands::FetchEmscripten => fetch::run_emscripten()?,
-        Commands::FetchFixtures(fetch_fixture_options) => {
-            fetch::run_fixtures(&fetch_fixture_options)?;
+        Commands::FetchFixtures => {
+            fetch::run_fixtures()?;
         }
         Commands::GenerateBindings => generate::run_bindings()?,
         Commands::GenerateFixtures(generate_fixtures_options) => {

docs/src/assets/js/playground.js

@@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
 }
 
 window.initializePlayground = async (opts) => {
-  const { Parser, Language } = window.TreeSitter;
+  const { Parser, Language, Query } = window.TreeSitter;
 
   const { local } = opts;
   if (local) {
@@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
       marks.forEach((m) => m.clear());
 
       if (tree && query) {
-        const captures = query.captures(
-          tree.rootNode,
-          { row: startRow, column: 0 },
-          { row: endRow, column: 0 },
-        );
+        const captures = query.captures(tree.rootNode, {
+          startPosition: { row: startRow, column: 0 },
+          endPosition: { row: endRow, column: 0 },
+        });
         let lastNodeId;
         for (const { name, node } of captures) {
           if (node.id === lastNodeId) continue;
@@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
       const queryText = queryEditor.getValue();
 
       try {
-        query = parser.language.query(queryText);
+        query = new Query(parser.language, queryText);
         let match;
 
         let row = 0;

docs/src/cli/fuzz.md

@@ -31,11 +31,13 @@ If `--lib-path` is used, the name of the language used to extract the library's
 
 ### `--edits <EDITS>`
 
-The maximum number of edits to perform. The default is 3.
+The maximum number of edits to perform. The default is 3. This value can also be set via the `TREE_SITTER_EDITS` environment
+variable.
 
 ### `--iterations <ITERATIONS>`
 
-The number of iterations to run. The default is 10.
+The number of iterations to run. The default is 10. This value can also be set via the `TREE_SITTER_ITERATIONS` environment
+variable.
 
 ### `-i/--include <INCLUDE>`
 

docs/src/creating-parsers/1-getting-started.md

@@ -41,6 +41,10 @@ cd tree-sitter-${LOWER_PARSER_NAME}
 The `LOWER_` prefix here means the "lowercase" name of the language.
 ```
 
+```admonish warning
+Dashes are not permitted via the CLI's `init` command and should not be used in parser names.
+```
+
 ### Init
 
 Once you've installed the `tree-sitter` CLI tool, you can start setting up your project, which will allow your parser to

docs/src/using-parsers/queries/2-operators.md

@@ -110,6 +110,69 @@ This pattern would match a set of possible keyword tokens, capturing them as `@k
 ] @keyword
 ```
 
+Alternations can have quantified alternants, and then can have their own
+quantifiers as well. See the following examples for an illustration of how these
+cases work:
+
+```query
+;;; SOURCE CODE ;;;
+; #include <foo>
+; #include <bar>
+; #include <baz>
+; // comment
+;;;;;;;;;;;;;;;;;;;
+
+[
+  (preproc_include)
+  (comment)
+]+ @capture
+; ^ Produces one match with four captures:
+; [
+;   "#include <foo>\n",
+;   "#include <bar>\n",
+;   "#include <baz>\n",
+;   "// comment",
+; ]
+;
+; Regex equivalent: [ab]+
+
+[
+  (preproc_include)+
+  (comment)
+] @capture
+; ^ Produces two matches; one with three captures, and one with one capture:
+; [
+;   "#include <foo>\n",
+;   "#include <bar>\n",
+;   "#include <baz>\n",
+; ],
+; [
+;   "// comment",
+; ]
+;
+; Regex equivalent: a+|b
+
+[
+  (preproc_include)
+  (comment)
+] @capture
+; ^ Produces four matches, each with one capture:
+; [
+;   "#include <foo>\n",
+; ],
+; [
+;   "#include <bar>\n",
+; ],
+; [
+;   "#include <baz>\n",
+; ],
+; [
+;   "// comment",
+; ]
+;
+; Regex equivalent: [ab]
+```
+
 ## Anchors
 
 The anchor operator, `.`, is used to constrain the ways in which child patterns are matched. It has different behaviors

docs/src/using-parsers/queries/4-api.md

@@ -59,3 +59,27 @@ bool ts_query_cursor_next_match(TSQueryCursor *, TSQueryMatch *match);
 
 This function will return `false` when there are no more matches. Otherwise, it will populate the `match` with data about
 which pattern matched and which nodes were captured.
+
+## Restricting the Query Range
+
+You can restrict the range in which the query is executed using byte offsets or point (row, column) positions:
+
+```c
+bool ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
+bool ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
+```
+
+These functions return matches that *intersect* with the given range. A match may be returned even if only part of it overlaps
+with the range.
+
+There are also "containing" variants that only return matches where all captured nodes are fully within the range:
+
+```c
+bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
+bool ts_query_cursor_set_containing_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
+```
+
+```admonish note
+For all of these functions, an end value of zero is treated as unbounded (the maximum possible value).
+This means passing a byte range of `(0, 0)` (or a point range of `{0, 0}, {0, 0}`) will match the entire tree, not an empty range.
+```

flake.nix

@@ -17,7 +17,7 @@
       eachSystem = lib.genAttrs systems;
       pkgsFor = inputs.nixpkgs.legacyPackages;
 
-      version = "0.26.3";
+      version = "0.26.10";
 
       fs = lib.fileset;
       src = fs.toSource {
@@ -333,6 +333,7 @@
               pkg-config
               llvm
               clang
+              clang-tools
               libclang
 
               nodejs_22
@@ -348,6 +349,7 @@
             ];
 
             shellHook = ''
+              export PATH="${pkgs.clang-tools}/bin:$PATH"
               echo "Tree-sitter Dev Environment"
               echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
               echo ""

lib/Cargo.toml

@@ -53,7 +53,7 @@ tree-sitter-language.workspace = true
 streaming-iterator = "0.1.9"
 
 [dependencies.wasmtime-c-api]
-version = "33.0.2"
+version = "36.0.12"
 optional = true
 package = "wasmtime-c-api-impl"
 default-features = false

lib/binding_rust/bindings.rs

@@ -106,6 +106,7 @@ pub struct TSLogger {
         ),
     >,
 }
+#[doc = " A summary of a change to a text document.\n\n The `start_byte` and `start_point` values must be less than or equal to the\n `old_end_byte` and `old_end_point` values, respectively. Passing an edit\n that violates these invariants may produce nonsensical results."]
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct TSInputEdit {
@@ -218,7 +219,7 @@ extern "C" {
     pub fn ts_parser_included_ranges(self_: *const TSParser, count: *mut u32) -> *const TSRange;
 }
 extern "C" {
-    #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n    and (row, column) position. The function should return a pointer to the\n    text and write its length to the [`bytes_read`] pointer. The parser does\n    not take ownership of this buffer; it just borrows it until it has\n    finished reading it. The function should write a zero value to the\n    [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n    of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n    `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to the progress callback returning true. This callback\n    is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read"]
+    #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n    and (row, column) position. The function should return a pointer to the\n    text and write its length to the [`bytes_read`] pointer. The parser does\n    not take ownership of this buffer; it just borrows it until it has\n    finished reading it. The function should write a zero value to the\n    [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n    of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n    `TSInputEncodingUTF8`, `TSInputEncodingUTF16LE`, `TSInputEncoding16BE`,\n    or `TSInputEncodingCustom`.\n 4. [`decode`]: A function to read one code point from the given input. This\n    function should return the number of bytes consumed and write the code point\n    to the [`code_point`] pointer, or write -1 if the input is invalid.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to the progress callback returning true. This callback\n    is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read\n [`decode`]: TSInput::decode\n [`code_point`]: TSDecodeFunction::code_point"]
     pub fn ts_parser_parse(
         self_: *mut TSParser,
         old_tree: *const TSTree,
@@ -298,7 +299,7 @@ extern "C" {
     pub fn ts_tree_included_ranges(self_: *const TSTree, length: *mut u32) -> *mut TSRange;
 }
 extern "C" {
-    #[doc = " Edit the syntax tree to keep it in sync with source code that has been\n edited.\n\n You must describe the edit both in terms of byte offsets and in terms of\n (row, column) coordinates."]
+    #[doc = " Edit the syntax tree to keep it in sync with source code that has been\n edited.\n\n You must describe the edit both in terms of byte offsets and in terms of\n (row, column) coordinates.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_tree_edit(self_: *mut TSTree, edit: *const TSInputEdit);
 }
 extern "C" {
@@ -488,7 +489,7 @@ extern "C" {
     ) -> TSNode;
 }
 extern "C" {
-    #[doc = " Edit the node to keep it in-sync with source code that has been edited.\n\n This function is only rarely needed. When you edit a syntax tree with the\n [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree\n afterward will already reflect the edit. You only need to use [`ts_node_edit`]\n when you have a [`TSNode`] instance that you want to keep and continue to use\n after an edit."]
+    #[doc = " Edit the node to keep it in-sync with source code that has been edited.\n\n This function is only rarely needed. When you edit a syntax tree with the\n [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree\n afterward will already reflect the edit. You only need to use [`ts_node_edit`]\n when you have a [`TSNode`] instance that you want to keep and continue to use\n after an edit.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_node_edit(self_: *mut TSNode, edit: *const TSInputEdit);
 }
 extern "C" {
@@ -496,11 +497,11 @@ extern "C" {
     pub fn ts_node_eq(self_: TSNode, other: TSNode) -> bool;
 }
 extern "C" {
-    #[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance."]
+    #[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_point_edit(point: *mut TSPoint, point_byte: *mut u32, edit: *const TSInputEdit);
 }
 extern "C" {
-    #[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance."]
+    #[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_range_edit(range: *mut TSRange, edit: *const TSInputEdit);
 }
 extern "C" {
@@ -697,7 +698,7 @@ extern "C" {
     pub fn ts_query_cursor_set_match_limit(self_: *mut TSQueryCursor, limit: u32);
 }
 extern "C" {
-    #[doc = " Set the range of bytes in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."]
+    #[doc = " Set the range of bytes in which the query will be executed.\n\n The query cursor will return matches that intersect with the given byte range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range\n unbounded.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."]
     pub fn ts_query_cursor_set_byte_range(
         self_: *mut TSQueryCursor,
         start_byte: u32,
@@ -705,7 +706,7 @@ extern "C" {
     ) -> bool;
 }
 extern "C" {
-    #[doc = " Set the range of (row, column) positions in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start point is greater than the end point, otherwise\n it will return `true`."]
+    #[doc = " Set the range of (row, column) positions in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the\n range unbounded.\n\n This will return `false` if the start point is greater than the end point, otherwise\n it will return `true`."]
     pub fn ts_query_cursor_set_point_range(
         self_: *mut TSQueryCursor,
         start_point: TSPoint,
@@ -713,7 +714,7 @@ extern "C" {
     ) -> bool;
 }
 extern "C" {
-    #[doc = " Set the byte range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"]
+    #[doc = " Set the byte range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500\n\n NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range\n unbounded."]
     pub fn ts_query_cursor_set_containing_byte_range(
         self_: *mut TSQueryCursor,
         start_byte: u32,
@@ -721,7 +722,7 @@ extern "C" {
     ) -> bool;
 }
 extern "C" {
-    #[doc = " Set the point range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"]
+    #[doc = " Set the point range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500\n\n NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the\n range unbounded."]
     pub fn ts_query_cursor_set_containing_point_range(
         self_: *mut TSQueryCursor,
         start_point: TSPoint,

lib/binding_rust/build.rs

@@ -39,7 +39,7 @@ fn main() {
     }
 
     config
-        .flag_if_supported("-std=c11")
+        .std("c11")
         .flag_if_supported("-fvisibility=hidden")
         .flag_if_supported("-Wshadow")
         .flag_if_supported("-Wno-unused-parameter")
@@ -49,6 +49,7 @@ fn main() {
         .include(&include_path)
         .define("_POSIX_C_SOURCE", "200112L")
         .define("_DEFAULT_SOURCE", None)
+        .define("_BSD_SOURCE", None)
         .define("_DARWIN_C_SOURCE", None)
         .warnings(false)
         .file(src_path.join("lib.c"))

lib/binding_rust/lib.rs

@@ -2471,7 +2471,7 @@ impl Query {
         let pattern_count = unsafe { ffi::ts_query_pattern_count(ptr.0) as usize };
 
         let mut capture_names = Vec::with_capacity(capture_count as usize);
-        let mut capture_quantifiers_vec = Vec::with_capacity(pattern_count as usize);
+        let mut capture_quantifiers_vec = Vec::with_capacity(pattern_count);
         let mut text_predicates_vec = Vec::with_capacity(pattern_count);
         let mut property_predicates_vec = Vec::with_capacity(pattern_count);
         let mut property_settings_vec = Vec::with_capacity(pattern_count);
@@ -2924,7 +2924,7 @@ impl Query {
         } else {
             Err(predicate_error(
                 row,
-                format!("Invalid arguments to {function_name} predicate. Missing key argument",),
+                format!("Invalid arguments to {function_name} predicate. Missing key argument"),
             ))
         }
     }
@@ -3746,11 +3746,11 @@ impl fmt::Display for QueryError {
 #[must_use]
 pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
     let mut indent_level = initial_indent_level;
-    let mut formatted = String::new();
+    let mut formatted = String::with_capacity(sexp.len());
     let mut has_field = false;
 
     let mut c_iter = sexp.chars().peekable();
-    let mut s = String::with_capacity(sexp.len());
+    let mut scratch = String::with_capacity(sexp.len());
     let mut quote = '\0';
     let mut saw_paren = false;
     let mut did_last = false;
@@ -3796,12 +3796,12 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
         Some(())
     };
 
-    while fetch_next_str(&mut s).is_some() {
-        if s.is_empty() && indent_level > 0 {
+    while fetch_next_str(&mut scratch).is_some() {
+        if scratch.is_empty() && indent_level > 0 {
             // ")"
             indent_level -= 1;
             write!(formatted, ")").unwrap();
-        } else if s.starts_with('(') {
+        } else if scratch.starts_with('(') {
             if has_field {
                 has_field = false;
             } else {
@@ -3815,27 +3815,27 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
             }
 
             // "(node_name"
-            write!(formatted, "{s}").unwrap();
+            write!(formatted, "{scratch}").unwrap();
 
             // "(MISSING node_name" or "(UNEXPECTED 'x'"
-            if s.starts_with("(MISSING") || s.starts_with("(UNEXPECTED") {
-                fetch_next_str(&mut s).unwrap();
-                if s.is_empty() {
+            if scratch.starts_with("(MISSING") || scratch.starts_with("(UNEXPECTED") {
+                fetch_next_str(&mut scratch).unwrap();
+                if scratch.is_empty() {
                     while indent_level > 0 {
                         indent_level -= 1;
                         write!(formatted, ")").unwrap();
                     }
                 } else {
-                    write!(formatted, " {s}").unwrap();
+                    write!(formatted, " {scratch}").unwrap();
                 }
             }
-        } else if s.ends_with(':') {
+        } else if scratch.ends_with(':') {
             // "field:"
             writeln!(formatted).unwrap();
             for _ in 0..indent_level {
                 write!(formatted, "  ").unwrap();
             }
-            write!(formatted, "{s} ").unwrap();
+            write!(formatted, "{scratch} ").unwrap();
             has_field = true;
             indent_level += 1;
         }

lib/binding_web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.26.3",
+  "version": "0.26.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "web-tree-sitter",
-      "version": "0.26.3",
+      "version": "0.26.10",
       "license": "MIT",
       "devDependencies": {
         "@eslint/js": "^9.39.1",

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.26.3",
+  "version": "0.26.10",
   "description": "Tree-sitter bindings for the web",
   "repository": {
     "type": "git",

lib/binding_web/script/build.js

@@ -44,6 +44,9 @@ async function build() {
     keepNames: true,
     external: ['fs/*', 'fs/promises'],
     resolveExtensions: ['.ts', '.js', format === 'esm' ? '.mjs' : '.cjs'],
+    ...(format === 'cjs' ? {
+      footer: { js: 'module.exports.default = module.exports;' },
+    } : {}),
   });
 
   // Copy the Wasm files to the appropriate spot, as esbuild doesn't "bundle" Wasm files

lib/include/tree_sitter/api.h

@@ -114,6 +114,13 @@ typedef struct TSLogger {
   void (*log)(void *payload, TSLogType log_type, const char *buffer);
 } TSLogger;
 
+/**
+ * A summary of a change to a text document.
+ *
+ * The `start_byte` and `start_point` values must be less than or equal to the
+ * `old_end_byte` and `old_end_point` values, respectively. Passing an edit
+ * that violates these invariants may produce nonsensical results.
+ */
 typedef struct TSInputEdit {
   uint32_t start_byte;
   uint32_t old_end_byte;
@@ -293,7 +300,11 @@ const TSRange *ts_parser_included_ranges(
  * 2. [`payload`]: An arbitrary pointer that will be passed to each invocation
  *    of the [`read`] function.
  * 3. [`encoding`]: An indication of how the text is encoded. Either
- *    `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.
+ *    `TSInputEncodingUTF8`, `TSInputEncodingUTF16LE`, `TSInputEncoding16BE`,
+ *    or `TSInputEncodingCustom`.
+ * 4. [`decode`]: A function to read one code point from the given input. This
+ *    function should return the number of bytes consumed and write the code point
+ *    to the [`code_point`] pointer, or write -1 if the input is invalid.
  *
  * This function returns a syntax tree on success, and `NULL` on failure. There
  * are four possible reasons for failure:
@@ -306,6 +317,8 @@ const TSRange *ts_parser_included_ranges(
  * [`payload`]: TSInput::payload
  * [`encoding`]: TSInput::encoding
  * [`bytes_read`]: TSInput::read
+ * [`decode`]: TSInput::decode
+ * [`code_point`]: TSDecodeFunction::code_point
  */
 TSTree *ts_parser_parse(
   TSParser *self,
@@ -437,6 +450,9 @@ TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length);
  *
  * You must describe the edit both in terms of byte offsets and in terms of
  * (row, column) coordinates.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_tree_edit(TSTree *self, const TSInputEdit *edit);
 
@@ -700,6 +716,9 @@ TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPo
  * afterward will already reflect the edit. You only need to use [`ts_node_edit`]
  * when you have a [`TSNode`] instance that you want to keep and continue to use
  * after an edit.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_node_edit(TSNode *self, const TSInputEdit *edit);
 
@@ -714,6 +733,9 @@ bool ts_node_eq(TSNode self, TSNode other);
  * This function updates a single point's byte offset and row/column position
  * based on an edit operation. This is useful for editing points without
  * requiring a tree or node instance.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit);
 
@@ -723,6 +745,9 @@ void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit
  * This function updates a range's start and end positions based on an edit
  * operation. This is useful for editing ranges without requiring a tree
  * or node instance.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_range_edit(TSRange *range, const TSInputEdit *edit);
 
@@ -1070,7 +1095,7 @@ void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit);
 /**
  * Set the range of bytes in which the query will be executed.
  *
- * The query cursor will return matches that intersect with the given point range.
+ * The query cursor will return matches that intersect with the given byte range.
  * This means that a match may be returned even if some of its captures fall
  * outside the specified range, as long as at least part of the match
  * overlaps with the range.
@@ -1079,6 +1104,9 @@ void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit);
  * than the specified range, but part of that node intersects with the range,
  * the entire match will be returned.
  *
+ * NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range
+ * unbounded.
+ *
  * This will return `false` if the start byte is greater than the end byte, otherwise
  * it will return `true`.
  */
@@ -1096,6 +1124,9 @@ bool ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, ui
  * than the specified range, but part of that node intersects with the range,
  * the entire match will be returned.
  *
+ * NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the
+ * range unbounded.
+ *
  * This will return `false` if the start point is greater than the end point, otherwise
  * it will return `true`.
  */
@@ -1109,6 +1140,9 @@ bool ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, T
  * matches where _all_ nodes are _fully_ contained within the given range. Both functions
  * can be used together, e.g. to search for any matches that intersect line 5000, as
  * long as they are fully contained within lines 4500-5500
+ *
+ * NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range
+ * unbounded.
  */
 bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
 
@@ -1120,6 +1154,9 @@ bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t sta
  * matches where _all_ nodes are _fully_ contained within the given range. Both functions
  * can be used together, e.g. to search for any matches that intersect line 5000, as
  * long as they are fully contained within lines 4500-5500
+ *
+ * NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the
+ * range unbounded.
  */
 bool ts_query_cursor_set_containing_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
 

lib/src/alloc.h

@@ -9,7 +9,7 @@ extern "C" {
 #include <stdio.h>
 #include <stdlib.h>
 
-#if defined(TREE_SITTER_HIDDEN_SYMBOLS) || defined(_WIN32)
+#if defined(TREE_SITTER_HIDE_SYMBOLS) || defined(_WIN32)
 #define TS_PUBLIC
 #else
 #define TS_PUBLIC __attribute__((visibility("default")))

lib/src/array.h

@@ -52,67 +52,96 @@ extern "C" {
 
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity) \
-  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity)        \
+  ((self)->contents = _array__reserve(           \
+    (void *)(self)->contents, &(self)->capacity, \
+    array_elem_size(self), new_capacity)         \
+  )
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((Array *)(self))
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                            \
-  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
-   (self)->contents[(self)->size++] = (element))
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__grow(                              \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    );                                                            \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count) \
-  do { \
-    if ((count) == 0) break; \
-    _array__grow((Array *)(self), count, array_elem_size(self)); \
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__grow(                                             \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    );                                                                           \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count); \
+    (self)->size += (count);                                                     \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other)                                       \
+#define array_push_all(self, other) \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, contents)                    \
-  _array__splice(                                               \
-    (Array *)(self), array_elem_size(self), (self)->size, \
-    0, count,  contents                                        \
+#define array_extend(self, count, other_contents)                 \
+  (self)->contents = _array__splice(                              \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
   )
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents)  \
-  _array__splice(                                                       \
-    (Array *)(self), array_elem_size(self), _index,                \
-    old_count, new_count, new_contents                                 \
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  (self)->contents = _array__splice(                                   \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
   )
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element) \
-  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+#define array_insert(self, _index, element)                     \
+  (self)->contents = _array__splice(                            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((Array *)(self), array_elem_size(self), _index)
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other) \
-  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+#define array_assign(self, other)                                   \
+  (self)->contents = _array__assign(                                \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )
 
 /// Swap one array with another
-#define array_swap(self, other) \
-  _array__swap((Array *)(self), (Array *)(other))
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array_swap_tmp;                            \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
+  } while (0)
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +186,90 @@ extern "C" {
 
 // Private
 
-typedef Array(void) Array;
-
-/// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(Array *self) {
-  if (self->contents) {
-    ts_free(self->contents);
-    self->contents = NULL;
-    self->size = 0;
-    self->capacity = 0;
-  }
-}
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(Array *self, size_t element_size,
-                                uint32_t index) {
-  ts_assert(index < self->size);
-  char *contents = (char *)self->contents;
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  ts_assert(index < *size);
+  char *contents = (char *)self_contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (self->size - index - 1) * element_size);
-  self->size--;
+          (*size - index - 1) * element_size);
+  (*size)--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
-  if (new_capacity > self->capacity) {
-    if (self->contents) {
-      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
     } else {
-      self->contents = ts_malloc(new_capacity * element_size);
+      new_contents = ts_malloc(new_capacity * element_size);
     }
-    self->capacity = new_capacity;
+    *capacity = new_capacity;
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
-  _array__reserve(self, element_size, other->size);
-  self->size = other->size;
-  memcpy(self->contents, other->contents, self->size * element_size);
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(Array *self, Array *other) {
-  Array swap = *other;
-  *other = *self;
-  *self = swap;
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
-  uint32_t new_size = self->size + count;
-  if (new_size > self->capacity) {
-    uint32_t new_capacity = self->capacity * 2;
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    _array__reserve(self, element_size, new_capacity);
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void _array__splice(Array *self, size_t element_size,
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t new_size = *size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  ts_assert(old_end <= self->size);
+  ts_assert(old_end <= *size);
 
-  _array__reserve(self, element_size, new_size);
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
 
-  char *contents = (char *)self->contents;
-  if (self->size > old_end) {
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (self->size - old_end) * element_size
+      (*size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
       );
     }
   }
-  self->size += new_count - old_count;
+  *size += new_count - old_count;
+
+  return new_contents;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

lib/src/parser.c

@@ -795,7 +795,13 @@ static Subtree ts_parser__reuse_node(
       reason = "is_missing";
     } else if (ts_subtree_is_fragile(result)) {
       reason = "is_fragile";
-    } else if (ts_parser__has_included_range_difference(self, byte_offset, end_byte_offset)) {
+    } else if (ts_parser__has_included_range_difference(
+                 self,
+                 byte_offset,
+                 ts_subtree_is_eof(result)
+                   ? end_byte_offset
+                   : end_byte_offset + ts_subtree_lookahead_bytes(result)
+               )) {
       reason = "contains_different_included_range";
     }
 
@@ -2179,7 +2185,7 @@ balance:
   ts_assert(self->finished_tree.ptr);
   if (!ts_parser__balance_subtree(self)) {
     self->canceled_balancing = true;
-    return false;
+    return NULL;
   }
   self->canceled_balancing = false;
   LOG("done");

lib/src/portable/endian.h

@@ -20,6 +20,7 @@
     defined(__GNU__) || \
     defined(__HAIKU__) || \
     defined(__illumos__) || \
+    defined(__redox__) || \
     defined(__NetBSD__) || \
     defined(__OpenBSD__) || \
     defined(__CYGWIN__) || \

lib/src/query.c

@@ -56,15 +56,15 @@ typedef struct {
  * Steps have some additional fields in order to handle the `.` (or "anchor") operator,
  * which forbids additional child nodes:
  * - `is_immediate` - Indicates that the node matching this step cannot be preceded
- *   by other sibling nodes that weren't specified in the pattern.
+ *    by other sibling nodes that weren't specified in the pattern.
  * - `is_last_child` - Indicates that the node matching this step cannot have any
- *   subsequent named siblings.
+ *    subsequent named siblings.
  *
  * For simple patterns, steps are matched in sequential order. But in order to
  * handle alternative/repeated/optional sub-patterns, query steps are not always
  * structured as a linear sequence; they sometimes need to split and merge. This
  * is done using the following fields:
- *  - `alternative_index` - The index of a different query step that serves as
+ * - `alternative_index` - The index of a different query step that serves as
  *    an alternative to this step. A `NONE` value represents no alternative.
  *    When a query state reaches a step with an alternative index, the state
  *    is duplicated, with one copy remaining at the original step, and one copy
@@ -75,21 +75,22 @@ typedef struct {
  * - `is_pass_through` - Indicates that state has no matching logic of its own,
  *    and exists only to split a state. One copy of the state advances immediately
  *    to the next step, and one moves to the alternative step.
- * - `alternative_is_immediate` - Indicates that this step's alternative step
- *    should be treated as if `is_immediate` is true.
+ * - `is_inside_alternation` - Indicates that state is inside an alternation.
+ *    Currently only written to quantifier steps, read by logic that maintains
+ *    correctness for quantifiers inside alternations.
  *
  * Steps also store some derived state that summarizes how they relate to other
  * steps within the same pattern. This is used to optimize the matching process:
- *  - `contains_captures` - Indicates that this step or one of its child steps
- *     has a non-empty `capture_ids` list.
- *  - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
- *     it and all of its subsequent sibling steps within the same parent pattern
- *     are guaranteed to match.
- *  - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
- *     for the entire top-level pattern. When iterating through a query's
- *     captures using `ts_query_cursor_next_capture`, this field is used to
- *     detect that a capture can safely be returned from a match that has not
- *     even completed yet.
+ * - `contains_captures` - Indicates that this step or one of its child steps
+ *    has a non-empty `capture_ids` list.
+ * - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
+ *    it and all of its subsequent sibling steps within the same parent pattern
+ *    are guaranteed to match.
+ * - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
+ *    for the entire top-level pattern. When iterating through a query's
+ *    captures using `ts_query_cursor_next_capture`, this field is used to
+ *    detect that a capture can safely be returned from a match that has not
+ *    even completed yet.
  */
 typedef struct {
   TSSymbol symbol;
@@ -104,7 +105,7 @@ typedef struct {
   bool is_last_child: 1;
   bool is_pass_through: 1;
   bool is_dead_end: 1;
-  bool alternative_is_immediate: 1;
+  bool is_inside_alternation: 1;
   bool contains_captures: 1;
   bool root_pattern_guaranteed: 1;
   bool parent_pattern_guaranteed: 1;
@@ -180,6 +181,8 @@ typedef struct {
  *    have already been returned.
  * - `capture_list_id` - A numeric id that can be used to retrieve the state's
  *    list of captures from the `CaptureListPool`.
+ * - `heap_insert_order` - A sequence number used to preserve discovery order
+ *    among finished states with the same capture position and pattern.
  * - `seeking_immediate_match` - A flag that indicates that the state's next
  *    step must be matched by the very next sibling. This is used when
  *    processing repetitions, or when processing a wildcard node followed by
@@ -193,6 +196,7 @@ typedef struct {
 typedef struct {
   uint32_t id;
   uint32_t capture_list_id;
+  uint32_t heap_insert_order;
   uint16_t start_depth;
   uint16_t step_index;
   uint16_t pattern_index;
@@ -203,6 +207,7 @@ typedef struct {
   bool needs_parent: 1;
 } QueryState;
 
+typedef Array(QueryState) QueryStateList;
 typedef Array(TSQueryCapture) CaptureList;
 
 /*
@@ -313,14 +318,19 @@ struct TSQuery {
 struct TSQueryCursor {
   const TSQuery *query;
   TSTreeCursor cursor;
-  Array(QueryState) states;
-  Array(QueryState) finished_states;
+  QueryStateList states;
+  QueryStateList finished_states;
+  // Tracks how much of finished_states is in heap order. Elements at indices
+  // < this value satisfy the min-heap property; elements >= this value are
+  // newly pushed and need to be sifted into place. Only used by `next_capture`.
+  uint32_t finished_states_heap_size;
   CaptureListPool capture_list_pool;
   uint32_t depth;
   uint32_t max_start_depth;
   TSRange included_range;
   TSRange containing_range;
   uint32_t next_state_id;
+  uint32_t next_finished_state_id;
   const TSQueryCursorOptions *query_options;
   TSQueryCursorState query_state;
   unsigned operation_count;
@@ -333,6 +343,7 @@ struct TSQueryCursor {
 static const TSQueryError PARENT_DONE = -1;
 static const uint16_t PATTERN_DONE_MARKER = UINT16_MAX;
 static const uint16_t NONE = UINT16_MAX;
+static const uint32_t CAPTURE_LIST_NONE = UINT32_MAX;
 static const TSSymbol WILDCARD_SYMBOL = 0;
 static const unsigned OP_COUNT_PER_QUERY_CALLBACK_CHECK = 100;
 
@@ -428,7 +439,7 @@ static CaptureListPool capture_list_pool_new(void) {
 }
 
 static void capture_list_pool_reset(CaptureListPool *self) {
-  for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
+  for (uint32_t i = 0; i < self->list.size; i++) {
     // This invalid size means that the list is not in use.
     array_get(&self->list, i)->size = UINT32_MAX;
   }
@@ -436,18 +447,18 @@ static void capture_list_pool_reset(CaptureListPool *self) {
 }
 
 static void capture_list_pool_delete(CaptureListPool *self) {
-  for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
+  for (uint32_t i = 0; i < self->list.size; i++) {
     array_delete(array_get(&self->list, i));
   }
   array_delete(&self->list);
 }
 
-static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) {
+static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint32_t id) {
   if (id >= self->list.size) return &self->empty_list;
   return array_get(&self->list, id);
 }
 
-static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) {
+static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint32_t id) {
   ts_assert(id < self->list.size);
   return array_get(&self->list, id);
 }
@@ -458,10 +469,10 @@ static bool capture_list_pool_is_empty(const CaptureListPool *self) {
   return self->free_capture_list_count == 0 && self->list.size >= self->max_capture_list_count;
 }
 
-static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
+static uint32_t capture_list_pool_acquire(CaptureListPool *self) {
   // First see if any already allocated capture list is currently unused.
   if (self->free_capture_list_count > 0) {
-    for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
+    for (uint32_t i = 0; i < self->list.size; i++) {
       if (array_get(&self->list, i)->size == UINT32_MAX) {
         array_clear(array_get(&self->list, i));
         self->free_capture_list_count--;
@@ -474,7 +485,7 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
   // doesn't put us over the requested maximum.
   uint32_t i = self->list.size;
   if (i >= self->max_capture_list_count) {
-    return NONE;
+    return CAPTURE_LIST_NONE;
   }
   CaptureList list;
   array_init(&list);
@@ -482,12 +493,146 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
   return i;
 }
 
-static void capture_list_pool_release(CaptureListPool *self, uint16_t id) {
+static void capture_list_pool_release(CaptureListPool *self, uint32_t id) {
   if (id >= self->list.size) return;
   array_get(&self->list, id)->size = UINT32_MAX;
   self->free_capture_list_count++;
 }
 
+/********************
+ * FinishedStateHeap
+ *
+ * A min-heap of finished query states, ordered by (byte offset of next
+ * unconsumed capture, pattern_index, insertion order). This allows
+ * ts_query_cursor_next_capture to find the earliest capture in O(1) instead
+ * of scanning all finished states. The heap is maintained lazily -
+ * ts_query_cursor__advance uses plain array_push, and next_capture sifts
+ * new elements into place via a tracked heap_size boundary.
+ ********************/
+
+static void finished_state_swap(QueryStateList *states, uint32_t a, uint32_t b) {
+  QueryState tmp = *array_get(states, a);
+  *array_get(states, a) = *array_get(states, b);
+  *array_get(states, b) = tmp;
+}
+
+// Compare two finished states by (byte offset of next unconsumed capture,
+// pattern_index, insertion order).
+static inline bool finished_state_precedes(
+  const QueryState *a,
+  const QueryState *b,
+  const CaptureListPool *pool
+) {
+  const CaptureList *a_caps = capture_list_pool_get(pool, a->capture_list_id);
+  const CaptureList *b_caps = capture_list_pool_get(pool, b->capture_list_id);
+  if (a->consumed_capture_count >= a_caps->size) return false;
+  if (b->consumed_capture_count >= b_caps->size) return true;
+  uint32_t a_byte = ts_node_start_byte(a_caps->contents[a->consumed_capture_count].node);
+  uint32_t b_byte = ts_node_start_byte(b_caps->contents[b->consumed_capture_count].node);
+  if (a_byte != b_byte) return a_byte < b_byte;
+  if (a->pattern_index != b->pattern_index) return a->pattern_index < b->pattern_index;
+  return a->heap_insert_order < b->heap_insert_order;
+}
+
+static void finished_state_sift_down(
+  QueryStateList *states,
+  uint32_t index,
+  const CaptureListPool *pool
+) {
+  uint32_t size = states->size;
+  while (true) {
+    uint32_t smallest = index;
+    uint32_t left = 2 * index + 1;
+    uint32_t right = 2 * index + 2;
+    if (left < size && finished_state_precedes(
+      array_get(states, left),
+      array_get(states, smallest),
+      pool
+    )) {
+      smallest = left;
+    }
+    if (right < size && finished_state_precedes(
+      array_get(states, right),
+      array_get(states, smallest),
+      pool
+    )) {
+      smallest = right;
+    }
+    if (smallest == index) break;
+    finished_state_swap(states, index, smallest);
+    index = smallest;
+  }
+}
+
+static void finished_state_sift_up(
+  QueryStateList *states,
+  uint32_t index,
+  const CaptureListPool *pool
+) {
+  while (index > 0) {
+    uint32_t parent = (index - 1) / 2;
+    if (finished_state_precedes(
+      array_get(states, index),
+      array_get(states, parent),
+      pool
+    )) {
+      finished_state_swap(states, index, parent);
+      index = parent;
+    } else {
+      break;
+    }
+  }
+}
+
+static inline void finished_state_pop(QueryStateList *states, const CaptureListPool *pool) {
+  if (states->size > 1) *array_front(states) = *array_back(states);
+  states->size--;
+  if (states->size > 0) finished_state_sift_down(states, 0, pool);
+}
+
+// Remove an element at an arbitrary index and restore heap order.
+static void finished_state_erase(
+  QueryStateList *states,
+  uint32_t index,
+  const CaptureListPool *pool
+) {
+  if (index == states->size - 1) {
+    states->size--;
+    return;
+  }
+  *array_get(states, index) = *array_back(states);
+  states->size--;
+  // The replacement element may need to go up or down.
+  if (index > 0 && finished_state_precedes(
+    array_get(states, index),
+    array_get(states, (index - 1) / 2),
+    pool
+  )) {
+    finished_state_sift_up(states, index, pool);
+  } else {
+    finished_state_sift_down(states, index, pool);
+  }
+}
+
+static void ts_query_cursor__push_finished_state(
+  TSQueryCursor *self,
+  QueryState *state
+) {
+  state->heap_insert_order = self->next_finished_state_id++;
+  array_push(&self->finished_states, *state);
+}
+
+static void ts_query_cursor__heapify_finished_states(TSQueryCursor *self) {
+  while (self->finished_states_heap_size < self->finished_states.size) {
+    finished_state_sift_up(
+      &self->finished_states,
+      self->finished_states_heap_size,
+      &self->capture_list_pool
+    );
+    self->finished_states_heap_size++;
+  }
+}
+
 /**************
  * Quantifiers
  **************/
@@ -816,17 +961,8 @@ static QueryStep query_step__new(
   QueryStep step = {
     .symbol = symbol,
     .depth = depth,
-    .field = 0,
     .alternative_index = NONE,
-    .negated_field_list_id = 0,
-    .contains_captures = false,
-    .is_last_child = false,
-    .is_named = false,
-    .is_pass_through = false,
-    .is_dead_end = false,
-    .root_pattern_guaranteed = false,
     .is_immediate = is_immediate,
-    .alternative_is_immediate = false,
   };
   for (unsigned i = 0; i < MAX_STEP_CAPTURE_COUNT; i++) {
     step.capture_ids[i] = NONE;
@@ -2231,6 +2367,7 @@ static TSQueryError ts_query__parse_pattern(
   Stream *stream,
   uint32_t depth,
   bool is_immediate,
+  bool is_inside_alternation,
   CaptureQuantifiers *capture_quantifiers
 ) {
   if (stream->next == 0) return TSQueryErrorSyntax;
@@ -2264,6 +2401,7 @@ static TSQueryError ts_query__parse_pattern(
         stream,
         depth,
         is_immediate,
+        true,
         &branch_capture_quantifiers
       );
 
@@ -2331,6 +2469,7 @@ static TSQueryError ts_query__parse_pattern(
           stream,
           depth,
           child_is_immediate,
+          is_inside_alternation,
           &child_capture_quantifiers
         );
         if (e == PARENT_DONE) {
@@ -2569,6 +2708,7 @@ static TSQueryError ts_query__parse_pattern(
           stream,
           depth + 1,
           child_is_immediate,
+          is_inside_alternation,
           &child_capture_quantifiers
         );
         // In the event we only parsed a predicate, meaning no new steps were added,
@@ -2680,6 +2820,7 @@ static TSQueryError ts_query__parse_pattern(
       stream,
       depth,
       is_immediate,
+      is_inside_alternation,
       &field_capture_quantifiers
     );
     if (e) {
@@ -2798,16 +2939,16 @@ static TSQueryError ts_query__parse_pattern(
   switch (quantifier) {
     case TSQuantifierOneOrMore:
       repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false);
+      repeat_step.is_inside_alternation = is_inside_alternation;
       repeat_step.alternative_index = starting_step_index;
       repeat_step.is_pass_through = true;
-      repeat_step.alternative_is_immediate = true;
       array_push(&self->steps, repeat_step);
       break;
     case TSQuantifierZeroOrMore:
       repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false);
+      repeat_step.is_inside_alternation = is_inside_alternation;
       repeat_step.alternative_index = starting_step_index;
       repeat_step.is_pass_through = true;
-      repeat_step.alternative_is_immediate = true;
       array_push(&self->steps, repeat_step);
 
       // Stop when `step->alternative_index` is `NONE` or it points to
@@ -2884,7 +3025,7 @@ TSQuery *ts_query_new(
       .is_non_local = false,
     }));
     CaptureQuantifiers capture_quantifiers = capture_quantifiers_new();
-    *error_type = ts_query__parse_pattern(self, &stream, 0, false, &capture_quantifiers);
+    *error_type = ts_query__parse_pattern(self, &stream, 0, false, false, &capture_quantifiers);
     array_push(&self->steps, query_step__new(0, PATTERN_DONE_MARKER, false));
 
     QueryPattern *pattern = array_back(&self->patterns);
@@ -2958,6 +3099,53 @@ TSQuery *ts_query_new(
         break;
       }
     }
+
+    // Fix up quantifier loop-backs within alternations. When a branch of an
+    // alternation has a + or * quantifier, the quantifier's pass_through step
+    // loops back to the branch's first step. However, the alternation linking
+    // assigns that same step's `alternative_index` to point to the _next_ branch.
+    // This causes the quantifier loop to incorrectly explore other alternation branches,
+    // when a quantified branch matches, loops back, and then fails to match. To correct
+    // this, we create "clean" copies of the branches' first steps without the link to the
+    // next branch. After a quantified branch matches, it loops back to the cleaned copy.
+    {
+      uint32_t pat_start = pattern->steps.offset;
+      uint32_t pat_end = pat_start + pattern->steps.length - 1; // exclude DONE
+
+      for (uint32_t i = pat_start; i < pat_end; i++) {
+        QueryStep *s = array_get(&self->steps, i);
+        // Ensure this step is a pass_through with a _backward_ alternative (a quantifier loop-back)
+        if (!s->is_pass_through || !s->is_inside_alternation
+            || s->alternative_index == NONE || s->alternative_index >= i) continue;
+
+        uint32_t target_idx = s->alternative_index;
+        QueryStep *target = array_get(&self->steps, target_idx);
+
+        // Check if the target has a forward alternative from alternation linking
+        uint16_t target_alt_index = target->alternative_index;
+        if (target_alt_index == NONE
+            || target_alt_index <= target_idx || target_alt_index >= pat_end) continue;
+
+        // Create a clean copy of the target step without the alternation alternative.
+        uint32_t copy_idx = self->steps.size;
+        QueryStep copy = *target;
+        copy.alternative_index = NONE;
+        uint16_t target_depth = target->depth;
+        array_push(&self->steps, copy);
+
+        // Add a dead_end that redirects to the pass through step after the target,
+        // so the pattern continues correctly after the cleaned copy matches.
+        QueryStep redirect = query_step__new(0, target_depth, false);
+        redirect.is_dead_end = true;
+        redirect.alternative_index = target_idx + 1;
+        array_push(&self->steps, redirect);
+
+        // Update the pass_through to loop back to the copy. Reacquire `s` since
+        // `self->steps` may have been reallocated.
+        s = array_get(&self->steps, i);
+        s->alternative_index = copy_idx;
+      }
+    }
   }
 
   if (!ts_query__analyze_patterns(self, error_offset)) {
@@ -3099,12 +3287,18 @@ bool ts_query__step_is_fallible(
   const TSQuery *self,
   uint16_t step_index
 ) {
-  ts_assert((uint32_t)step_index + 1 < self->steps.size);
+  unsigned i = 1;
   QueryStep *step = array_get(&self->steps, step_index);
-  QueryStep *next_step = array_get(&self->steps, step_index + 1);
+  QueryStep *next_step;
+  do {
+    ts_assert((uint32_t)step_index + i < self->steps.size);
+    next_step = array_get(&self->steps, step_index + i);
+    i++;
+  } while (next_step->is_pass_through);
   return (
     next_step->depth != PATTERN_DONE_MARKER &&
-    next_step->depth > step->depth &&
+    (next_step->depth > step->depth ||
+        (next_step->depth == step->depth && next_step->is_immediate)) &&
     (!next_step->parent_pattern_guaranteed || step->symbol == WILDCARD_SYMBOL)
   );
 }
@@ -3232,10 +3426,12 @@ void ts_query_cursor_exec(
 
   array_clear(&self->states);
   array_clear(&self->finished_states);
+  self->finished_states_heap_size = 0;
   ts_tree_cursor_reset(&self->cursor, node);
   capture_list_pool_reset(&self->capture_list_pool);
   self->on_visible_node = true;
   self->next_state_id = 0;
+  self->next_finished_state_id = 0;
   self->depth = 0;
   self->ascending = false;
   self->halted = false;
@@ -3507,7 +3703,8 @@ static void ts_query_cursor__add_state(
   );
   array_insert(&self->states, index, ((QueryState) {
     .id = UINT32_MAX,
-    .capture_list_id = NONE,
+    .capture_list_id = CAPTURE_LIST_NONE,
+    .heap_insert_order = UINT32_MAX,
     .step_index = pattern->step_index,
     .pattern_index = pattern->pattern_index,
     .start_depth = start_depth,
@@ -3527,13 +3724,13 @@ static CaptureList *ts_query_cursor__prepare_to_capture(
   QueryState *state,
   unsigned state_index_to_preserve
 ) {
-  if (state->capture_list_id == NONE) {
+  if (state->capture_list_id == CAPTURE_LIST_NONE) {
     state->capture_list_id = capture_list_pool_acquire(&self->capture_list_pool);
 
     // If there are no capture lists left in the pool, then terminate whichever
     // state has captured the earliest node in the document, and steal its
     // capture list.
-    if (state->capture_list_id == NONE) {
+    if (state->capture_list_id == CAPTURE_LIST_NONE) {
       self->did_exceed_match_limit = true;
       uint32_t state_index, byte_offset, pattern_index;
       if (
@@ -3552,7 +3749,7 @@ static CaptureList *ts_query_cursor__prepare_to_capture(
         );
         QueryState *other_state = array_get(&self->states, state_index);
         state->capture_list_id = other_state->capture_list_id;
-        other_state->capture_list_id = NONE;
+        other_state->capture_list_id = CAPTURE_LIST_NONE;
         other_state->dead = true;
         CaptureList *list = capture_list_pool_get_mut(
           &self->capture_list_pool,
@@ -3606,10 +3803,10 @@ static QueryState *ts_query_cursor__copy_state(
   const QueryState *state = *state_ref;
   uint32_t state_index = (uint32_t)(state - self->states.contents);
   QueryState copy = *state;
-  copy.capture_list_id = NONE;
+  copy.capture_list_id = CAPTURE_LIST_NONE;
 
   // If the state has captures, copy its capture list.
-  if (state->capture_list_id != NONE) {
+  if (state->capture_list_id != CAPTURE_LIST_NONE) {
     CaptureList *new_captures = ts_query_cursor__prepare_to_capture(self, &copy, state_index);
     if (!new_captures) return NULL;
     const CaptureList *old_captures = capture_list_pool_get(
@@ -3765,7 +3962,7 @@ static inline bool ts_query_cursor__advance(
             (state->start_depth > self->depth || self->depth == 0)
           ) {
             LOG("  finish pattern %u\n", state->pattern_index);
-            array_push(&self->finished_states, *state);
+            ts_query_cursor__push_finished_state(self, state);
             did_match = true;
             deleted_count++;
           }
@@ -4130,17 +4327,17 @@ static inline bool ts_query_cursor__advance(
               QueryState *copy = ts_query_cursor__copy_state(self, &child_state);
               if (copy) {
                 LOG(
-                  "  split state for branch. pattern:%u, from_step:%u, to_step:%u, immediate:%d, capture_count: %u\n",
+                  "  split state for branch. pattern:%u, from_step:%u, to_step:%u, pass_through:%d, capture_count:%u\n",
                   copy->pattern_index,
                   copy->step_index,
                   next_step->alternative_index,
-                  next_step->alternative_is_immediate,
+                  next_step->is_pass_through,
                   capture_list_pool_get(&self->capture_list_pool, copy->capture_list_id)->size
                 );
                 end_index++;
                 copy_count++;
                 copy->step_index = child_step->alternative_index;
-                if (child_step->alternative_is_immediate) {
+                if (child_step->is_pass_through) {
                   copy->seeking_immediate_match = true;
                 }
               }
@@ -4181,7 +4378,10 @@ static inline bool ts_query_cursor__advance(
               &right_contains_left
             );
             if (left_contains_right) {
-              if (state->step_index == other_state->step_index) {
+              if (
+                state->step_index == other_state->step_index &&
+                (other_state->seeking_immediate_match || !state->seeking_immediate_match)
+              ) {
                 LOG(
                   "  drop shorter state. pattern: %u, step_index: %u\n",
                   state->pattern_index,
@@ -4195,7 +4395,10 @@ static inline bool ts_query_cursor__advance(
               other_state->has_in_progress_alternatives = true;
             }
             if (right_contains_left) {
-              if (state->step_index == other_state->step_index) {
+              if (
+                state->step_index == other_state->step_index &&
+                (state->seeking_immediate_match || !other_state->seeking_immediate_match)
+              ) {
                 LOG(
                   "  drop shorter state. pattern: %u, step_index: %u\n",
                   state->pattern_index,
@@ -4227,7 +4430,7 @@ static inline bool ts_query_cursor__advance(
                 LOG("  defer finishing pattern %u\n", state->pattern_index);
               } else {
                 LOG("  finish pattern %u\n", state->pattern_index);
-                array_push(&self->finished_states, *state);
+                ts_query_cursor__push_finished_state(self, state);
                 array_erase(&self->states, (uint32_t)(state - self->states.contents));
                 did_match = true;
                 j--;
@@ -4265,8 +4468,22 @@ bool ts_query_cursor_next_match(
       return false;
     }
   }
+  if (self->finished_states_heap_size > 0) {
+    ts_query_cursor__heapify_finished_states(self);
+  }
 
-  QueryState *state = array_get(&self->finished_states, 0);
+  uint32_t state_index = 0;
+  if (self->finished_states_heap_size > 0) {
+    for (uint32_t i = 1; i < self->finished_states.size; i++) {
+      QueryState *state = array_get(&self->finished_states, i);
+      QueryState *earliest_state = array_get(&self->finished_states, state_index);
+      if (state->heap_insert_order < earliest_state->heap_insert_order) {
+        state_index = i;
+      }
+    }
+  }
+
+  QueryState *state = array_get(&self->finished_states, state_index);
   if (state->id == UINT32_MAX) state->id = self->next_state_id++;
   match->id = state->id;
   match->pattern_index = state->pattern_index;
@@ -4277,7 +4494,12 @@ bool ts_query_cursor_next_match(
   match->captures = captures->contents;
   match->capture_count = captures->size;
   capture_list_pool_release(&self->capture_list_pool, state->capture_list_id);
-  array_erase(&self->finished_states, 0);
+  if (self->finished_states_heap_size > 0) {
+    finished_state_erase(&self->finished_states, state_index, &self->capture_list_pool);
+    self->finished_states_heap_size = self->finished_states.size;
+  } else {
+    array_erase(&self->finished_states, state_index);
+  }
   return true;
 }
 
@@ -4285,6 +4507,10 @@ void ts_query_cursor_remove_match(
   TSQueryCursor *self,
   uint32_t match_id
 ) {
+  if (self->finished_states_heap_size > 0) {
+    ts_query_cursor__heapify_finished_states(self);
+  }
+
   for (unsigned i = 0; i < self->finished_states.size; i++) {
     const QueryState *state = array_get(&self->finished_states, i);
     if (state->id == match_id) {
@@ -4292,7 +4518,12 @@ void ts_query_cursor_remove_match(
         &self->capture_list_pool,
         state->capture_list_id
       );
-      array_erase(&self->finished_states, i);
+      if (self->finished_states_heap_size > 0) {
+        finished_state_erase(&self->finished_states, i, &self->capture_list_pool);
+        self->finished_states_heap_size = self->finished_states.size;
+      } else {
+        array_erase(&self->finished_states, i);
+      }
       return;
     }
   }
@@ -4321,6 +4552,9 @@ bool ts_query_cursor_next_capture(
   // be discovered in order, because patterns can overlap. Search for matches
   // until there is a finished capture that is before any unfinished capture.
   for (;;) {
+    // Sift any newly pushed finished states into the heap.
+    ts_query_cursor__heapify_finished_states(self);
+
     // First, find the earliest capture in an unfinished match.
     uint32_t first_unfinished_capture_byte;
     uint32_t first_unfinished_pattern_index;
@@ -4334,13 +4568,14 @@ bool ts_query_cursor_next_capture(
       &first_unfinished_state_is_definite
     );
 
-    // Then find the earliest capture in a finished match. It must occur
-    // before the first capture in an *unfinished* match.
+    // Then find the earliest capture in a finished match. The finished_states
+    // array is maintained as a min-heap, so the earliest is always at index 0.
+    // Clean up fully-consumed and out-of-range states from the heap root first.
     QueryState *first_finished_state = NULL;
     uint32_t first_finished_capture_byte = first_unfinished_capture_byte;
     uint32_t first_finished_pattern_index = first_unfinished_pattern_index;
-    for (unsigned i = 0; i < self->finished_states.size;) {
-      QueryState *state = array_get(&self->finished_states, i);
+    while (self->finished_states.size > 0) {
+      QueryState *state = array_get(&self->finished_states, 0);
       const CaptureList *captures = capture_list_pool_get(
         &self->capture_list_pool,
         state->capture_list_id
@@ -4352,7 +4587,8 @@ bool ts_query_cursor_next_capture(
           &self->capture_list_pool,
           state->capture_list_id
         );
-        array_erase(&self->finished_states, i);
+        finished_state_pop(&self->finished_states, &self->capture_list_pool);
+        self->finished_states_heap_size = self->finished_states.size;
         continue;
       }
 
@@ -4371,6 +4607,7 @@ bool ts_query_cursor_next_capture(
       // Skip captures that are outside of the cursor's range.
       if (node_outside_of_range) {
         state->consumed_capture_count++;
+        finished_state_sift_down(&self->finished_states, 0, &self->capture_list_pool);
         continue;
       }
 
@@ -4386,7 +4623,7 @@ bool ts_query_cursor_next_capture(
         first_finished_capture_byte = node_start_byte;
         first_finished_pattern_index = state->pattern_index;
       }
-      i++;
+      break;
     }
 
     // If there is finished capture that is clearly before any unfinished
@@ -4413,6 +4650,11 @@ bool ts_query_cursor_next_capture(
       match->capture_count = captures->size;
       *capture_index = state->consumed_capture_count;
       state->consumed_capture_count++;
+      // If this state is in the finished_states heap, its sort key has changed
+      // (next capture is now later in the document). Restore heap order.
+      if (state == first_finished_state) {
+        finished_state_sift_down(&self->finished_states, 0, &self->capture_list_pool);
+      }
       return true;
     }
 

lib/src/subtree.h

@@ -252,7 +252,7 @@ static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
 // Get a subtree's children, which are allocated immediately before the
 // tree's own heap data.
 #define ts_subtree_children(self) \
-  ((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
+  ((self).data.is_inline ? (Subtree *)NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
 
 static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) {
   if (self->data.is_inline) {

lib/src/tree_cursor.c

@@ -153,8 +153,10 @@ static inline bool ts_tree_cursor_child_iterator_previous(
 // TSTreeCursor - lifecycle
 
 TSTreeCursor ts_tree_cursor_new(TSNode node) {
-  TSTreeCursor self = {NULL, NULL, {0, 0, 0}};
-  ts_tree_cursor_init((TreeCursor *)&self, node);
+  TreeCursor cursor = {0};
+  ts_tree_cursor_init(&cursor, node);
+  TSTreeCursor self = {0};
+  memcpy(&self, &cursor, sizeof(cursor));
   return self;
 }
 
@@ -697,12 +699,13 @@ const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
 
 TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
   const TreeCursor *cursor = (const TreeCursor *)_cursor;
-  TSTreeCursor res = {NULL, NULL, {0, 0}};
-  TreeCursor *copy = (TreeCursor *)&res;
-  copy->tree = cursor->tree;
-  copy->root_alias_symbol = cursor->root_alias_symbol;
-  array_init(&copy->stack);
-  array_push_all(&copy->stack, &cursor->stack);
+  TreeCursor copy = {0};
+  copy.tree = cursor->tree;
+  copy.root_alias_symbol = cursor->root_alias_symbol;
+  array_init(&copy.stack);
+  array_push_all(&copy.stack, &cursor->stack);
+  TSTreeCursor res = {0};
+  memcpy(&res, &copy, sizeof(copy));
   return res;
 }
 

lib/src/unicode.h

@@ -53,8 +53,14 @@ static inline uint32_t ts_decode_utf16_le(
   uint32_t length,
   int32_t *code_point
 ) {
+  if (length < 2) {
+    *code_point = TS_DECODE_ERROR;
+    return length;
+  }
   uint32_t i = 0;
-  U16_NEXT_LE(((uint16_t *)string), i, length, *code_point);
+  // length is in bytes; U16_NEXT indexes into uint16_t*, so its length
+  // parameter must be in code units (length / 2), not bytes.
+  U16_NEXT_LE(((uint16_t *)string), i, length / 2, *code_point);
   return i * 2;
 }
 
@@ -63,8 +69,14 @@ static inline uint32_t ts_decode_utf16_be(
   uint32_t length,
   int32_t *code_point
 ) {
+  if (length < 2) {
+    *code_point = TS_DECODE_ERROR;
+    return length;
+  }
   uint32_t i = 0;
-  U16_NEXT_BE(((uint16_t *)string), i, length, *code_point);
+  // length is in bytes; U16_NEXT indexes into uint16_t*, so its length
+  // parameter must be in code units (length / 2), not bytes.
+  U16_NEXT_BE(((uint16_t *)string), i, length / 2, *code_point);
   return i * 2;
 }
 

lib/src/wasm_store.c

@@ -80,15 +80,15 @@ typedef struct {
 } LanguageWasmInstance;
 
 typedef struct {
-  uint32_t reset_heap;
-  uint32_t proc_exit;
-  uint32_t abort;
-  uint32_t assert_fail;
-  uint32_t notify_memory_growth;
-  uint32_t debug_message;
-  uint32_t at_exit;
-  uint32_t args_get;
-  uint32_t args_sizes_get;
+  wasmtime_func_t reset_heap;
+  wasmtime_func_t proc_exit;
+  wasmtime_func_t abort;
+  wasmtime_func_t assert_fail;
+  wasmtime_func_t notify_memory_growth;
+  wasmtime_func_t debug_message;
+  wasmtime_func_t at_exit;
+  wasmtime_func_t args_get;
+  wasmtime_func_t args_sizes_get;
 } BuiltinFunctionIndices;
 
 // TSWasmStore - A struct that allows a given `Parser` to use Wasm-backed
@@ -104,7 +104,7 @@ struct TSWasmStore {
   Array(LanguageWasmInstance) language_instances;
   uint32_t current_memory_offset;
   uint32_t current_function_table_offset;
-  uint32_t *stdlib_fn_indices;
+  wasmtime_func_t *stdlib_fn_indices;
   BuiltinFunctionIndices builtin_fn_indices;
   wasmtime_global_t stack_pointer_global;
   wasm_globaltype_t *const_i32_type;
@@ -255,7 +255,7 @@ static bool wasm_dylink_info__parse(
  * Native callbacks exposed to Wasm modules
  *******************************************/
 
- static wasm_trap_t *callback__abort(
+static wasm_trap_t *callback__abort(
   void *env,
   wasmtime_caller_t* caller,
   wasmtime_val_raw_t *args_and_results,
@@ -360,23 +360,65 @@ static wasm_trap_t *callback__lexer_eof(
 }
 
 typedef struct {
-  uint32_t *storage_location;
+  void *storage_location;
   wasmtime_func_unchecked_callback_t callback;
   wasm_functype_t *type;
 } FunctionDefinition;
 
-static void *copy(const void *data, size_t size) {
+typedef struct {
+  const uint8_t *data;
+  size_t size;
+} WasmMemory;
+
+static bool wasm_memory__contains(const WasmMemory *memory, int32_t address, size_t size) {
+  if (address < 0) return false;
+  size_t start = (size_t)address;
+  return start <= memory->size && size <= memory->size - start;
+}
+
+static bool wasm_memory__read(const WasmMemory *memory, int32_t address, void *result, size_t size) {
+  if (!wasm_memory__contains(memory, address, size)) return false;
+  memcpy(result, &memory->data[address], size);
+  return true;
+}
+
+static bool wasm_memory__string_length(const WasmMemory *memory, int32_t address, size_t *length) {
+  if (address < 0 || (size_t)address >= memory->size) return false;
+  const uint8_t *data = &memory->data[address];
+  size_t limit = memory->size - (size_t)address;
+  for (size_t i = 0; i < limit; i++) {
+    if (data[i] == 0) {
+      *length = i;
+      return true;
+    }
+  }
+  return false;
+}
+
+static void *copy(const WasmMemory *memory, int32_t address, size_t size, bool *ok) {
+  if (!*ok || size == 0) return NULL;
+  if (!wasm_memory__contains(memory, address, size)) {
+    *ok = false;
+    return NULL;
+  }
   void *result = ts_malloc(size);
-  memcpy(result, data, size);
+  memcpy(result, &memory->data[address], size);
   return result;
 }
 
 static void *copy_unsized_static_array(
-  const uint8_t *data,
+  const WasmMemory *memory,
   int32_t start_address,
   const int32_t all_addresses[],
-  size_t address_count
+  size_t address_count,
+  bool *ok
 ) {
+  if (!*ok || start_address == 0) return NULL;
+  if (start_address < 0) {
+    *ok = false;
+    return NULL;
+  }
+
   int32_t end_address = 0;
   for (unsigned i = 0; i < address_count; i++) {
     if (all_addresses[i] > start_address) {
@@ -388,28 +430,48 @@ static void *copy_unsized_static_array(
 
   if (!end_address) return NULL;
   size_t size = end_address - start_address;
+  if (!wasm_memory__contains(memory, start_address, size)) {
+    *ok = false;
+    return NULL;
+  }
   void *result = ts_malloc(size);
-  memcpy(result, &data[start_address], size);
+  memcpy(result, &memory->data[start_address], size);
   return result;
 }
 
 static void *copy_strings(
-  const uint8_t *data,
+  const WasmMemory *memory,
   int32_t array_address,
   size_t count,
-  StringData *string_data
+  StringData *string_data,
+  bool *ok
 ) {
+  if (!*ok) return NULL;
+  if (count > SIZE_MAX / sizeof(char *)) {
+    *ok = false;
+    return NULL;
+  }
+  if (count > (SIZE_MAX / sizeof(int32_t)) ||
+      !wasm_memory__contains(memory, array_address, count * sizeof(int32_t))) {
+    *ok = false;
+    return NULL;
+  }
+
   const char **result = ts_malloc(count * sizeof(char *));
   for (unsigned i = 0; i < count; i++) {
     int32_t address;
-    memcpy(&address, &data[array_address + i * sizeof(address)], sizeof(address));
+    memcpy(&address, &memory->data[array_address + i * sizeof(address)], sizeof(address));
     if (address == 0) {
       result[i] = (const char *)-1;
     } else {
-      const uint8_t *string = &data[address];
-      uint32_t len = strlen((const char *)string);
+      size_t len;
+      if (!wasm_memory__string_length(memory, address, &len) || len > UINT32_MAX) {
+        ts_free(result);
+        *ok = false;
+        return NULL;
+      }
       result[i] = (const char *)(uintptr_t)string_data->size;
-      array_extend(string_data, len + 1, string);
+      array_extend(string_data, len + 1, &memory->data[address]);
     }
   }
   for (unsigned i = 0; i < count; i++) {
@@ -423,16 +485,54 @@ static void *copy_strings(
 }
 
 static void *copy_string(
-  const uint8_t *data,
-  int32_t address
+  const WasmMemory *memory,
+  int32_t address,
+  bool *ok
 ) {
-  const char *string = (const char *)&data[address];
-  size_t len = strlen(string);
+  if (!*ok) return NULL;
+  size_t len;
+  if (!wasm_memory__string_length(memory, address, &len)) {
+    *ok = false;
+    return NULL;
+  }
+  const char *string = (const char *)&memory->data[address];
   char *result = ts_malloc(len + 1);
   memcpy(result, string, len + 1);
   return result;
 }
 
+static void delete_partially_loaded_language(
+  TSLanguage *language,
+  StringData *symbol_name_buffer,
+  StringData *field_name_buffer
+) {
+  if (language) {
+    ts_free((void *)language->alias_map);
+    ts_free((void *)language->alias_sequences);
+    ts_free((void *)language->external_scanner.symbol_map);
+    ts_free((void *)language->field_map_entries);
+    ts_free((void *)language->field_map_slices);
+    ts_free((void *)language->field_names);
+    ts_free((void *)language->lex_modes);
+    ts_free((void *)language->name);
+    ts_free((void *)language->parse_actions);
+    ts_free((void *)language->parse_table);
+    ts_free((void *)language->primary_state_ids);
+    ts_free((void *)language->public_symbol_map);
+    ts_free((void *)language->reserved_words);
+    ts_free((void *)language->small_parse_table);
+    ts_free((void *)language->small_parse_table_map);
+    ts_free((void *)language->supertype_map_entries);
+    ts_free((void *)language->supertype_map_slices);
+    ts_free((void *)language->supertype_symbols);
+    ts_free((void *)language->symbol_metadata);
+    ts_free((void *)language->symbol_names);
+    ts_free(language);
+  }
+  array_delete(symbol_name_buffer);
+  array_delete(field_name_buffer);
+}
+
 static bool name_eq(const wasm_name_t *name, const char *string) {
   return strncmp(string, name->data, name->size) == 0;
 }
@@ -476,15 +576,11 @@ void language_id_delete(WasmLanguageId *self) {
 }
 
 static wasmtime_extern_t get_builtin_extern(
-  wasmtime_table_t *table,
-  unsigned index
+  wasmtime_func_t *func
 ) {
   return (wasmtime_extern_t) {
     .kind = WASMTIME_EXTERN_FUNC,
-    .of.func = (wasmtime_func_t) {
-      .store_id = table->store_id,
-      .__private = index
-    }
+    .of.func = *func
   };
 }
 
@@ -519,21 +615,21 @@ static bool ts_wasm_store__provide_builtin_import(
 
   // Builtin functions
   else if (name_eq(import_name, "__assert_fail")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.assert_fail);
+    *import = get_builtin_extern(&self->builtin_fn_indices.assert_fail);
   } else if (name_eq(import_name, "__cxa_atexit")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.at_exit);
+    *import = get_builtin_extern(&self->builtin_fn_indices.at_exit);
   } else if (name_eq(import_name, "args_get")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_get);
+    *import = get_builtin_extern(&self->builtin_fn_indices.args_get);
   } else if (name_eq(import_name, "args_sizes_get")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_sizes_get);
+    *import = get_builtin_extern(&self->builtin_fn_indices.args_sizes_get);
   } else if (name_eq(import_name, "abort")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.abort);
+    *import = get_builtin_extern(&self->builtin_fn_indices.abort);
   } else if (name_eq(import_name, "proc_exit")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.proc_exit);
+    *import = get_builtin_extern(&self->builtin_fn_indices.proc_exit);
   } else if (name_eq(import_name, "emscripten_notify_memory_growth")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.notify_memory_growth);
+    *import = get_builtin_extern(&self->builtin_fn_indices.notify_memory_growth);
   } else if (name_eq(import_name, "tree_sitter_debug_message")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.debug_message);
+    *import = get_builtin_extern(&self->builtin_fn_indices.debug_message);
   } else {
     return false;
   }
@@ -575,6 +671,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   wasmtime_module_t *stdlib_module = NULL;
   wasm_memorytype_t *memory_type = NULL;
   wasm_tabletype_t *table_type = NULL;
+  wasmtime_func_t *lexer_funcs = NULL;
 
   // Define functions called by scanners via function pointers on the lexer.
   LexerInWasmMemory lexer = {
@@ -583,34 +680,34 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   };
   FunctionDefinition lexer_definitions[] = {
     {
-      (uint32_t *)&lexer.advance,
+      &lexer.advance,
       callback__lexer_advance,
       wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.mark_end,
+      &lexer.mark_end,
       callback__lexer_mark_end,
       wasm_functype_new_1_0(wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.get_column,
+      &lexer.get_column,
       callback__lexer_get_column,
       wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.is_at_included_range_start,
+      &lexer.is_at_included_range_start,
       callback__lexer_is_at_included_range_start,
       wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.eof,
+      &lexer.eof,
       callback__lexer_eof,
       wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
   };
 
   // Define builtin functions that can be imported by scanners.
-  BuiltinFunctionIndices builtin_fn_indices;
+  BuiltinFunctionIndices builtin_fn_indices = {0};
   FunctionDefinition builtin_definitions[] = {
     {
       &builtin_fn_indices.proc_exit,
@@ -657,18 +754,16 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   // Create all of the Wasm functions.
   unsigned builtin_definitions_len = array_len(builtin_definitions);
   unsigned lexer_definitions_len = array_len(lexer_definitions);
+  lexer_funcs = ts_calloc(lexer_definitions_len, sizeof(wasmtime_func_t));
   for (unsigned i = 0; i < builtin_definitions_len; i++) {
     FunctionDefinition *definition = &builtin_definitions[i];
-    wasmtime_func_t func;
-    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
-    *definition->storage_location = func.__private;
+    wasmtime_func_t *func = (wasmtime_func_t *)definition->storage_location;
+    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, func);
     wasm_functype_delete(definition->type);
   }
   for (unsigned i = 0; i < lexer_definitions_len; i++) {
     FunctionDefinition *definition = &lexer_definitions[i];
-    wasmtime_func_t func;
-    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
-    *definition->storage_location = func.__private;
+    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &lexer_funcs[i]);
     wasm_functype_delete(definition->type);
   }
 
@@ -763,7 +858,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
     .memory = memory,
     .function_table = function_table,
     .language_instances = array_new(),
-    .stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(uint32_t)),
+    .stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(wasmtime_func_t)),
     .builtin_fn_indices = builtin_fn_indices,
     .stack_pointer_global = stack_pointer_global,
     .current_memory_offset = 0,
@@ -816,7 +911,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
 
   // Process the stdlib module's exports.
   for (unsigned i = 0; i < stdlib_symbols_len; i++) {
-    self->stdlib_fn_indices[i] = UINT32_MAX;
+    self->stdlib_fn_indices[i] = (wasmtime_func_t){.store_id = 0};
   }
   wasmtime_module_exports(stdlib_module, &export_types);
   for (unsigned i = 0; i < export_types.size; i++) {
@@ -851,20 +946,20 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
       }
 
       if (name_eq(name, "reset_heap")) {
-        self->builtin_fn_indices.reset_heap = export.of.func.__private;
+        self->builtin_fn_indices.reset_heap = export.of.func;
         continue;
       }
 
       for (unsigned j = 0; j < stdlib_symbols_len; j++) {
         if (name_eq(name, STDLIB_SYMBOLS[j])) {
-          self->stdlib_fn_indices[j] = export.of.func.__private;
+          self->stdlib_fn_indices[j] = export.of.func;
           break;
         }
       }
     }
   }
 
-  if (self->builtin_fn_indices.reset_heap == UINT32_MAX) {
+  if (self->builtin_fn_indices.reset_heap.store_id == 0) {
     wasm_error->kind = TSWasmErrorKindInstantiate;
     format(
       &wasm_error->message,
@@ -874,7 +969,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   }
 
   for (unsigned i = 0; i < stdlib_symbols_len; i++) {
-    if (self->stdlib_fn_indices[i] == UINT32_MAX) {
+    if (self->stdlib_fn_indices[i].store_id == 0) {
       wasm_error->kind = TSWasmErrorKindInstantiate;
       format(
         &wasm_error->message,
@@ -904,13 +999,13 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   }
   for (unsigned i = 0; i < lexer_definitions_len; i++) {
     FunctionDefinition *definition = &lexer_definitions[i];
-    wasmtime_func_t func = {function_table.store_id, *definition->storage_location};
-    wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = func};
+    wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = lexer_funcs[i]};
     error = wasmtime_table_set(context, &function_table, table_index, &func_val);
     ts_assert(!error);
     *(int32_t *)(definition->storage_location) = table_index;
     table_index++;
   }
+  ts_free(lexer_funcs);
 
   self->current_function_table_offset = table_index;
   self->lexer_address = initial_memory_pages * MEMORY_PAGE_SIZE;
@@ -937,6 +1032,7 @@ error:
   if (message.size) wasm_byte_vec_delete(&message);
   if (export_types.size) wasm_exporttype_vec_delete(&export_types);
   if (imports) ts_free(imports);
+  ts_free(lexer_funcs);
   return NULL;
 }
 
@@ -1016,8 +1112,6 @@ static bool ts_wasm_store__instantiate(
   // Construct the language function name as string.
   format(&language_function_name, "tree_sitter_%s", language_name);
 
-  const uint64_t store_id = self->function_table.store_id;
-
   // Build the imports list for the module.
   wasm_importtype_vec_t import_types = WASM_EMPTY_VEC;
   wasmtime_module_imports(module, &import_types);
@@ -1038,8 +1132,7 @@ static bool ts_wasm_store__instantiate(
     bool defined_in_stdlib = false;
     for (unsigned j = 0; j < array_len(STDLIB_SYMBOLS); j++) {
       if (name_eq(import_name, STDLIB_SYMBOLS[j])) {
-        uint16_t address = self->stdlib_fn_indices[j];
-        imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = {store_id, address}};
+        imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = self->stdlib_fn_indices[j]};
         defined_in_stdlib = true;
         break;
       }
@@ -1179,6 +1272,9 @@ const TSLanguage *ts_wasm_store_load_language(
   WasmDylinkInfo dylink_info;
   wasmtime_module_t *module = NULL;
   wasmtime_error_t *error = NULL;
+  TSLanguage *language = NULL;
+  StringData symbol_name_buffer = array_new();
+  StringData field_name_buffer = array_new();
   wasm_error->kind = TSWasmErrorKindNone;
 
   if (!wasm_dylink_info__parse((const unsigned char *)wasm, wasm_len, &dylink_info)) {
@@ -1219,10 +1315,17 @@ const TSLanguage *ts_wasm_store_load_language(
   LanguageInWasmMemory wasm_language;
   wasmtime_context_t *context = wasmtime_store_context(self->store);
   const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
-  memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory));
+  WasmMemory wasm_memory = {
+    .data = memory,
+    .size = wasmtime_memory_data_size(context, &self->memory),
+  };
+  bool valid_wasm_memory = true;
+  if (!wasm_memory__read(&wasm_memory, language_address, &wasm_language, sizeof(LanguageInWasmMemory))) {
+    goto invalid_language_memory;
+  }
 
   bool has_supertypes =
-    wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS &&
+    wasm_language.abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS &&
     wasm_language.supertype_count > 0;
 
   int32_t addresses[] = {
@@ -1259,10 +1362,7 @@ const TSLanguage *ts_wasm_store_load_language(
   };
   uint32_t address_count = array_len(addresses);
 
-  TSLanguage *language = ts_calloc(1, sizeof(TSLanguage));
-  StringData symbol_name_buffer = array_new();
-  StringData field_name_buffer = array_new();
-
+  language = ts_calloc(1, sizeof(TSLanguage));
   *language = (TSLanguage) {
     .abi_version = wasm_language.abi_version,
     .symbol_count = wasm_language.symbol_count,
@@ -1278,40 +1378,54 @@ const TSLanguage *ts_wasm_store_load_language(
     .keyword_capture_token = wasm_language.keyword_capture_token,
     .metadata = wasm_language.metadata,
     .parse_table = copy(
-      &memory[wasm_language.parse_table],
-      wasm_language.large_state_count * wasm_language.symbol_count * sizeof(uint16_t)
+      &wasm_memory,
+      wasm_language.parse_table,
+      wasm_language.large_state_count * wasm_language.symbol_count * sizeof(uint16_t),
+      &valid_wasm_memory
     ),
     .parse_actions = copy_unsized_static_array(
-      memory,
+      &wasm_memory,
       wasm_language.parse_actions,
       addresses,
-      address_count
+      address_count,
+      &valid_wasm_memory
     ),
     .symbol_names = copy_strings(
-      memory,
+      &wasm_memory,
       wasm_language.symbol_names,
       wasm_language.symbol_count + wasm_language.alias_count,
-      &symbol_name_buffer
+      &symbol_name_buffer,
+      &valid_wasm_memory
     ),
     .symbol_metadata = copy(
-      &memory[wasm_language.symbol_metadata],
-      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbolMetadata)
+      &wasm_memory,
+      wasm_language.symbol_metadata,
+      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbolMetadata),
+      &valid_wasm_memory
     ),
     .public_symbol_map = copy(
-      &memory[wasm_language.public_symbol_map],
-      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.public_symbol_map,
+      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbol),
+      &valid_wasm_memory
     ),
     .lex_modes = copy(
-      &memory[wasm_language.lex_modes],
-      wasm_language.state_count * sizeof(TSLexerMode)
+      &wasm_memory,
+      wasm_language.lex_modes,
+      wasm_language.state_count * sizeof(TSLexerMode),
+      &valid_wasm_memory
     ),
   };
+  if (!valid_wasm_memory) goto invalid_language_memory;
 
   if (language->field_count > 0 && language->production_id_count > 0) {
     language->field_map_slices = copy(
-      &memory[wasm_language.field_map_slices],
-      wasm_language.production_id_count * sizeof(TSMapSlice)
+      &wasm_memory,
+      wasm_language.field_map_slices,
+      wasm_language.production_id_count * sizeof(TSMapSlice),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
 
     // Determine the number of field map entries by finding the greatest index
     // in any of the slices.
@@ -1325,22 +1439,29 @@ const TSLanguage *ts_wasm_store_load_language(
     }
 
     language->field_map_entries = copy(
-      &memory[wasm_language.field_map_entries],
-      field_map_entry_count * sizeof(TSFieldMapEntry)
+      &wasm_memory,
+      wasm_language.field_map_entries,
+      field_map_entry_count * sizeof(TSFieldMapEntry),
+      &valid_wasm_memory
     );
     language->field_names = copy_strings(
-      memory,
+      &wasm_memory,
       wasm_language.field_names,
       wasm_language.field_count + 1,
-      &field_name_buffer
+      &field_name_buffer,
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (has_supertypes) {
     language->supertype_symbols = copy(
-      &memory[wasm_language.supertype_symbols],
-      wasm_language.supertype_count * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.supertype_symbols,
+      wasm_language.supertype_count * sizeof(TSSymbol),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
 
     // Determine the number of supertype map slices by finding the greatest
     // supertype ID.
@@ -1353,18 +1474,24 @@ const TSLanguage *ts_wasm_store_load_language(
     }
 
     language->supertype_map_slices = copy(
-      &memory[wasm_language.supertype_map_slices],
-      (largest_supertype + 1) * sizeof(TSMapSlice)
+      &wasm_memory,
+      wasm_language.supertype_map_slices,
+      (largest_supertype + 1) * sizeof(TSMapSlice),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
 
     TSSymbol last_supertype = language->supertype_symbols[language->supertype_count - 1];
     TSMapSlice last_slice = language->supertype_map_slices[last_supertype];
     uint32_t supertype_map_entry_count = last_slice.index + last_slice.length;
 
     language->supertype_map_entries = copy(
-      &memory[wasm_language.supertype_map_entries],
-      supertype_map_entry_count * sizeof(char *)
+      &wasm_memory,
+      wasm_language.supertype_map_entries,
+      supertype_map_entry_count * sizeof(char *),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->max_alias_sequence_length > 0 && language->production_id_count > 0) {
@@ -1372,59 +1499,95 @@ const TSLanguage *ts_wasm_store_load_language(
     int32_t alias_map_size = 0;
     for (;;) {
       TSSymbol symbol;
-      memcpy(&symbol, &memory[wasm_language.alias_map + alias_map_size], sizeof(symbol));
+      if (!wasm_memory__read(&wasm_memory, wasm_language.alias_map + alias_map_size, &symbol, sizeof(symbol))) {
+        goto invalid_language_memory;
+      }
       alias_map_size += sizeof(TSSymbol);
       if (symbol == 0) break;
       uint16_t value_count;
-      memcpy(&value_count, &memory[wasm_language.alias_map + alias_map_size], sizeof(value_count));
+      if (!wasm_memory__read(&wasm_memory, wasm_language.alias_map + alias_map_size, &value_count, sizeof(value_count))) {
+        goto invalid_language_memory;
+      }
       alias_map_size += sizeof(uint16_t);
       alias_map_size += value_count * sizeof(TSSymbol);
     }
     language->alias_map = copy(
-      &memory[wasm_language.alias_map],
-      alias_map_size
+      &wasm_memory,
+      wasm_language.alias_map,
+      alias_map_size,
+      &valid_wasm_memory
     );
     language->alias_sequences = copy(
-      &memory[wasm_language.alias_sequences],
-      wasm_language.production_id_count * wasm_language.max_alias_sequence_length * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.alias_sequences,
+      wasm_language.production_id_count * wasm_language.max_alias_sequence_length * sizeof(TSSymbol),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->state_count > language->large_state_count) {
     uint32_t small_state_count = wasm_language.state_count - wasm_language.large_state_count;
     language->small_parse_table_map = copy(
-      &memory[wasm_language.small_parse_table_map],
-      small_state_count * sizeof(uint32_t)
+      &wasm_memory,
+      wasm_language.small_parse_table_map,
+      small_state_count * sizeof(uint32_t),
+      &valid_wasm_memory
     );
     language->small_parse_table = copy_unsized_static_array(
-      memory,
+      &wasm_memory,
       wasm_language.small_parse_table,
       addresses,
-      address_count
+      address_count,
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) {
     language->primary_state_ids = copy(
-      &memory[wasm_language.primary_state_ids],
-      wasm_language.state_count * sizeof(TSStateId)
+      &wasm_memory,
+      wasm_language.primary_state_ids,
+      wasm_language.state_count * sizeof(TSStateId),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) {
-    language->name = copy_string(memory, wasm_language.name);
-    language->reserved_words = copy(
-        &memory[wasm_language.reserved_words],
-        wasm_language.max_reserved_word_set_size * sizeof(TSSymbol)
-    );
+    language->name = copy_string(&wasm_memory, wasm_language.name, &valid_wasm_memory);
+    if (!valid_wasm_memory) goto invalid_language_memory;
     language->max_reserved_word_set_size = wasm_language.max_reserved_word_set_size;
+
+    // Determine the number of reserved word sets by finding the maximum
+    // reserved_word_set_id across all lex modes.
+    uint16_t max_reserved_word_set_id = 0;
+    for (uint32_t i = 0; i < wasm_language.state_count; i++) {
+      uint16_t id = language->lex_modes[i].reserved_word_set_id;
+      if (id > max_reserved_word_set_id) max_reserved_word_set_id = id;
+    }
+
+    if (max_reserved_word_set_id > 0 && language->max_reserved_word_set_size > 0) {
+      uint32_t reserved_word_count =
+        (max_reserved_word_set_id + 1) * language->max_reserved_word_set_size;
+      language->reserved_words = copy(
+          &wasm_memory,
+          wasm_language.reserved_words,
+          reserved_word_count * sizeof(TSSymbol),
+          &valid_wasm_memory
+      );
+      if (!valid_wasm_memory) goto invalid_language_memory;
+    }
   }
 
   if (language->external_token_count > 0) {
     language->external_scanner.symbol_map = copy(
-      &memory[wasm_language.external_scanner.symbol_map],
-      wasm_language.external_token_count * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.external_scanner.symbol_map,
+      wasm_language.external_token_count * sizeof(TSSymbol),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
     language->external_scanner.states = (void *)(uintptr_t)wasm_language.external_scanner.states;
   }
 
@@ -1476,7 +1639,13 @@ const TSLanguage *ts_wasm_store_load_language(
 
   return language;
 
+invalid_language_memory:
+  wasm_error->kind = TSWasmErrorKindInstantiate;
+  format(&wasm_error->message, "invalid language memory address");
+  goto error;
+
 error:
+  delete_partially_loaded_language(language, &symbol_name_buffer, &field_name_buffer);
   if (module) wasmtime_module_delete(module);
   return NULL;
 }
@@ -1526,7 +1695,13 @@ bool ts_wasm_store_add_language(
 
     LanguageInWasmMemory wasm_language;
     const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
-    memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory));
+    WasmMemory wasm_memory = {
+      .data = memory,
+      .size = wasmtime_memory_data_size(context, &self->memory),
+    };
+    if (!wasm_memory__read(&wasm_memory, language_address, &wasm_language, sizeof(LanguageInWasmMemory))) {
+      return false;
+    }
     array_push(&self->language_instances, ((LanguageWasmInstance) {
       .language_id = language_id_clone(language_module->language_id),
       .instance = instance,
@@ -1546,16 +1721,13 @@ bool ts_wasm_store_add_language(
 
 void ts_wasm_store_reset_heap(TSWasmStore *self) {
   wasmtime_context_t *context = wasmtime_store_context(self->store);
-  wasmtime_func_t func = {
-    self->function_table.store_id,
-    self->builtin_fn_indices.reset_heap
-  };
+  wasmtime_func_t *func = &self->builtin_fn_indices.reset_heap;
   wasm_trap_t *trap = NULL;
   wasmtime_val_t args[1] = {
     {.of.i32 = ts_wasm_store__heap_address(self), .kind = WASMTIME_I32},
   };
 
-  wasmtime_error_t *error = wasmtime_func_call(context, &func, args, 1, NULL, 0, &trap);
+  wasmtime_error_t *error = wasmtime_func_call(context, func, args, 1, NULL, 0, &trap);
   ts_assert(!error);
   ts_assert(!trap);
 }

test/fixtures/fixtures.json

@@ -1,17 +1,17 @@
 [
-  ["bash","v0.25.0"],
-  ["c","v0.24.1"],
-  ["cpp","v0.23.4"],
-  ["embedded-template","v0.25.0"],
-  ["go","v0.25.0"],
-  ["html","v0.23.2"],
-  ["java","v0.23.5"],
-  ["javascript","v0.25.0"],
-  ["jsdoc","v0.23.2"],
-  ["json","v0.24.8"],
-  ["php","v0.24.2"],
-  ["python","v0.23.6"],
-  ["ruby","v0.23.1"],
-  ["rust","v0.24.0"],
-  ["typescript","v0.23.2"]
-]
+  ["bash","v0.25.0", null],
+  ["c","v0.24.1", null],
+  ["cpp","v0.23.4", null],
+  ["embedded-template","v0.25.0", null],
+  ["go","v0.25.0", null],
+  ["html","v0.23.2", null],
+  ["java","v0.23.5", null],
+  ["javascript","v0.25.0", null],
+  ["jsdoc","v0.23.2", null],
+  ["json","v0.24.8", null],
+  ["php","v0.24.2", "upstream_test_fixture"],
+  ["python","v0.23.6", null],
+  ["ruby","v0.23.1", null],
+  ["rust","v0.24.0", null],
+  ["typescript","v0.23.2", null]
+]

test/fixtures/test_grammars/external_lookahead_eof_boundary/corpus.txt

@@ -0,0 +1,13 @@
+=====================
+opened and closed span
+=====================
+``
+---
+(document (span (open_delim) (close_delim)))
+
+==================
+unclosed delimiter
+==================
+`
+---
+(document (unclosed_delim))

test/fixtures/test_grammars/external_lookahead_eof_boundary/grammar.js

@@ -0,0 +1,15 @@
+// External scanner whose token choice depends on `lexer->eof()`: at a
+// '`' it peeks past mark_end for a matching close, emitting open_delim
+// if one is found or unclosed_delim if eof is reached first. Inside an
+// open span the same '`' is emitted as close_delim.
+
+export default grammar({
+    name: 'external_lookahead_eof_boundary',
+
+    externals: $ => [$.open_delim, $.close_delim, $.unclosed_delim],
+
+    rules: {
+        document: $ => repeat(choice($.span, $.unclosed_delim)),
+        span: $ => seq($.open_delim, $.close_delim),
+    }
+});

test/fixtures/test_grammars/external_lookahead_eof_boundary/scanner.c

@@ -0,0 +1,54 @@
+#include "tree_sitter/parser.h"
+
+enum {
+    OPEN_DELIM,
+    CLOSE_DELIM,
+    UNCLOSED_DELIM,
+};
+
+void *tree_sitter_external_lookahead_eof_boundary_external_scanner_create(void) { return NULL; }
+
+void tree_sitter_external_lookahead_eof_boundary_external_scanner_destroy(void *payload) {}
+
+unsigned tree_sitter_external_lookahead_eof_boundary_external_scanner_serialize(
+    void *payload, char *buffer
+) {
+    return 0;
+}
+
+void tree_sitter_external_lookahead_eof_boundary_external_scanner_deserialize(
+    void *payload, const char *buffer, unsigned length
+) {}
+
+bool tree_sitter_external_lookahead_eof_boundary_external_scanner_scan(
+    void *payload, TSLexer *lexer, const bool *valid_symbols
+) {
+    if (lexer->lookahead != '`') return false;
+
+    lexer->advance(lexer, false);
+    lexer->mark_end(lexer);
+
+    // Mid-span: emit the closer.
+    if (valid_symbols[CLOSE_DELIM]) {
+        lexer->result_symbol = CLOSE_DELIM;
+        return true;
+    }
+
+    // At an opener: peek past mark_end for a matching '`'. The eof()
+    // probe makes the result depend on the current included ranges.
+    if (valid_symbols[OPEN_DELIM]) {
+        while (!lexer->eof(lexer)) {
+            if (lexer->lookahead == '`') {
+                lexer->result_symbol = OPEN_DELIM;
+                return true;
+            }
+            lexer->advance(lexer, false);
+        }
+        if (valid_symbols[UNCLOSED_DELIM]) {
+            lexer->result_symbol = UNCLOSED_DELIM;
+            return true;
+        }
+    }
+
+    return false;
+}

test/fixtures/test_grammars/terminal_supertype/expected_error.txt

@@ -0,0 +1 @@
+Terminal rule 'A' cannot be used as a supertype

test/fixtures/test_grammars/terminal_supertype/grammar.js

@@ -0,0 +1,18 @@
+module.exports = grammar({
+  name: 'terminal_supertype',
+
+  supertypes: $ => [$.A],
+  extras: $ => [$.lineComment, $.lineComment2],
+
+  rules: {
+    source_file: $ => choice(
+      $.B,
+      $.A,
+    ),
+    lineComment: $ => /;;.*/,
+    lineComment2: $ => /##.*/,
+    A: $ => /xyz/,
+    C: $ => "xyz",
+    B: $ => seq('abc', $.C),
+  }
+});

test/fixtures/test_grammars/utf16_surrogate_oob/corpus.txt

@@ -0,0 +1,7 @@
+==============================
+utf16 supplementary character
+==============================
+🦋
+------------------------------
+
+(program (supplementary))

test/fixtures/test_grammars/utf16_surrogate_oob/grammar.js

@@ -0,0 +1,18 @@
+/// A minimal grammar for testing UTF-16 decode behavior.
+/// The external scanner distinguishes surrogate code points (0xD800-0xDFFF)
+/// from supplementary-plane characters (> 0xFFFF).
+module.exports = grammar({
+  name: 'utf16_surrogate_oob',
+
+  externals: $ => [
+    $.surrogate,
+    $.supplementary,
+  ],
+
+  rules: {
+    program: $ => repeat(choice(
+      $.surrogate,
+      $.supplementary,
+    )),
+  },
+});

test/fixtures/test_grammars/utf16_surrogate_oob/scanner.c

@@ -0,0 +1,47 @@
+#include "tree_sitter/parser.h"
+
+enum TokenType {
+  SURROGATE,
+  SUPPLEMENTARY,
+};
+
+void *tree_sitter_utf16_surrogate_oob_external_scanner_create(void) { return NULL; }
+void tree_sitter_utf16_surrogate_oob_external_scanner_destroy(void *p) {}
+unsigned tree_sitter_utf16_surrogate_oob_external_scanner_serialize(void *p, char *b) { return 0; }
+void tree_sitter_utf16_surrogate_oob_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
+
+bool tree_sitter_utf16_surrogate_oob_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  int32_t c = lexer->lookahead;
+
+  // Raw surrogate code point (0xD800-0xDFFF): the decoder returned an unpaired
+  // surrogate, which is what happens when a lead surrogate is at the end of a chunk.
+  //
+  // The correct token for `test_utf16_le_decode_does_not_read_oob`
+  // The incorrect token for this grammar's corpus test.
+  if (c >= 0xD800 && c <= 0xDFFF && valid_symbols[SURROGATE]) {
+    lexer->advance(lexer, false);
+    lexer->mark_end(lexer);
+    lexer->result_symbol = SURROGATE;
+    return true;
+  }
+
+  // Supplementary-plane character (>0xFFFF): the decoder formed a character from
+  // a surrogate pair. For `test_utf16_le_decode_does_not_read_oob`, this should
+  // only be possible via an OOB read when the lead surrogate is at the end of a
+  // chunk.
+  //
+  // The incorrect token for `test_utf16_le_decode_does_not_read_oob`
+  // The correct token for this grammar's corpus test.
+  if (c > 0xFFFF && valid_symbols[SUPPLEMENTARY]) {
+    lexer->advance(lexer, false);
+    lexer->mark_end(lexer);
+    lexer->result_symbol = SUPPLEMENTARY;
+    return true;
+  }
+
+  return false;
+}

test/fixtures/test_grammars/wasm_realloc_clobber_region/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A leading zero-width token
+==========================
+
+hello
+
+---
+
+(document (zero_width))

test/fixtures/test_grammars/wasm_realloc_clobber_region/grammar.js

@@ -0,0 +1,10 @@
+export default grammar({
+    name: 'wasm_realloc_clobber_region',
+
+    extras: _ => [/\s/],
+    externals: $ => [$.zero_width],
+
+    rules: {
+        document: $ => seq($.zero_width, 'hello'),
+    }
+});

test/fixtures/test_grammars/wasm_realloc_clobber_region/scanner.c

@@ -0,0 +1,68 @@
+#include "tree_sitter/parser.h"
+#include <string.h>
+
+#ifdef __wasm__
+  #define panic(x) __builtin_trap()
+#else
+  #include <assert.h>
+  #define panic(x) assert(0)
+#endif
+
+enum TokenType {
+  ZERO_WIDTH_TOKEN
+};
+
+void *tree_sitter_wasm_realloc_clobber_region_external_scanner_create(void) {
+  size_t a_count = 32;
+  void *a = malloc(a_count);
+  memset(a, 0x41, a_count); // fill `a` with 'A's
+  for (int i = 0; i < 32; i++) {
+    // This condition is unreachable, but it prevents wasi-sdk clang from optimizing
+    // out the allocation to `a`.
+    if (((char *)a)[i] != 0x41) panic();
+  }
+
+  size_t b_count = 16;
+  void *b = malloc(b_count);
+  memset(b, 0x42, b_count); // fill `b` with 'B's
+
+  // `a` is now first on the free list
+  free(a);
+  // We're re-allocating the last region, so `realloc`'s optimization to re-use the region
+  // by altering its metadata's size (thus avoiding a memcpy) should be used.
+  void *c = realloc(b, 24); // realloc `b` should still contain 'B's
+
+  // ensure the contents of `b` were not clobbered. A previous bug caused `a`s region
+  // to be used for `c`.
+  char *p = (char *)c;
+  for (int i = 0; i < (int)b_count; i++) {
+    if (p[i] != 0x42) panic();
+  }
+
+  free(c);
+  return NULL;
+}
+
+bool tree_sitter_wasm_realloc_clobber_region_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = ZERO_WIDTH_TOKEN;
+  return true;
+}
+
+unsigned tree_sitter_wasm_realloc_clobber_region_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_wasm_realloc_clobber_region_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_wasm_realloc_clobber_region_external_scanner_destroy(void *payload) {}

test/fixtures/test_grammars/wasm_realloc_overflow_heap/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A leading zero-width token
+==========================
+
+hello
+
+---
+
+(document (zero_width))

test/fixtures/test_grammars/wasm_realloc_overflow_heap/grammar.js

@@ -0,0 +1,10 @@
+export default grammar({
+    name: 'wasm_realloc_overflow_heap',
+
+    extras: _ => [/\s/],
+    externals: $ => [$.zero_width],
+
+    rules: {
+        document: $ => seq($.zero_width, 'hello'),
+    }
+});

test/fixtures/test_grammars/wasm_realloc_overflow_heap/scanner.c

@@ -0,0 +1,43 @@
+#include "tree_sitter/parser.h"
+
+// Constant copied from `crates/language/wasm/src/stdlib.c`,
+// Must be kept in sync for a reliable repro. 
+#define MAX_HEAP_SIZE (4 * 1024 * 1024)
+
+enum TokenType {
+  ZERO_WIDTH_TOKEN
+};
+
+void *tree_sitter_wasm_realloc_overflow_heap_external_scanner_create(void) {
+  size_t large_size = MAX_HEAP_SIZE - 64;
+  void *p = malloc(large_size);
+  void *q = malloc(4);
+  p = realloc(p, 4);
+  free(p);
+  free(q);
+  return NULL;
+}
+
+bool tree_sitter_wasm_realloc_overflow_heap_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = ZERO_WIDTH_TOKEN;
+  return true;
+}
+
+unsigned tree_sitter_wasm_realloc_overflow_heap_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_wasm_realloc_overflow_heap_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_wasm_realloc_overflow_heap_external_scanner_destroy(void *payload) {}