release v0.26.13

`ExactSizeIterator` implements `std::iter::Iterator`, which is already
`must_use`.

(cherry picked from commit 54a46eb49b)

.github/actions/cache/action.yml

@@ -10,7 +10,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       id: cache
       with:
         path: |
@@ -22,4 +22,5 @@ runs:
           'lib/src/array.h',
           'lib/src/alloc.h',
           'test/fixtures/grammars/*/**/src/*.c',
+          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}

.github/workflows/build.yml

@@ -32,6 +32,7 @@ jobs:
           # When adding a new `target`:
           # 1. Define a new platform alias above
           # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
+          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
           - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
           - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
           - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
@@ -46,8 +47,8 @@ jobs:
 
           # Extra features
           - { platform: linux-arm64     , features: wasm }
-          - { platform: linux-x64       , features: wasm }
-          - { platform: macos-arm64     , features: wasm }
+          - { platform: linux-x64       , features: wasm , run-wasm-test: true }
+          - { platform: macos-arm64     , features: wasm , run-wasm-test: true }
           - { platform: macos-x64       , features: wasm }
 
           # Cross-compilation
@@ -85,14 +86,37 @@ jobs:
         } >> $GITHUB_ENV
 
     - name: Get emscripten version
-      if: contains(matrix.features, 'wasm')
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
 
-    - name: Install Emscripten
-      if: contains(matrix.features, 'wasm')
-      uses: mymindstorm/setup-emsdk@v14
+    - name: Cache Emscripten SDK
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/cache@v4
       with:
-        version: ${{ env.EMSCRIPTEN_VERSION }}
+        path: emsdk
+        key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
+
+    - name: Install Emscripten
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      run: |
+        if [[ ! -d emsdk ]]; then
+          git clone --depth 1 https://github.com/emscripten-core/emsdk.git
+        fi
+        cd emsdk
+        ./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
+        ./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
+        echo "$PWD" >> "$GITHUB_PATH"
+        echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
+        echo "EMSDK=$PWD" >> "$GITHUB_ENV"
+        echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
+
+    - name: Set up Node.js
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/setup-node@v4
+      with:
+        node-version: 24
+        cache: npm
+        cache-dependency-path: lib/binding_web/package-lock.json
 
     - name: Set up Rust
       uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -117,34 +141,12 @@ jobs:
       if: matrix.platform == 'windows-x64'
       uses: msys2/setup-msys2@v2
       with:
-        update: true
         install: |
           mingw-w64-x86_64-toolchain
           mingw-w64-x86_64-clang
           mingw-w64-x86_64-make
           mingw-w64-x86_64-cmake
 
-    # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
-    # the `mismatched-lifetime-syntaxes` lint
-    - name: Build wasmtime library (Windows x64 MSYS2)
-      if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64'
-      run: |
-        mkdir -p target
-        WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
-                           jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
-        cd target/wasmtime-${WASMTIME_VERSION}
-        cmake -S crates/c-api -B target/c-api \
-          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
-          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
-          -DWASMTIME_FEATURE_CRANELIFT=ON \
-          -DWASMTIME_TARGET='x86_64-pc-windows-gnu'
-        cmake --build target/c-api && cmake --install target/c-api
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
-      env:
-        WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
-
     - name: Build C library (Windows x64 MSYS2 CMake)
       if: matrix.platform == 'windows-x64'
       shell: msys2 {0}
@@ -171,26 +173,23 @@ jobs:
       env:
         WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
 
-    # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
-    # the `mismatched-lifetime-syntaxes` lint
-    - name: Build wasmtime library
-      if: contains(matrix.features, 'wasm')
+    - name: Download wasmtime C API
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       run: |
         mkdir -p target
         WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
                            jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
-        cd target/wasmtime-${WASMTIME_VERSION}
-        cmake -S crates/c-api -B target/c-api \
-          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
-          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
-          -DWASMTIME_FEATURE_CRANELIFT=ON \
-          -DWASMTIME_TARGET='${{ matrix.target }}'
-        cmake --build target/c-api && cmake --install target/c-api
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
+        case '${{ matrix.target }}' in
+          x86_64-unknown-linux-gnu)  WT_TARGET=x86_64-linux ;;
+          aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
+          x86_64-apple-darwin)       WT_TARGET=x86_64-macos ;;
+          aarch64-apple-darwin)      WT_TARGET=aarch64-macos ;;
+        esac
+        curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
+          | tar xJf - -C target
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
       env:
         WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
 
     - name: Build C library (make)
       if: runner.os != 'Windows'
@@ -207,7 +206,7 @@ jobs:
         make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
       env:
         PLATFORM: ${{ matrix.platform }}
-        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
     - name: Build C library (CMake)
       if: "!matrix.cross"
@@ -227,10 +226,10 @@ jobs:
         cmake --build build/shared --verbose
       env:
         CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
-        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
+        WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
 
     - name: Build Wasm library
-      if: contains(matrix.features, 'wasm')
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       shell: bash
       run: |
         cd lib/binding_web
@@ -247,7 +246,8 @@ jobs:
       run: cargo check --no-default-features --target='${{ matrix.target }}'
 
     - name: Build target
-      run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE
+      if: "!inputs.run-test"
+      run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
       env:
         PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
 
@@ -265,20 +265,20 @@ jobs:
       run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
 
     - name: Generate Wasm fixtures
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true'
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
       run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
 
     - name: Run main tests
       if: inputs.run-test && !matrix.no-run
-      run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}'
+      run: cargo test --workspace --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
 
     - name: Run Wasm tests
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm')
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
       run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
 
     - name: Upload CLI artifact
-      if: "!matrix.no-run"
-      uses: actions/upload-artifact@v5
+      if: "!inputs.run-test && !matrix.no-run"
+      uses: actions/upload-artifact@v6
       with:
         name: tree-sitter.${{ matrix.platform }}
         path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
@@ -286,8 +286,8 @@ jobs:
         retention-days: 7
 
     - name: Upload Wasm artifacts
-      if: matrix.platform == 'linux-x64'
-      uses: actions/upload-artifact@v5
+      if: "!inputs.run-test && matrix.platform == 'linux-x64'"
+      uses: actions/upload-artifact@v6
       with:
         name: tree-sitter.wasm
         path: |

.github/workflows/crate_versions.yml

@@ -0,0 +1,22 @@
+name: Crate Versions Check
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+  workflow_dispatch:
+
+jobs:
+  check-crates:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Check crates against crates.io
+        uses: katyo/publish-crates@v2
+        with:
+          dry-run: true

.github/workflows/release.yml

@@ -44,7 +44,9 @@ jobs:
 
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
+            chmod +x $exe
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
+            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
@@ -54,12 +56,14 @@ jobs:
         with:
           subject-path: |
             target/tree-sitter-*.gz
+            target/tree-sitter-cli-*.zip
             target/web-tree-sitter.tar.gz
 
       - name: Create release
         run: |-
           gh release create $GITHUB_REF_NAME \
             target/tree-sitter-*.gz \
+            target/tree-sitter-cli-*.zip \
             target/web-tree-sitter.tar.gz
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/wasm_exports.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Build C library (make)
         run: make -j CFLAGS="$CFLAGS"
         env:
-          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
       - name: Build Wasm Library
         working-directory: lib/binding_web

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.26.3"
+        VERSION "0.26.13"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
@@ -33,7 +33,8 @@ if(MSVC)
 else()
   target_compile_options(tree-sitter PRIVATE
                          -Wall -Wextra -Wshadow -Wpedantic
-                         -Werror=incompatible-pointer-types)
+                         -Werror=incompatible-pointer-types
+                         -Werror=strict-aliasing -Wstrict-aliasing=2)
 endif()
 
 if(TREE_SITTER_FEATURE_WASM)
@@ -81,7 +82,7 @@ set_target_properties(tree-sitter
                       SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
                       DEFINE_SYMBOL "")
 
-target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
 
 include(GNUInstallDirs)
 

Cargo.toml

@@ -14,7 +14,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.26.3"
+version = "0.26.13"
 authors = [
   "Max Brunsfeld <maxbrunsfeld@gmail.com>",
   "Amaan Qureshi <amaanq12@gmail.com>",
@@ -153,11 +153,11 @@ walkdir = "2.5.0"
 wasmparser = "0.243.0"
 webbrowser = "1.0.5"
 
-tree-sitter = { version = "0.26.3", path = "./lib" }
-tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" }
-tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" }
-tree-sitter-config = { version = "0.26.3", path = "./crates/config" }
-tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" }
-tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" }
+tree-sitter = { version = "0.26.13", path = "./lib" }
+tree-sitter-generate = { version = "0.26.13", path = "./crates/generate", default-features = false }
+tree-sitter-loader = { version = "0.26.13", path = "./crates/loader" }
+tree-sitter-config = { version = "0.26.13", path = "./crates/config" }
+tree-sitter-highlight = { version = "0.26.13", path = "./crates/highlight" }
+tree-sitter-tags = { version = "0.26.13", path = "./crates/tags" }
 
 tree-sitter-language = { version = "0.1", path = "./crates/language" }

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.26.3
+VERSION := 0.26.13
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -22,9 +22,9 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -122,7 +122,6 @@ test-wasm:
 
 lint:
 	cargo update --workspace --locked --quiet
-	cargo check --workspace --all-targets
 	cargo fmt --all --check
 	cargo clippy --workspace --all-targets -- -D warnings
 

Package.swift

@@ -27,6 +27,7 @@ let package = Package(
                         .headerSearchPath("src"),
                         .define("_POSIX_C_SOURCE", to: "200112L"),
                         .define("_DEFAULT_SOURCE"),
+                        .define("_BSD_SOURCE"),
                         .define("_DARWIN_C_SOURCE"),
                 ]),
     ],

build.zig

@@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
 
     lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
     lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
+    lib.root_module.addCMacro("_BSD_SOURCE", "");
     lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
 
     if (wasm) {

build.zig.zon

@@ -1,7 +1,7 @@
 .{
     .name = .tree_sitter,
     .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.26.3",
+    .version = "0.26.13",
     .minimum_zig_version = "0.14.1",
     .paths = .{
         "build.zig",
@@ -13,83 +13,83 @@
     },
     .dependencies = .{
         .wasmtime_c_api_aarch64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIfPIgdw2YnV3QyiFQ2NHdrxrXzzCdjYJyxJDOta",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-aarch64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAK1DGQSFd3QeAM37CBAgGjcO6SP4PTDitKUyv2FL",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAIt97QZi7Pf7nNJ2mVY6uxA80Klyuvvtop3pLMRK",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-aarch64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AADHpGQQi5Sb9gmPvUbAP3eDfMT7SloTyWHq0bULy",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAAO48QQf91w9RmmUDHTja8DrXZA1n6Bmc8waW3qe",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-aarch64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAA--3gIS9QGKTh_CdihUxebSyDLRgMyaSLErr0uU",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAI196wa9pwADoA2RbCDp5F7bKQg1iOPq6gIh8-FH",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-aarch64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AALs2GATaXyZRY1eSj0MqcnCPA-rvwQKDLzNKoO7k",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-windows-c-api.zip",
-            .hash = "N-V-__8AAC9u4wXfqd1Q6XyQaC8_DbQZClXux60Vu5743N05",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-aarch64-windows-c-api.zip",
+            .hash = "N-V-__8AACG7oQSQzo4Yqb2ROtmf0Te6z25qE2D1zZVOhOjT",
             .lazy = true,
         },
         .wasmtime_c_api_armv7_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-armv7-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAHXe8gWs3s83Cc5G6SIq0_jWxj8fGTT5xG4vb6-x",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-armv7-linux-c-api.tar.xz",
+            .hash = "N-V-__8AACfgaQOJMJ60s9tEEnIGqCc5oCc8DjV6WCCU8jfv",
             .lazy = true,
         },
         .wasmtime_c_api_i686_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAN2pzgUUfulRCYnipSfis9IIYHoTHVlieLRmKuct",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-i686-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAA0YpwPx6WET15rgQX6MuRYgJrBFFhGJ6T1o7bXd",
             .lazy = true,
         },
         .wasmtime_c_api_i686_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-windows-c-api.zip",
-            .hash = "N-V-__8AAJu0YAUUTFBLxFIOi-MSQVezA6MMkpoFtuaf2Quf",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-i686-windows-c-api.zip",
+            .hash = "N-V-__8AAEuShgR9nTTtALm1vg7ydAa-XHeFJoCeS6EwWMYO",
             .lazy = true,
         },
         .wasmtime_c_api_riscv64gc_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-riscv64gc-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAG8m-gc3E3AIImtTZ3l1c7HC6HUWazQ9OH5KACX4",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-riscv64gc-linux-c-api.tar.xz",
+            .hash = "N-V-__8AALE0UwVQ2O0JUsq-bvNiae9w9rF-DcoZReRwc3VO",
             .lazy = true,
         },
         .wasmtime_c_api_s390x_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-s390x-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAH314gd-gE4IBp2uvAL3gHeuW1uUZjMiLLeUdXL_",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-s390x-linux-c-api.tar.xz",
+            .hash = "N-V-__8AADt6VAT4U8GJtiOuTxT_d--hYS7I8sOYeMDhsmYm",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIPNRwfNkznebrcGb0IKUe7f35bkuZEYOjcx6q3f",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-x86_64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAOOKmQR9Gt_MNxWAdciCw9I0GLtvDhlcgC3dd8DU",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAI8EDwcyTtk_Afhk47SEaqfpoRqGkJeZpGs69ChF",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-x86_64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAB_LmASYgb6KhwBhZV3_1VCPN_qIBCiLONea2VGO",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAGtGNgVaOpHSxC22IjrampbRIy6lLwscdcAE8nG1",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-x86_64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAG9NVQNTKddBttoabwkDR_z9SjWpxC60kIzHiLZ5",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_mingw = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-mingw-c-api.zip",
-            .hash = "N-V-__8AAPS2PAbVix50L6lnddlgazCPTz3whLUFk1qnRtnZ",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-x86_64-mingw-c-api.zip",
+            .hash = "N-V-__8AABqH-QT-RZQZvYQr6KKwTgDYN36-F3zEld6zltCe",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAF-WEQe0nzvi09PgusM5i46FIuCKJmIDWUleWgQ3",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-x86_64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAHE1nATox7TV8A9MUpHA3T7VMRW__JIGXvwtBBnd",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-windows-c-api.zip",
-            .hash = "N-V-__8AAKGNXwbpJQsn0_6kwSIVDDWifSg8cBzf7T2RzsC9",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.14/wasmtime-v36.0.14-x86_64-windows-c-api.zip",
+            .hash = "N-V-__8AAN9FXgVEOFW9RhCY3rKCYgg-HT7O8eBuFNOPI2Av",
             .lazy = true,
         },
     },

crates/cli/Cargo.toml

@@ -67,7 +67,7 @@ wasmparser.workspace = true
 webbrowser.workspace = true
 
 tree-sitter.workspace = true
-tree-sitter-generate.workspace = true
+tree-sitter-generate = { workspace = true, features = ["load"] }
 tree-sitter-config.workspace = true
 tree-sitter-highlight.workspace = true
 tree-sitter-loader.workspace = true
@@ -81,3 +81,36 @@ tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter
 tempfile.workspace = true
 pretty_assertions.workspace = true
 unindent.workspace = true
+
+[package.metadata.binstall]
+pkg-fmt = "zip"
+
+[package.metadata.binstall.overrides.aarch64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.armv7-unknown-linux-gnueabihf]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.powerpc64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-powerpc64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-x64{ archive-suffix }"

crates/cli/README.md

@@ -11,16 +11,15 @@ The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars fr
 
 ### Installation
 
-You can install the `tree-sitter-cli` with `cargo`:
+You can install the `tree-sitter-cli` with `cargo-binstall`:
 
 ```sh
-cargo install --locked tree-sitter-cli
+cargo binstall tree-sitter-cli
 ```
 
-or with `npm`:
-
+or you can build it from source:
 ```sh
-npm install tree-sitter-cli
+cargo install --locked tree-sitter-cli
 ```
 
 You can also download a pre-built binary for your platform from [the releases page].

crates/cli/benches/benchmark.rs

@@ -19,9 +19,7 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
 static EXAMPLE_FILTER: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
 static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
-    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
-        .map(|s| s.parse::<usize>().unwrap())
-        .unwrap_or(5)
+    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
 });
 static TEST_LOADER: LazyLock<Loader> =
     LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));

crates/cli/npm/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.26.3",
+  "version": "0.26.13",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "tree-sitter-cli",
-      "version": "0.26.3",
+      "version": "0.26.13",
       "hasInstallScript": true,
       "license": "MIT",
       "bin": {

crates/cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.26.3",
+  "version": "0.26.13",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

crates/cli/src/fuzz.rs

@@ -44,11 +44,13 @@ pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
 
 pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
 
+pub const DEFAULT_EDIT_COUNT: usize = 3;
 pub static EDIT_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
+    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(DEFAULT_EDIT_COUNT));
 
+pub const DEFAULT_ITERATION_COUNT: usize = 10;
 pub static ITERATION_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
+    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(DEFAULT_ITERATION_COUNT));
 
 fn int_env_var(name: &'static str) -> Option<usize> {
     env::var(name).ok().and_then(|e| e.parse().ok())
@@ -221,7 +223,7 @@ pub fn fuzz_language_corpus(
                 }
 
                 // Perform a random series of edits and reparse.
-                let edit_count = rand.unsigned(*EDIT_COUNT);
+                let edit_count = rand.unsigned(options.edits);
                 let mut undo_stack = Vec::with_capacity(edit_count);
                 for _ in 0..=edit_count {
                     let edit = get_random_edit(&mut rand, &input);
@@ -253,7 +255,7 @@ pub fn fuzz_language_corpus(
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
+                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 

crates/cli/src/fuzz/scope_sequence.rs

@@ -49,7 +49,7 @@ impl ScopeSequence {
         for i in 0..(self.0.len().max(other.0.len())) {
             let stack = &self.0.get(i);
             let other_stack = &other.0.get(i);
-            if *stack != *other_stack && ![b'\r', b'\n'].contains(&text[i]) {
+            if *stack != *other_stack && !b"\r\n".contains(&text[i]) {
                 let containing_range = known_changed_ranges
                     .iter()
                     .find(|range| range.start_point <= position && position < range.end_point);

crates/cli/src/highlight.rs

@@ -190,20 +190,14 @@ fn parse_style(style: &mut Style, json: Value) {
     if let Value::Object(entries) = json {
         for (property_name, value) in entries {
             match property_name.as_str() {
-                "bold" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.bold();
-                    }
+                "bold" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.bold();
                 }
-                "italic" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.italic();
-                    }
+                "italic" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.italic();
                 }
-                "underline" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.underline();
-                    }
+                "underline" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.underline();
                 }
                 "color" => {
                     if let Some(color) = parse_color(value) {

crates/cli/src/init.rs

@@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
 use crc32fast::hash as crc32;
 use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
 use indoc::{formatdoc, indoc};
-use log::warn;
+use log::info;
 use rand::{thread_rng, Rng};
 use semver::Version;
 use serde::{Deserialize, Serialize};
@@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
 const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
 const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
 
-const TREE_SITTER_JSON_SCHEMA: &str =
+pub const TREE_SITTER_JSON_SCHEMA: &str =
     "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -356,7 +356,7 @@ pub fn generate_grammar_files(
                     "tree-sitter-cli":"#},
                 );
             if !contents.contains("module") {
-                warn!("Updating package.json");
+                info!("Migrating package.json to ESM");
                 contents = contents.replace(
                     r#""repository":"#,
                     indoc! {r#"
@@ -378,6 +378,7 @@ pub fn generate_grammar_files(
             |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if contents.contains("module.exports") {
+                    info!("Migrating grammars.js to ESM");
                     contents = contents.replace("module.exports =", "export default");
                     write_file(path, contents)?;
                 }
@@ -393,10 +394,16 @@ pub fn generate_grammar_files(
         allow_update,
         |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
         |path| {
-            let contents = fs::read_to_string(path)?;
+            let mut contents = fs::read_to_string(path)?;
             if !contents.contains("Zig artifacts") {
-                warn!("Replacing .gitignore");
-                generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
+                info!("Adding zig entries to .gitignore");
+                contents.push('\n');
+                contents.push_str(indoc! {"
+                # Zig artifacts
+                .zig-cache/
+                zig-cache/
+                zig-out/
+                "});
             }
             Ok(())
         },
@@ -409,8 +416,13 @@ pub fn generate_grammar_files(
         |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
         |path| {
             let mut contents = fs::read_to_string(path)?;
-            contents = contents.replace("bindings/c/* ", "bindings/c/** ");
+            let c_bindings_entry = "bindings/c/* ";
+            if contents.contains(c_bindings_entry) {
+                info!("Updating c bindings entry in .gitattributes");
+                contents = contents.replace(c_bindings_entry, "bindings/c/** ");
+            }
             if !contents.contains("Zig bindings") {
+                info!("Adding zig entries to .gitattributes");
                 contents.push('\n');
                 contents.push_str(indoc! {"
                 # Zig bindings
@@ -438,39 +450,40 @@ pub fn generate_grammar_files(
             }, |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if !contents.contains("#[cfg(with_highlights_query)]") {
-                let replacement = indoc! {r#"
-                    #[cfg(with_highlights_query)]
-                    /// The syntax highlighting query for this grammar.
-                    pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
+                    info!("Updating query constants in bindings/rust/lib.rs");
+                    let replacement = indoc! {r#"
+                        #[cfg(with_highlights_query)]
+                        /// The syntax highlighting query for this grammar.
+                        pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
 
-                    #[cfg(with_injections_query)]
-                    /// The language injection query for this grammar.
-                    pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
+                        #[cfg(with_injections_query)]
+                        /// The language injection query for this grammar.
+                        pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
 
-                    #[cfg(with_locals_query)]
-                    /// The local variable query for this grammar.
-                    pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
+                        #[cfg(with_locals_query)]
+                        /// The local variable query for this grammar.
+                        pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
 
-                    #[cfg(with_tags_query)]
-                    /// The symbol tagging query for this grammar.
-                    pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
-                    "#}
-                    .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                    .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                    .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                    .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
-                contents = contents
-                    .replace(
-                        indoc! {r#"
-                        // NOTE: uncomment these to include any queries that this grammar contains:
+                        #[cfg(with_tags_query)]
+                        /// The symbol tagging query for this grammar.
+                        pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
+                        "#}
+                        .replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
+                        .replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
+                        .replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
+                        .replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
+                    contents = contents
+                        .replace(
+                            indoc! {r#"
+                            // NOTE: uncomment these to include any queries that this grammar contains:
 
-                        // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-                        // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-                        // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-                        // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-                        "#},
-                        &replacement,
-                    );
+                            // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
+                            // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
+                            // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
+                            // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
+                            "#},
+                            &replacement,
+                        );
                 }
                 write_file(path, contents)?;
                 Ok(())
@@ -483,6 +496,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("wasm32-unknown-unknown") {
+                        info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
                         let replacement = indoc!{r#"
                             c_config.flag("-utf-8");
 
@@ -503,19 +517,18 @@ pub fn generate_grammar_files(
                                     wasm_src.join("string.c"),
                                 ]);
                             }
-                        "#};
-
-                        let indented_replacement = replacement
+                        "#}
                             .lines()
                             .map(|line| if line.is_empty() { line.to_string() } else { format!("    {line}") })
                             .collect::<Vec<_>>()
                             .join("\n");
 
-                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &indented_replacement);
+                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &replacement);
                     }
 
                     // Introduce configuration variables for dynamic query inclusion
                     if !contents.contains("with_highlights_query") {
+                        info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
                         let replaced = indoc! {r#"
                                 c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
                             }"#}
@@ -542,10 +555,10 @@ pub fn generate_grammar_files(
                                 }
                             }"#}
                             .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
-                            .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                            .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                            .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                            .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
+                            .replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
+                            .replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
+                            .replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
+                            .replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
 
                         contents = contents.replace(
                             &replaced,
@@ -572,6 +585,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("\"LICENSE\"") {
+                        info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
                         write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
                     }
                     Ok(())
@@ -592,7 +606,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("Object.defineProperty") {
-                        warn!("Replacing index.js");
+                        info!("Replacing index.js");
                         generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -606,7 +620,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("export default binding") {
-                        warn!("Replacing index.d.ts");
+                        info!("Replacing index.d.ts");
                         generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -627,7 +641,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("import") {
-                        warn!("Replacing binding_test.js");
+                        info!("Replacing binding_test.js");
                         generate_file(
                             path,
                             BINDING_TEST_JS_TEMPLATE,
@@ -650,6 +664,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("fs.exists(") {
+                        info!("Replacing `fs.exists` calls in binding.gyp");
                         write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
                     }
                     Ok(())
@@ -662,14 +677,17 @@ pub fn generate_grammar_files(
 
     // Generate C bindings
     if tree_sitter_config.bindings.c {
+        let kebab_case_name = language_name.to_kebab_case();
         missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
-            let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
+            let header_name = format!("tree-sitter-{kebab_case_name}.h");
+            let old_file = &path.join(&header_name);
             if allow_update && fs::exists(old_file).unwrap_or(false) {
+                info!("Removing bindings/c/{header_name}");
                 fs::remove_file(old_file)?;
             }
             missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
                 missing_path(
-                    include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
+                    include_path.join(&header_name),
                     |path| {
                         generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
                     },
@@ -678,7 +696,7 @@ pub fn generate_grammar_files(
             })?;
 
             missing_path(
-                path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
+                path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
                 |path| {
                     generate_file(
                         path,
@@ -698,23 +716,40 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
-                        warn!("Replacing Makefile");
+                        info!("Replacing Makefile");
                         generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
                     } else {
-                        contents = contents
-                            .replace(
-                                indoc! {r"
-                                $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
-                                "},
-                                indoc! {r"
-                                $(SRC_DIR)/grammar.json: grammar.js
-                                        $(TS) generate --no-parser $^
+                        let replaced = indoc! {r"
+                            $(PARSER): $(SRC_DIR)/grammar.json
+                                    $(TS) generate $^
+                            "};
+                        if contents.contains(replaced) {
+                            info!("Adding --no-parser target to Makefile");
+                            contents = contents
+                                .replace(
+                                    replaced,
+                                    indoc! {r"
+                                    $(SRC_DIR)/grammar.json: grammar.js
+                                            $(TS) generate --no-parser $^
 
-                                $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
-                                "}
-                            );
+                                    $(PARSER): $(SRC_DIR)/grammar.json
+                                            $(TS) generate $^
+                                    "}
+                                );
+                        }
+                        if !contents.contains("\nDESCRIPTION :=") {
+                            if let Some(version_line) = contents.lines().find(|l| l.starts_with("VERSION := ")) {
+                                info!("Adding DESCRIPTION to Makefile");
+                                let description = generate_opts.description.map_or_else(
+                                    || format!("{} grammar for tree-sitter", generate_opts.camel_parser_name),
+                                    str::to_string,
+                                );
+                                contents = contents.replace(
+                                    version_line,
+                                    &format!("{version_line}\nDESCRIPTION := {description}"),
+                                );
+                            }
+                        }
                         write_file(path, contents)?;
                     }
                     Ok(())
@@ -726,8 +761,8 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let contents = fs::read_to_string(path)?;
+                    let replaced_contents = contents
                         .replace("add_custom_target(test", "add_custom_target(ts-test")
                         .replace(
                             &formatdoc! {r#"
@@ -775,7 +810,10 @@ pub fn generate_grammar_files(
                                                COMMENT "Generating parser.c")
                             "#}
                         );
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating CMakeLists.txt");
+                        write_file(path, replaced_contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -811,7 +849,8 @@ pub fn generate_grammar_files(
     // Generate Python bindings
     if tree_sitter_config.bindings.python {
         missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
-            let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
+            let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
+            let lang_path = path.join(&snake_case_grammar_name);
             missing_path(&lang_path, create_dir)?;
 
             missing_path_else(
@@ -821,6 +860,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("PyModuleDef_Init") {
+                        info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
                         contents = contents
                             .replace("PyModule_Create", "PyModuleDef_Init")
                             .replace(
@@ -861,8 +901,8 @@ pub fn generate_grammar_files(
                 },
                 |path| {
                     let contents = fs::read_to_string(path)?;
-                    if !contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.py");
+                    if contents.contains("uncomment these to include any queries") {
+                        info!("Replacing __init__.py");
                         generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -876,9 +916,10 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.pyi");
+                        info!("Replacing __init__.pyi");
                         generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
                     } else if !contents.contains("CapsuleType") {
+                        info!("Updating __init__.pyi");
                         contents = contents
                             .replace(
                                 "from typing import Final",
@@ -910,6 +951,7 @@ pub fn generate_grammar_files(
                     |path| {
                         let mut contents = fs::read_to_string(path)?;
                         if !contents.contains("Parser(Language(") {
+                            info!("Updating Language function in bindings/python/tests/test_binding.py");
                             contents = contents
                                 .replace("tree_sitter.Language(", "Parser(Language(")
                                 .replace(".language())\n", ".language()))\n")
@@ -930,11 +972,29 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let contents = fs::read_to_string(path)?;
+                    let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("build_ext") {
-                        warn!("Replacing setup.py");
+                        info!("Replacing setup.py");
                         generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
+                    } else {
+                        if !contents.contains(" and not get_config_var") {
+                            info!("Updating Python free-threading support in setup.py");
+                            contents = contents.replace(
+                                r#"startswith("cp"):"#,
+                                r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
+                            );
+                            write_file(path, &contents)?;
+                        }
+                        if !contents.contains("include(\"src/*.c\")") {
+                            info!("Updating sdist file list in setup.py");
+                            let contents = contents.replace(
+                               "include(\"src/tree_sitter/*.h\")",
+                               "include(\"src/tree_sitter/*.h\")\n        self.filelist.include(\"src/*.c\")",
+                            );
+                            write_file(path, &contents)?;
+                        }
                     }
+
                     Ok(())
                 },
             )?;
@@ -953,6 +1013,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cp310-*") {
+                        info!("Updating dependencies in pyproject.toml");
                         contents = contents
                             .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
                             .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
@@ -990,15 +1051,23 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let contents = fs::read_to_string(path)?;
+                    let replaced_contents = contents
                         .replace(
                             "https://github.com/ChimeHQ/SwiftTreeSitter",
                             "https://github.com/tree-sitter/swift-tree-sitter",
                         )
-                        .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
-                        .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
-                    write_file(path, contents)?;
+                        .replace("version: \"0.8.0\")", "version: \"0.10.0\")")
+                        .replace("version: \"0.9.0\")", "version: \"0.10.0\")")
+                        .replace("(name: \"SwiftTreeSitter\", url:", "(url:")
+                        .replace(
+                            "    \"SwiftTreeSitter\"",
+                            "    .product(name: \"SwiftTreeSitter\", package: \"swift-tree-sitter\")",
+                        );
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating tree-sitter dependency in Package.swift");
+                        write_file(path, replaced_contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -1016,7 +1085,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains("b.pkg_hash.len") {
-                    warn!("Replacing build.zig");
+                    info!("Replacing build.zig");
                     generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1031,7 +1100,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains(".name = .tree_sitter_") {
-                    warn!("Replacing build.zig.zon");
+                    info!("Replacing build.zig.zon");
                     generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1047,7 +1116,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("ts.Language") {
-                        warn!("Replacing root.zig");
+                        info!("Replacing root.zig");
                         generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
                     } else {
                         Ok(())
@@ -1189,17 +1258,20 @@ fn generate_file(
         .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name)
         .replace(
             HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.highlights_query_path,
+            &generate_opts.highlights_query_path.replace('\\', "/"),
         )
         .replace(
             INJECTIONS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.injections_query_path,
+            &generate_opts.injections_query_path.replace('\\', "/"),
         )
         .replace(
             LOCALS_QUERY_PATH_PLACEHOLDER,
-            generate_opts.locals_query_path,
+            &generate_opts.locals_query_path.replace('\\', "/"),
         )
-        .replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
+        .replace(
+            TAGS_QUERY_PATH_PLACEHOLDER,
+            &generate_opts.tags_query_path.replace('\\', "/"),
+        );
 
     if let Some(name) = generate_opts.author_name {
         replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);
@@ -1350,24 +1422,18 @@ fn generate_file(
         replacement = replacement
             .replace(
                 PARSER_URL_STRIPPED_PLACEHOLDER,
-                &repository.replace("https://", "").to_lowercase(),
+                &repository.replace("https://", ""),
             )
-            .replace(PARSER_URL_PLACEHOLDER, &repository.to_lowercase());
+            .replace(PARSER_URL_PLACEHOLDER, repository);
     } else {
         replacement = replacement
             .replace(
                 PARSER_URL_STRIPPED_PLACEHOLDER,
-                &format!(
-                    "github.com/tree-sitter/tree-sitter-{}",
-                    language_name.to_lowercase()
-                ),
+                &format!("github.com/tree-sitter/tree-sitter-{language_name}"),
             )
             .replace(
                 PARSER_URL_PLACEHOLDER,
-                &format!(
-                    "https://github.com/tree-sitter/tree-sitter-{}",
-                    language_name.to_lowercase()
-                ),
+                &format!("https://github.com/tree-sitter/tree-sitter-{language_name}"),
             );
     }
 

crates/cli/src/main.rs

@@ -16,11 +16,11 @@ use semver::Version as SemverVersion;
 use tree_sitter::{ffi, Parser, Point};
 use tree_sitter_cli::{
     fuzz::{
-        fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED,
-        LOG_GRAPH_ENABLED, START_SEED,
+        fuzz_language_corpus, FuzzOptions, DEFAULT_EDIT_COUNT, DEFAULT_ITERATION_COUNT, EDIT_COUNT,
+        ITERATION_COUNT, LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED,
     },
     highlight::{self, HighlightOptions},
-    init::{generate_grammar_files, JsonConfigOpts},
+    init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
     input::{get_input, get_tmp_source_file, CliInput},
     logger,
     parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
@@ -234,7 +234,7 @@ struct Parse {
     /// Output the parse data in a pretty-printed CST format
     #[arg(long = "cst", short = 'c')]
     pub output_cst: bool,
-    /// Show parsing statistic
+    /// Show parsing statistics
     #[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")]
     pub stat: bool,
     /// Interrupt the parsing process by timeout (µs)
@@ -391,11 +391,15 @@ struct Fuzz {
     /// library's language function
     #[arg(long)]
     pub lang_name: Option<String>,
-    /// Maximum number of edits to perform per fuzz test
-    #[arg(long)]
+    #[arg(
+        long,
+        help=format!("Maximum number of edits to perform per fuzz test (Default: {DEFAULT_EDIT_COUNT})")
+    )]
     pub edits: Option<usize>,
-    /// Number of fuzzing iterations to run per test
-    #[arg(long)]
+    #[arg(
+        long,
+        help=format!("Number of fuzzing iterations to run per test (Default: {DEFAULT_ITERATION_COUNT})")
+    )]
     pub iterations: Option<usize>,
     /// Only fuzz corpus test cases whose name matches the given regex
     #[arg(long, short)]
@@ -867,10 +871,26 @@ impl Init {
 
             (opts.name.clone(), Some(opts))
         } else {
-            let mut json = serde_json::from_str::<TreeSitterJSON>(
-                &fs::read_to_string(current_dir.join("tree-sitter.json"))
-                    .with_context(|| "Failed to read tree-sitter.json")?,
-            )?;
+            let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
+                .with_context(|| "Failed to read tree-sitter.json")?;
+
+            let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
+            if json.schema.is_none() {
+                json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
+            }
+
+            let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
+            // Write the re-serialized config back, as newly added optional boolean fields
+            // will be included with explicit `false`s rather than implict `null`s
+            if self.update && !old_config.trim().eq(new_config.trim()) {
+                info!("Updating tree-sitter.json");
+                fs::write(
+                    current_dir.join("tree-sitter.json"),
+                    serde_json::to_string_pretty(&json)?,
+                )
+                .with_context(|| "Failed to write tree-sitter.json")?;
+            }
+
             (json.grammars.swap_remove(0).name, None)
         };
 
@@ -955,11 +975,21 @@ impl Build {
         } else {
             let output_path = if let Some(ref path) = self.output {
                 let path = Path::new(path);
-                if path.is_absolute() {
+                let full_path = if path.is_absolute() {
                     path.to_path_buf()
                 } else {
                     current_dir.join(path)
-                }
+                };
+                let parent_path = full_path
+                    .parent()
+                    .context("Output path must have a parent")?;
+                let name = full_path
+                    .file_name()
+                    .context("Ouput path must have a filename")?;
+                fs::create_dir_all(parent_path).context("Failed to create output path")?;
+                let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
+                canon_path.push(name);
+                canon_path
             } else {
                 let file_name = grammar_path
                     .file_stem()
@@ -984,7 +1014,7 @@ impl Build {
 
             loader
                 .compile_parser_at_path(&grammar_path, output_path, flags)
-                .unwrap();
+                .context("Failed to compile parser")?;
         }
         Ok(())
     }
@@ -1323,16 +1353,20 @@ impl Test {
                 self.json_summary,
             )?;
             test_summary.test_num = 1;
+        } else {
+            warn!("Test corpus not found at {}", test_corpus_dir.display());
         }
 
         // Check that all of the queries are valid.
         let query_dir = current_dir.join("queries");
-        check_test(
-            test::check_queries_at_path(language, &query_dir),
-            &test_summary,
-            self.json_summary,
-        )?;
-        test_summary.test_num = 1;
+        if query_dir.is_dir() {
+            check_test(
+                test::check_queries_at_path(language, &query_dir),
+                &test_summary,
+                self.json_summary,
+            )?;
+            test_summary.test_num = 1;
+        }
 
         // Run the syntax highlighting tests.
         let test_highlight_dir = test_dir.join("highlight");
@@ -1622,6 +1656,7 @@ impl Highlight {
         let loader_config = config.get()?;
         loader.find_all_languages(&loader_config)?;
         loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
+        let languages = loader.languages_at_path(current_dir)?;
 
         let cancellation_flag = util::cancel_on_signal();
 
@@ -1702,7 +1737,6 @@ impl Highlight {
             } => {
                 let path = get_tmp_source_file(&contents)?;
 
-                let languages = loader.languages_at_path(current_dir)?;
                 let language = languages
                     .iter()
                     .find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
@@ -1733,7 +1767,6 @@ impl Highlight {
                     if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
                         (l, lc)
                     } else {
-                        let languages = loader.languages_at_path(current_dir)?;
                         let language = languages
                             .first()
                             .map(|(l, _)| l.clone())
@@ -1927,7 +1960,7 @@ impl DumpLanguages {
                 concat!(
                     "name: {}\n",
                     "scope: {}\n",
-                    "parser: {:?}\n",
+                    "parser: {}\n",
                     "highlights: {:?}\n",
                     "file_types: {:?}\n",
                     "content_regex: {:?}\n",
@@ -1935,7 +1968,7 @@ impl DumpLanguages {
                 ),
                 configuration.language_name,
                 configuration.scope.as_ref().unwrap_or(&String::new()),
-                language_path,
+                language_path.display(),
                 configuration.highlights_filenames,
                 configuration.file_types,
                 configuration.content_regex,

crates/cli/src/parse.rs

@@ -306,13 +306,13 @@ pub fn parse_file_at_path(
                 }
                 writeln!(&mut io::stderr(), "{message}").unwrap();
             } else {
+                #[rustfmt::skip]
                 let colors = &[
-                    AnsiColor::White,
-                    AnsiColor::Red,
-                    AnsiColor::Blue,
-                    AnsiColor::Green,
-                    AnsiColor::Cyan,
-                    AnsiColor::Yellow,
+                    AnsiColor::White, AnsiColor::Red, AnsiColor::Blue, AnsiColor::Green,
+                    AnsiColor::Cyan, AnsiColor::Yellow, AnsiColor::Magenta,
+                    AnsiColor::BrightWhite, AnsiColor::BrightRed, AnsiColor::BrightBlue,
+                    AnsiColor::BrightGreen, AnsiColor::BrightCyan, AnsiColor::BrightYellow,
+                    AnsiColor::BrightMagenta,
                 ];
                 if message.starts_with("process version:") {
                     let comma_idx = message.find(',').unwrap();
@@ -321,7 +321,7 @@ pub fn parse_file_at_path(
                         .unwrap();
                 }
                 let color = if use_color {
-                    Some(colors[curr_version])
+                    Some(colors[curr_version % colors.len()])
                 } else {
                     None
                 };
@@ -433,7 +433,7 @@ pub fn parse_file_at_path(
     let parse_duration = parse_time.elapsed();
 
     let stdout = io::stdout();
-    let mut stdout = stdout.lock();
+    let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
 
     if let Some(mut tree) = tree {
         if opts.debug_graph && !opts.edits.is_empty() {
@@ -510,12 +510,11 @@ pub fn parse_file_at_path(
                 }
             }
             cursor.reset(tree.root_node());
-            println!();
+            writeln!(&mut stdout)?;
         }
 
         if opts.output == ParseOutput::Cst {
             render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
-            println!();
         }
 
         if opts.output == ParseOutput::Xml {
@@ -582,11 +581,11 @@ pub fn parse_file_at_path(
                         }
                         let start = node.start_position();
                         let end = node.end_position();
-                        write!(&mut stdout, " srow=\"{}\"", start.row)?;
-                        write!(&mut stdout, " scol=\"{}\"", start.column)?;
-                        write!(&mut stdout, " erow=\"{}\"", end.row)?;
-                        write!(&mut stdout, " ecol=\"{}\"", end.column)?;
-                        write!(&mut stdout, ">")?;
+                        write!(
+                            &mut stdout,
+                            " srow=\"{}\" scol=\"{}\" erow=\"{}\" ecol=\"{}\">",
+                            start.row, start.column, end.row, end.column
+                        )?;
                         tags.push(node.kind());
                         needs_newline = true;
                     }
@@ -782,10 +781,14 @@ pub fn render_cst<'a, 'b: 'a>(
     let total_width = lossy_source_code
         .lines()
         .enumerate()
-        .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
+        .map(|(row, col)| {
+            row.checked_ilog10().unwrap_or(0) as usize
+                + col.len().checked_ilog10().unwrap_or(0) as usize
+                + 1
+        })
         .max()
         .unwrap_or(1);
-    let mut indent_level = 1;
+    let mut indent_level = usize::from(!opts.no_ranges);
     let mut did_visit_children = false;
     let mut in_error = false;
     loop {
@@ -883,35 +886,24 @@ fn write_node_text(
                     0
                 };
             let formatted_line = render_line_feed(line, opts);
-            if !opts.no_ranges {
-                write!(
-                    out,
-                    "{}{}{}{}{}{}",
-                    if multiline { "\n" } else { "" },
-                    if multiline {
-                        render_node_range(opts, cursor, is_named, true, total_width, node_range)
-                    } else {
-                        String::new()
-                    },
-                    if multiline {
-                        "  ".repeat(indent_level + 1)
-                    } else {
-                        String::new()
-                    },
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            } else {
-                write!(
-                    out,
-                    "\n{}{}{}{}",
-                    "  ".repeat(indent_level + 1),
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            }
+            write!(
+                out,
+                "{}{}{}{}{}{}",
+                if multiline { "\n" } else { " " },
+                if multiline && !opts.no_ranges {
+                    render_node_range(opts, cursor, is_named, true, total_width, node_range)
+                } else {
+                    String::new()
+                },
+                if multiline {
+                    "  ".repeat(indent_level + 1)
+                } else {
+                    String::new()
+                },
+                paint(quote_color, &String::from(quote)),
+                paint(color, &render_node_text(&formatted_line)),
+                paint(quote_color, &String::from(quote)),
+            )?;
         }
     }
 
@@ -935,30 +927,27 @@ fn render_node_range(
     range: Range,
 ) -> String {
     let has_field_name = cursor.field_name().is_some();
+    let start = range.start_point;
+    let end = range.end_point;
     let range_color = if is_named && !is_multiline && !has_field_name {
         opts.parse_theme.row_color_named
     } else {
         opts.parse_theme.row_color
     };
 
-    let remaining_width_start = (total_width
-        - (range.start_point.row as f64).log10() as usize
-        - (range.start_point.column as f64).log10() as usize)
-        .max(1);
-    let remaining_width_end = (total_width
-        - (range.end_point.row as f64).log10() as usize
-        - (range.end_point.column as f64).log10() as usize)
-        .max(1);
+    let remaining_width = |row: usize, col: usize| {
+        (total_width
+            .saturating_sub(row.checked_ilog10().unwrap_or(0) as usize)
+            .saturating_sub(col.checked_ilog10().unwrap_or(0) as usize))
+        .max(1)
+    };
+    let remaining_width_start = remaining_width(start.row, start.column);
+    let remaining_width_end = remaining_width(end.row, end.column);
     paint(
         range_color,
         &format!(
             "{}:{}{:remaining_width_start$}- {}:{}{:remaining_width_end$}",
-            range.start_point.row,
-            range.start_point.column,
-            ' ',
-            range.end_point.row,
-            range.end_point.column,
-            ' ',
+            start.row, start.column, ' ', end.row, end.column, ' ',
         ),
     )
 }
@@ -1011,10 +1000,9 @@ fn cst_render_node(
         } else {
             opts.parse_theme.node_kind
         };
-        write!(out, "{}", paint(kind_color, node.kind()),)?;
+        write!(out, "{}", paint(kind_color, node.kind()))?;
 
         if node.child_count() == 0 {
-            write!(out, " ")?;
             // Node text from a pattern or external scanner
             write_node_text(
                 opts,

crates/cli/src/query.rs

@@ -37,7 +37,7 @@ pub fn query_file_at_path(
     test_summary: Option<&mut TestSummary>,
 ) -> Result<()> {
     let stdout = io::stdout();
-    let mut stdout = stdout.lock();
+    let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
 
     let query_source = fs::read_to_string(query_path)
         .with_context(|| format!("Error reading query file {}", query_path.display()))?;

crates/cli/src/tags.rs

@@ -49,7 +49,7 @@ pub fn generate_tags(
                 &mut stdout,
                 "{indent_str}{:<10}\t | {:<8}\t{} {} - {} `{}`",
                 str::from_utf8(&source[tag.name_range]).unwrap_or(""),
-                &config.syntax_type_name(tag.syntax_type_id),
+                config.syntax_type_name(tag.syntax_type_id),
                 if tag.is_definition { "def" } else { "ref" },
                 tag.span.start,
                 tag.span.end,
@@ -59,7 +59,7 @@ pub fn generate_tags(
                 if docs.len() > 120 {
                     write!(&mut stdout, "\t{:?}...", docs.get(0..120).unwrap_or(""))?;
                 } else {
-                    write!(&mut stdout, "\t{:?}", &docs)?;
+                    write!(&mut stdout, "\t{docs:?}")?;
                 }
             }
             writeln!(&mut stdout)?;

crates/cli/src/templates/makefile

@@ -1,6 +1,7 @@
 LANGUAGE_NAME := tree-sitter-KEBAB_PARSER_NAME
 HOMEPAGE_URL := PARSER_URL
 VERSION := PARSER_VERSION
+DESCRIPTION := PARSER_DESCRIPTION
 
 # repository
 SRC_DIR := src

crates/cli/src/templates/package.swift

@@ -14,7 +14,7 @@ let package = Package(
         .library(name: "PARSER_CLASS_NAME", targets: ["PARSER_CLASS_NAME"]),
     ],
     dependencies: [
-        .package(name: "SwiftTreeSitter", url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.9.0"),
+        .package(url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.10.0"),
     ],
     targets: [
         .target(
@@ -31,7 +31,7 @@ let package = Package(
         .testTarget(
             name: "PARSER_CLASS_NAMETests",
             dependencies: [
-                "SwiftTreeSitter",
+                .product(name: "SwiftTreeSitter", package: "swift-tree-sitter"),
                 "PARSER_CLASS_NAME",
             ],
             path: "bindings/swift/PARSER_CLASS_NAMETests"

crates/cli/src/templates/setup.py

@@ -32,7 +32,7 @@ class BuildExt(build_ext):
 class BdistWheel(bdist_wheel):
     def get_tag(self):
         python, abi, platform = super().get_tag()
-        if python.startswith("cp"):
+        if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
             python, abi = "cp310", "abi3"
         return python, abi, platform
 
@@ -42,6 +42,7 @@ class EggInfo(egg_info):
         super().find_sources()
         self.filelist.recursive_include("queries", "*.scm")
         self.filelist.include("src/tree_sitter/*.h")
+        self.filelist.include("src/*.c")
 
 
 setup(

crates/cli/src/test.rs

@@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
             render_assertion_results("queries", &self.query_results)?;
         }
 
+        write!(f, "{}", self.parse_stats)?;
+
         Ok(())
     }
 }
@@ -605,11 +607,13 @@ pub fn run_tests_at_path(
     test_summary: &mut TestSummary,
 ) -> Result<()> {
     let test_entry = parse_tests(&opts.path)?;
-    let mut _log_session = None;
 
-    if opts.debug_graph {
-        _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?);
-    } else if opts.debug {
+    let _log_session = if opts.debug_graph {
+        Some(util::log_graphs(parser, "log.html", opts.open_log)?)
+    } else {
+        None
+    };
+    if opts.debug {
         parser.set_logger(Some(Box::new(|log_type, message| {
             if log_type == LogType::Lex {
                 io::stderr().write_all(b"  ").unwrap();
@@ -642,22 +646,20 @@ pub fn run_tests_at_path(
 }
 
 pub fn check_queries_at_path(language: &Language, path: &Path) -> Result<()> {
-    if path.exists() {
-        for entry in WalkDir::new(path)
-            .into_iter()
-            .filter_map(std::result::Result::ok)
-            .filter(|e| {
-                e.file_type().is_file()
-                    && e.path().extension().and_then(OsStr::to_str) == Some("scm")
-                    && !e.path().starts_with(".")
-            })
-        {
-            let filepath = entry.file_name().to_str().unwrap_or("");
-            let content = fs::read_to_string(entry.path())
-                .with_context(|| format!("Error reading query file {filepath:?}"))?;
-            Query::new(language, &content)
-                .with_context(|| format!("Error in query file {filepath:?}"))?;
-        }
+    for entry in WalkDir::new(path)
+        .into_iter()
+        .filter_map(std::result::Result::ok)
+        .filter(|e| {
+            e.file_type().is_file()
+                && e.path().extension().and_then(OsStr::to_str) == Some("scm")
+                && !e.path().starts_with(".")
+        })
+    {
+        let filepath = entry.file_name().to_str().unwrap_or("");
+        let content = fs::read_to_string(entry.path())
+            .with_context(|| format!("Error reading query file {filepath:?}"))?;
+        Query::new(language, &content)
+            .with_context(|| format!("Error in query file {filepath:?}"))?;
     }
     Ok(())
 }
@@ -867,13 +869,14 @@ fn run_tests(
                 let tree = parser.parse(&input, None).unwrap();
                 let parse_rate = {
                     let parse_time = start.elapsed();
-                    let true_parse_rate = tree.root_node().byte_range().len() as f64
-                        / (parse_time.as_nanos() as f64 / 1_000_000.0);
+                    let byte_len = tree.root_node().byte_range().len();
+                    let true_parse_rate =
+                        byte_len as f64 / (parse_time.as_nanos() as f64 / 1_000_000.0);
                     let adj_parse_rate = adjusted_parse_rate(&tree, parse_time);
 
                     test_summary.parse_stats.total_parses += 1;
                     test_summary.parse_stats.total_duration += parse_time;
-                    test_summary.parse_stats.total_bytes += tree.root_node().byte_range().len();
+                    test_summary.parse_stats.total_bytes += byte_len;
 
                     Some((true_parse_rate, adj_parse_rate))
                 };

crates/cli/src/test_highlight.rs

@@ -13,10 +13,10 @@ use crate::{
 
 #[derive(Debug)]
 pub struct Failure {
-    row: usize,
-    column: usize,
-    expected_highlight: String,
-    actual_highlights: Vec<String>,
+    pub(crate) row: usize,
+    pub(crate) column: usize,
+    pub(crate) expected_highlight: String,
+    pub(crate) actual_highlights: Vec<String>,
 }
 
 impl std::error::Error for Failure {}
@@ -126,6 +126,7 @@ pub fn test_highlights(
         Ok(())
     }
 }
+
 pub fn iterate_assertions(
     assertions: &[Assertion],
     highlights: &[(Utf8Point, Utf8Point, Highlight)],
@@ -142,49 +143,48 @@ pub fn iterate_assertions(
         expected_capture_name: expected_highlight,
     } in assertions
     {
-        let mut passed = false;
-        let mut end_column = position.column + length - 1;
+        // Iterate through all of the highlights that start at or before this assertion's
+        // position, looking for one that matches the assertion.
         actual_highlights.clear();
-
-        // The assertions are ordered by position, so skip past all of the highlights that
-        // end at or before this assertion's position.
-        'highlight_loop: while let Some(highlight) = highlights.get(i) {
+        let mut passed = false;
+        let end_column = position.column + length - 1;
+        for highlight in &highlights[i..] {
+            // The assertions are ordered by position, so skip past all of the highlights that
+            // end at or before this assertion's position.
             if highlight.1 <= *position {
                 i += 1;
                 continue;
             }
+            if (highlight.0.row > position.row)
+                || (highlight.0.row == position.row && highlight.0.column > end_column)
+            {
+                break;
+            }
 
-            // Iterate through all of the highlights that start at or before this assertion's
-            // position, looking for one that matches the assertion.
-            let mut j = i;
-            while let (false, Some(highlight)) = (passed, highlights.get(j)) {
-                end_column = position.column + length - 1;
-                if highlight.0.row >= position.row && highlight.0.column > end_column {
-                    break 'highlight_loop;
-                }
-
-                // If the highlight matches the assertion, or if the highlight doesn't
-                // match the assertion but it's negative, this test passes. Otherwise,
-                // add this highlight to the list of actual highlights that span the
-                // assertion's position, in order to generate an error message in the event
-                // of a failure.
-                let highlight_name = &highlight_names[(highlight.2).0];
-                if (*highlight_name == *expected_highlight) == *negative {
-                    actual_highlights.push(highlight_name);
-                } else {
-                    passed = true;
-                    break 'highlight_loop;
-                }
-
-                j += 1;
+            // If the highlight matches the assertion, or if the highlight doesn't
+            // match the assertion but it's negative, this test passes. Otherwise,
+            // add this highlight to the list of actual highlights that span the
+            // assertion's position, in order to generate an error message in the event
+            // of a failure.
+            let highlight_name = &highlight_names[(highlight.2).0];
+            if (*highlight_name == *expected_highlight) == *negative {
+                actual_highlights.push(highlight_name);
+            } else {
+                passed = true;
+                break;
             }
         }
 
         if !passed {
+            let mut expected = String::with_capacity(expected_highlight.len() + 1);
+            if *negative {
+                expected.push('!');
+            }
+            expected.push_str(expected_highlight);
             return Err(Failure {
                 row: position.row,
                 column: end_column,
-                expected_highlight: expected_highlight.clone(),
+                expected_highlight: expected,
                 actual_highlights: actual_highlights.into_iter().cloned().collect(),
             }
             .into());

crates/cli/src/tests/corpus_test.rs

@@ -274,7 +274,7 @@ pub fn test_language_corpus(
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
+                    println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 
@@ -393,12 +393,12 @@ fn test_feature_corpus_files() {
                     failure_count += 1;
                 }
             } else {
-                eprintln!("Expected error message but got none for test grammar '{language_name}'",);
+                eprintln!("Expected error message but got none for test grammar '{language_name}'");
                 failure_count += 1;
             }
         } else {
             if let Err(e) = &generate_result {
-                eprintln!("Unexpected error for test grammar '{language_name}':\n{e}",);
+                eprintln!("Unexpected error for test grammar '{language_name}':\n{e}");
                 failure_count += 1;
                 continue;
             }

crates/cli/src/tests/detect_language.rs

@@ -127,6 +127,52 @@ fn detect_language_by_double_barrel_file_extension() {
     );
 }
 
+#[test]
+fn detect_language_with_dots_in_filename() {
+    let blade_dir = tree_sitter_dir(
+        r#"{
+  "grammars": [
+    {
+      "name": "blade_dots",
+      "path": ".",
+      "scope": "source.blade",
+      "file-types": [
+        "blade.php"
+      ]
+    },
+    {
+      "name": "php_dots",
+      "path": ".",
+      "scope": "source.php",
+      "file-types": [
+        "php"
+      ]
+    }
+  ],
+  "metadata": {
+    "version": "0.0.1"
+  }
+}
+"#,
+        "blade_dots",
+    );
+
+    let mut loader = Loader::with_parser_lib_path(scratch_dir().to_path_buf());
+    let config = loader
+        .find_language_configurations_at_path(blade_dir.path(), false)
+        .unwrap();
+
+    // this is just to validate that we can read the tree-sitter.json correctly
+    assert_eq!(config[0].scope.as_ref().unwrap(), "source.blade");
+
+    let file_name = blade_dir.path().join("foo.bar.baz.blade.php");
+    fs::write(&file_name, "").unwrap();
+    assert_eq!(
+        get_lang_scope(&loader, &file_name),
+        Some("source.blade".into())
+    );
+}
+
 #[test]
 fn detect_language_without_filename() {
     let gitignore_dir = tree_sitter_dir(

crates/cli/src/tests/parser_test.rs

@@ -249,6 +249,64 @@ fn test_parsing_with_custom_utf16_be_input() {
     assert_eq!(root.child(0).unwrap().kind(), "function_item");
 }
 
+#[test]
+fn test_utf16_decode_does_not_read_oob() {
+    // Test for a buffer over-read in ts_decode_utf16_le/be when a lead surrogate
+    // is the last code unit in a chunk. The test grammar's external scanner
+    // distinguishes surrogate code points from supplementary-plane characters,
+    // making the over-read directly observable in the parse tree.
+    //
+    // Buffer layout:
+    //   buf[0] = 0xD83E  (lead surrogate)
+    //   buf[1] = 0xDD8B  (POISON: fake trail surrogate, adjacent in memory)
+    //
+    // The callback returns only buf[0..1] (one code unit = 2 bytes).
+    //
+    // When functioning correctly, this test passes a length of 2 bytes, which is
+    // interpreted as 2/2 = 1 code unit, and thus doesn't over-read into the "poison"
+    // fake trail surrogate. If an over-read does occur, the scanner sees a
+    // supplementary token.
+    let mut parser = Parser::new();
+    let language = get_test_fixture_language("utf16_surrogate_oob");
+    parser.set_language(&language).unwrap();
+
+    let buf = vec![
+        0xD83E, // lead surrogate (the only "visible" code unit)
+        0xDD8B, // POISON: adjacent in Vec memory, past the chunk
+    ];
+    assert_eq!("🦋", String::from_utf16(&buf).unwrap());
+
+    let mut callback = |offset: usize, _position: Point| -> &[u16] {
+        // only expose buf[0], never buf[1]
+        if offset >= 1 {
+            return [].as_slice();
+        }
+        &buf[0..1]
+    };
+
+    // Use the parse function matching the host endianness, since the
+    // buffer contains native u16 values.
+    #[cfg(target_endian = "little")]
+    let tree = parser
+        .parse_utf16_le_with_options(&mut callback, None, None)
+        .unwrap();
+    #[cfg(target_endian = "big")]
+    let tree = parser
+        .parse_utf16_be_with_options(&mut callback, None, None)
+        .unwrap();
+
+    let root = tree.root_node();
+
+    // Correct: scanner sees raw surrogate (0xD83E) -> `surrogate` node
+    // Incorrect: scanner sees supplementary (U+1F98B, aka 🦋) -> `supplementary` node
+    assert_eq!(
+        root.to_sexp(),
+        "(program (surrogate))",
+        "buffer over-read: decoder read past chunk boundary and formed a \
+         supplementary character from OOB adjacent memory"
+    );
+}
+
 #[test]
 fn test_parsing_with_callback_returning_owned_strings() {
     let mut parser = Parser::new();
@@ -1071,7 +1129,7 @@ fn test_parsing_with_timeout_during_balancing() {
                 Some(ParseOptions::new().progress_callback(&mut |state| {
                     // Because we've already finished parsing, we should only be resuming the
                     // balancing phase.
-                    assert!(state.current_byte_offset() == current_byte_offset);
+                    assert_eq!(state.current_byte_offset(), current_byte_offset);
                     ControlFlow::Continue(())
                 })),
             )

crates/cli/src/tests/query_test.rs

@@ -255,6 +255,51 @@ fn test_query_errors_on_invalid_syntax() {
     });
 }
 
+#[test]
+fn test_query_errors_on_anchor_at_group_edge() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+
+        // Anchors between siblings, or at the first/last position of a *node*
+        // pattern, are valid.
+        assert!(Query::new(&language, "((_) . (_))").is_ok());
+        assert!(Query::new(&language, "(program (_) (_) .)").is_ok());
+        assert!(Query::new(&language, "(program (_)* @x . (_))").is_ok());
+
+        // A `.` at the edge of a *group* is rejected. A group is not a node, so it
+        // has no last child to anchor, and there is no sibling within the group to
+        // anchor to.
+        assert_eq!(
+            Query::new(&language, "((_) .)").unwrap_err(),
+            QueryError {
+                row: 0,
+                offset: 5,
+                column: 5,
+                kind: QueryErrorKind::Syntax,
+                message: [
+                    "((_) .)", //
+                    "     ^"
+                ]
+                .join("\n")
+            }
+        );
+        assert_eq!(
+            Query::new(&language, "(program ((_)+ .)? (_))").unwrap_err(),
+            QueryError {
+                row: 0,
+                offset: 15,
+                column: 15,
+                kind: QueryErrorKind::Syntax,
+                message: [
+                    "(program ((_)+ .)? (_))", //
+                    "               ^"
+                ]
+                .join("\n")
+            }
+        );
+    });
+}
+
 #[test]
 fn test_query_errors_on_invalid_symbols() {
     allocations::record(|| {
@@ -370,6 +415,16 @@ fn test_query_errors_on_invalid_symbols() {
                 message: "\"fakefield\"".to_string()
             }
         );
+        assert_eq!(
+            Query::new(&language, "(MISS)").unwrap_err(),
+            QueryError {
+                row: 0,
+                offset: 1,
+                column: 1,
+                kind: QueryErrorKind::NodeType,
+                message: "\"MISS\"".to_string(),
+            }
+        );
     });
 }
 
@@ -1176,6 +1231,209 @@ fn test_query_matches_with_immediate_siblings() {
     });
 }
 
+#[test]
+fn test_query_matches_with_anchor_after_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            "(program (comment)* @doc . (function_declaration name: (identifier) @name))",
+        )
+        .unwrap();
+
+        // No comments and the function is not the first child. An anchor after a
+        // zero-matched quantifier is vacuous, so the function still matches.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+class X {}
+function foo() {}
+",
+            &[(0, vec![("name", "foo")])],
+        );
+
+        // With at least one comment the anchor applies, so the comments must
+        // immediately precede the function.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+// c
+function foo() {}
+",
+            &[(0, vec![("doc", "// c"), ("name", "foo")])],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_anchor_after_nested_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            r#"
+            (_
+              (field_definition
+                property: (_) @name
+                value: (_)? @value
+              ) @field
+              .
+              ";" @semicolon
+            )
+            "#,
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "class Foo { bar; baz = 0; }",
+            &[
+                (
+                    0,
+                    vec![("field", "bar"), ("name", "bar"), ("semicolon", ";")],
+                ),
+                (
+                    0,
+                    vec![
+                        ("field", "baz = 0"),
+                        ("name", "baz"),
+                        ("value", "0"),
+                        ("semicolon", ";"),
+                    ],
+                ),
+            ],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_last_child_anchor_after_optional() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(preproc_if (preproc_def)+ @def . (preproc_else)? @else .)",
+        )
+        .unwrap();
+
+        // The optional `(preproc_else)?` is absent, so the trailing anchor's
+        // last-child requirement transfers to the last `preproc_def`. A trailing
+        // comment means the def is not the last child, so nothing matches.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+#if X
+#define A
+// c
+#endif
+",
+            &[],
+        );
+
+        // With the def as the last child, the (else-less) match is allowed.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+#if X
+#define A
+#endif
+",
+            &[(0, vec![("def", "#define A\n")])],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_anchors_on_both_sides_of_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            "(program (lexical_declaration) @a . (comment)* . (function_declaration) @b)",
+        )
+        .unwrap();
+
+        // Anchors on both sides of a zero-matched quantifier collapse into a single
+        // adjacency constraint: with no comments, the declaration must be immediately
+        // followed by the function.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+const a = 1;
+const b = 2;
+function foo() {}
+",
+            &[(0, vec![("a", "const b = 2;"), ("b", "function foo() {}")])],
+        );
+
+        // With a comment present the quantifier is non-zero, so the anchors apply
+        // normally: the comment must sit immediately between the declaration and the
+        // function.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+const b = 2;
+// c
+function foo() {}
+",
+            &[(0, vec![("a", "const b = 2;"), ("b", "function foo() {}")])],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_leading_anchor_before_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(translation_unit . (comment)* (function_definition) @f)",
+        )
+        .unwrap();
+
+        // The leading `.` anchors the comment run to the parent's first child. When the
+        // run matches zero comments, that first-child requirement transfers to the
+        // function, so it matches only when it is itself the first child.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+int main() {}
+",
+            &[(0, vec![("f", "int main() {}")])],
+        );
+
+        // The function is the second child, so with no leading comments it must not match.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+int a;
+int main() {}
+",
+            &[],
+        );
+
+        // With a leading comment the run starts at the first child and the function follows.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+// c
+int main() {}
+",
+            &[(0, vec![("f", "int main() {}")])],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_last_named_child() {
     allocations::record(|| {
@@ -1348,6 +1606,35 @@ fn test_query_matches_with_repeated_leaf_nodes() {
     });
 }
 
+#[test]
+fn test_query_matches_optional_capture_before_uncaptured_required_sibling() {
+    allocations::record(|| {
+        let language = get_language("rust");
+        let query = Query::new(&language, "(block (line_comment)? @doc (line_comment))").unwrap();
+
+        // The optional `(line_comment)? @doc` before an *uncaptured* required
+        // `(line_comment)` yields two candidate completions at the block:
+        //     - one where the optional captured `// a` (the required node is then `// b`),
+        //     - one where the optional matched zero (the required node absorbs `// a`,
+        //     leaving `@doc` unbound).
+        // The zero-match completion's captures are a strict subset of the other's, so the
+        // longest-match rule must drop it (there is exactly one match). This regressed when
+        // the dedup pass gained an early-break. Without the accompanying capture-position
+        // sort, the break skips the subset "loser" and it leaks as an extra empty match.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            fn f() {
+                // a
+                // b
+            }
+            ",
+            &[(0, vec![("doc", "// a")])],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_optional_nodes_inside_of_repetitions() {
     allocations::record(|| {
@@ -1592,6 +1879,148 @@ fn test_query_matches_with_leading_zero_or_more_repeated_leaf_nodes() {
     });
 }
 
+#[test]
+fn test_matches_with_anchor_sibling_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[(0, vec![("name", "a")]), (0, vec![("name", "b")])],
+        );
+    });
+}
+
+#[test]
+fn test_matches_with_anchor_sibling_with_quantifier_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)+
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[(0, vec![("name", "a")]), (0, vec![("name", "b")])],
+        );
+    });
+}
+
+#[test]
+fn test_matches_with_anchor_sibling_with_quantifier_captured_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)+ @doc
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[
+                (0, vec![("doc", "// A"), ("name", "a")]),
+                (0, vec![("doc", "// B"), ("name", "b")]),
+            ],
+        );
+    });
+}
+
+#[test]
+fn test_matches_anchored_quantified_sibling_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(translation_unit (comment)* @comment . (declaration) @decl)",
+        )
+        .unwrap();
+        assert_query_matches(
+            &language,
+            &query,
+            "
+void foo() {}
+
+// this one has
+// two comments
+extern int baz;
+
+// this one has a comment
+extern int bar;
+",
+            &[
+                (
+                    0,
+                    vec![
+                        ("comment", "// this one has"),
+                        ("comment", "// two comments"),
+                        ("decl", "extern int baz;"),
+                    ],
+                ),
+                (
+                    0,
+                    vec![
+                        ("comment", "// this one has a comment"),
+                        ("decl", "extern int bar;"),
+                    ],
+                ),
+            ],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_trailing_optional_nodes() {
     allocations::record(|| {
@@ -3075,6 +3504,74 @@ fn test_query_matches_with_deeply_nested_patterns_with_fields() {
     });
 }
 
+#[test]
+fn test_query_alternation_with_inner_quantifier() {
+    let language = get_language("c");
+    let source_code = "#include <foo>
+#include <bar>
+#include <baz>
+
+// comment";
+    let matches = &[
+        (
+            0,
+            vec![
+                ("capture", "#include <foo>\n"),
+                ("capture", "#include <bar>\n"),
+                ("capture", "#include <baz>\n"),
+            ],
+        ),
+        (0, vec![("capture", "// comment")]),
+    ];
+
+    let query = "[
+       (preproc_include)+
+       (comment)
+    ] @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+
+    let query = "[
+       (comment)
+       (preproc_include)+
+    ] @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+}
+
+#[test]
+fn test_query_alternation_with_outer_quantifier() {
+    let language = get_language("c");
+    let source_code = "#include <foo>
+#include <bar>
+#include <baz>
+
+// comment";
+    let matches = &[(
+        0,
+        vec![
+            ("capture", "#include <foo>\n"),
+            ("capture", "#include <bar>\n"),
+            ("capture", "#include <baz>\n"),
+            ("capture", "// comment"),
+        ],
+    )];
+
+    let query = "[
+        (preproc_include)
+        (comment)
+    ]+ @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+
+    let query = "([
+        (preproc_include)
+        (comment)
+    ] (_)?)+ @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+}
+
 #[test]
 fn test_query_matches_with_alternations_and_predicates() {
     allocations::record(|| {
@@ -5961,3 +6458,64 @@ export default grammar({
 
     assert_query_matches(&language, &query, source, &[(0, vec![("tuple", "()")])]);
 }
+
+#[test]
+fn test_last_child_anchor_looks_past_hidden_repeat() {
+    let language = get_test_fixture_language("last_child_anchor_past_hidden_repeat");
+
+    let source = "T a.b.c\nL a.b.c\nN a!.b!.c\n";
+
+    let query = Query::new(
+        &language,
+        "
+        (trailing_sep (name) @last .)
+        (leading_sep (name) @last .)
+        (trailing_named (name) @last .)
+        ",
+    )
+    .unwrap();
+
+    assert_query_matches(
+        &language,
+        &query,
+        source,
+        &[
+            (0, vec![("last", "c")]),
+            (1, vec![("last", "c")]),
+            (2, vec![("last", "c")]),
+        ],
+    );
+
+    let query = Query::new(
+        &language,
+        "
+        (trailing_sep . (name) @first)
+        (trailing_sep (name) @a . (name) @b)
+        ",
+    )
+    .unwrap();
+
+    assert_query_matches(
+        &language,
+        &query,
+        source,
+        &[
+            (0, vec![("first", "a")]),
+            (1, vec![("a", "a"), ("b", "b")]),
+            (1, vec![("a", "b"), ("b", "c")]),
+        ],
+    );
+}
+
+#[test]
+fn test_last_child_anchor_looks_past_hidden_node() {
+    allocations::record(|| {
+        let language = get_language("c");
+
+        let query = Query::new(&language, "(translation_unit (_) @last .)").unwrap();
+
+        let source = "enum E { A };\nint x;\nint y;\n";
+
+        assert_query_matches(&language, &query, source, &[(0, vec![("last", "int y;")])]);
+    });
+}

crates/cli/src/tests/test_highlight_test.rs

@@ -4,7 +4,7 @@ use tree_sitter_highlight::{Highlight, Highlighter};
 use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
 use crate::{
     query_testing::{parse_position_comments, Assertion, Utf8Point},
-    test_highlight::get_highlight_positions,
+    test_highlight::{get_highlight_positions, iterate_assertions, Failure},
 };
 
 #[test]
@@ -68,3 +68,195 @@ fn test_highlight_test_with_basic_test() {
         ]
     );
 }
+
+#[test]
+fn test_assertion_with_non_matching_highlight_at_same_position() {
+    // Test that an assertion fails when the highlight at the position does not match
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![Assertion::new(1, 0, 1, false, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(1, 0), Utf8Point::new(1, 5), Highlight(1)), // "variable" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 1);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, vec!["variable".to_string()]);
+}
+
+#[test]
+fn test_assertion_with_exact_matching_highlight() {
+    // Test exact match: assertion and highlight have same start and end
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 5, 3, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_contained_within_highlight() {
+    // Test where assertion is fully contained within a larger highlight
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 3, 2, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 0), Utf8Point::new(0, 10), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_overlapping_highlight_start() {
+    // Test where assertion starts before highlight but overlaps with it
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 3, 4, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 5), Utf8Point::new(0, 10), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_with_no_highlights() {
+    // Test that an assertion fails when there are no highlights at all
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, false, String::from("keyword"))];
+    let highlights = vec![];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}
+
+#[test]
+fn test_assertion_with_highlight_ending_before() {
+    // Test where highlight ends before the assertion starts
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 10, 1, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 10);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}
+
+#[test]
+fn test_negative_assertion_with_non_matching_highlight() {
+    // Test that a negative assertion passes when the specified highlight is NOT present
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, true, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(1)), // "variable" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_negative_assertion_with_matching_highlight() {
+    // Test that a negative assertion fails when the specified highlight IS present
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, true, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(0)), // "keyword" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "!keyword");
+    assert_eq!(err.actual_highlights, vec!["keyword".to_string()]);
+}
+
+#[test]
+fn test_multiple_assertions_sequential() {
+    // Test multiple assertions in sequence with non-overlapping highlights
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![
+        Assertion::new(0, 0, 3, false, String::from("keyword")),
+        Assertion::new(0, 10, 1, false, String::from("variable")),
+    ];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 3), Highlight(0)), // "keyword"
+        (Utf8Point::new(0, 10), Utf8Point::new(0, 11), Highlight(1)), // "variable"
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 2);
+}
+
+#[test]
+fn test_multiple_highlights_at_same_position() {
+    // Test where multiple highlights overlap at the assertion position
+    let highlight_names = vec![
+        "keyword".to_string(),
+        "variable".to_string(),
+        "function".to_string(),
+    ];
+    let assertions = vec![Assertion::new(0, 5, 1, false, String::from("variable"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 10), Highlight(0)), // "keyword" spans entire range
+        (Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(1)), // "variable" at assertion position
+        (Utf8Point::new(0, 7), Utf8Point::new(0, 12), Highlight(2)), // "function" overlaps
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertions_across_multiple_rows() {
+    // Test assertions on different rows
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![
+        Assertion::new(0, 5, 3, false, String::from("keyword")),
+        Assertion::new(2, 10, 1, false, String::from("variable")),
+    ];
+    let highlights = vec![
+        (Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(0)), // "keyword" on row 0
+        (Utf8Point::new(2, 10), Utf8Point::new(2, 11), Highlight(1)), // "variable" on row 2
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 2);
+}
+
+#[test]
+fn test_assertion_should_not_match_highlight_on_later_row() {
+    // Test logic for early exit when highlight is on a later row than the assertion
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 5, 3, false, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(1, 0), Utf8Point::new(1, 5), Highlight(0)), // wrong row
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 7); // end_column
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}

crates/cli/src/tests/tree_test.rs

@@ -795,3 +795,44 @@ fn get_changed_ranges(
     *tree = new_tree;
     result
 }
+
+// Regression test for an incremental reparse bug where an external
+// scanner's choice depends on lexer->eof() (and thus on the parser's
+// current included ranges). The cached token at byte 0 was emitted when
+// the included range stopped just after the opener. Widening the range
+// to include a later matching delimiter must invalidate that cached
+// token so the scanner re-runs and emits the open form instead of the
+// unclosed form.
+#[test]
+fn test_reuse_invalidates_scanner_token_when_included_range_expands() {
+    let language = get_test_fixture_language("external_lookahead_eof_boundary");
+    let mut parser = Parser::new();
+    parser.set_language(&language).unwrap();
+
+    let source = "``";
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 1,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 1),
+        }])
+        .unwrap();
+    let tree1 = parser.parse(source, None).unwrap();
+    assert_eq!(tree1.root_node().to_sexp(), "(document (unclosed_delim))");
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 2,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 2),
+        }])
+        .unwrap();
+    let tree2 = parser.parse(source, Some(&tree1)).unwrap();
+    assert_eq!(
+        tree2.root_node().to_sexp(),
+        "(document (span (open_delim) (close_delim)))"
+    );
+}

crates/cli/src/tests/wasm_language_test.rs

@@ -118,6 +118,32 @@ fn test_load_fixture_language_wasm() {
     });
 }
 
+#[test]
+fn test_wasm_realloc_smaller_size() {
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let mut parser = Parser::new();
+        let language = get_test_fixture_language_wasm("wasm_realloc_overflow_heap");
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse("hello", None).unwrap();
+        assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
+    });
+}
+
+#[test]
+fn test_wasm_realloc_clobber_region() {
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let mut parser = Parser::new();
+        let language = get_test_fixture_language_wasm("wasm_realloc_clobber_region");
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse("hello", None).unwrap();
+        assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
+    });
+}
+
 #[test]
 fn test_load_multiple_wasm_languages() {
     allocations::record(|| {
@@ -273,6 +299,55 @@ fn test_load_wasm_errors() {
     });
 }
 
+#[test]
+fn test_load_wasm_language_with_reserved_words() {
+    // This test exercises a grammar with multiple reserved word sets loaded via WASM.
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let language = get_test_fixture_language_wasm("reserved_words");
+
+        let mut parser = Parser::new();
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+
+        // "if" and "while" are globally reserved, so using them as identifiers
+        // should produce an error recovery.
+        let tree = parser
+            .parse("var a =\n\nif (something) {\n  c();\n}", None)
+            .unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            concat!(
+                "(program ",
+                "(ERROR (identifier)) ",
+                "(if_statement (parenthesized_expression (identifier)) ",
+                "(block (expression_statement (call_expression (identifier))))))",
+            )
+        );
+
+        // "if" and "while" are NOT reserved in the 'property' context, so they
+        // can appear as object keys without error.
+        let tree = parser
+            .parse("var x = {\n  if: a,\n  while: b,\n};", None)
+            .unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            concat!(
+                "(program (var_declaration (identifier) (object ",
+                "(pair (identifier) (identifier)) (pair (identifier) (identifier)))))"
+            )
+        );
+
+        // "var" IS reserved in the 'property' context, so using it as a property
+        // key triggers error recovery.
+        let tree = parser.parse("var x = {\nvar y = z;", None).unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            "(program (ERROR (identifier)) (var_declaration (identifier) (identifier)))"
+        );
+    });
+}
+
 #[test]
 fn test_wasm_oom() {
     allocations::record(|| {

crates/cli/src/version.rs

@@ -292,6 +292,9 @@ impl Version {
         } else {
             self.current_dir.join("Makefile")
         };
+        if !makefile_path.exists() {
+            return Ok(());
+        }
 
         self.update_file_with(&makefile_path, |content| {
             content

crates/cli/src/wasm.rs

@@ -15,7 +15,7 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)>
         .unwrap();
     let wasm_filename = format!("tree-sitter-{grammar_name}.wasm");
     let contents = fs::read(language_dir.join(&wasm_filename)).with_context(|| {
-        format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.",)
+        format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.")
     })?;
     Ok((grammar_name, contents))
 }

crates/generate/src/build_tables/build_parse_table.rs

@@ -348,8 +348,7 @@ impl<'a> ParseTableBuilder<'a> {
         if !self.actual_conflicts.is_empty() {
             warn!(
                 "unnecessary conflicts:\n  {}",
-                &self
-                    .actual_conflicts
+                self.actual_conflicts
                     .iter()
                     .map(|conflict| {
                         conflict
@@ -764,6 +763,7 @@ impl<'a> ParseTableBuilder<'a> {
 
             // If the SHIFT action has higher precedence, remove all the REDUCE actions.
             let mut shift_is_less = false;
+            let mut shift_is_equal = false;
             let mut shift_is_more = false;
             for p in shift_precedence {
                 match Self::compare_precedence(
@@ -775,7 +775,7 @@ impl<'a> ParseTableBuilder<'a> {
                 ) {
                     Ordering::Greater => shift_is_more = true,
                     Ordering::Less => shift_is_less = true,
-                    Ordering::Equal => {}
+                    Ordering::Equal => shift_is_equal = true,
                 }
             }
 
@@ -784,8 +784,28 @@ impl<'a> ParseTableBuilder<'a> {
             }
             // If the REDUCE actions have higher precedence, remove the SHIFT action.
             else if shift_is_less && !shift_is_more {
-                entry.actions.pop();
-                conflicting_items.retain(|item| item.is_done());
+                // Exception: if one SHIFT interpretation ties the REDUCE actions in
+                // precedence while another has lower precedence, and the REDUCE
+                // actions are purely right associative, honor that right
+                // associativity by shifting rather than reducing. The
+                // lower-precedence interpretation coexists with the tying one, so on
+                // its own it must not force a REDUCE that would flip the tie to left
+                // associative.
+                if shift_is_equal
+                    && matches!(
+                        (
+                            reduction_info.has_left_assoc,
+                            reduction_info.has_non_assoc,
+                            reduction_info.has_right_assoc,
+                        ),
+                        (false, false, true)
+                    )
+                {
+                    entry.actions.drain(0..entry.actions.len() - 1);
+                } else {
+                    entry.actions.pop();
+                    conflicting_items.retain(|item| item.is_done());
+                }
             }
             // If the SHIFT and REDUCE actions have the same predence, consider
             // the REDUCE actions' associativity.

crates/generate/src/build_tables/minimize_parse_table.rs

@@ -349,7 +349,7 @@ impl Minimizer<'_> {
         new_token: Symbol,
     ) -> bool {
         if new_token == Symbol::end_of_nonterminal_extra() {
-            debug!("split states {left_id} {right_id} - end of non-terminal extra",);
+            debug!("split states {left_id} {right_id} - end of non-terminal extra");
             return true;
         }
 

crates/generate/src/dsl.js

@@ -34,6 +34,9 @@ function blank() {
 }
 
 function field(name, rule) {
+  if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`);
+  }
   return {
     type: "FIELD",
     name,

crates/generate/src/generate.rs

@@ -8,6 +8,7 @@ use std::{
 };
 
 use bitflags::bitflags;
+#[cfg(feature = "load")]
 use log::warn;
 use node_types::VariableInfo;
 use regex::{Regex, RegexBuilder};
@@ -41,7 +42,7 @@ pub use parse_grammar::ParseGrammarError;
 use prepare_grammar::prepare_grammar;
 pub use prepare_grammar::PrepareGrammarError;
 use render::render_c_code;
-pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
+pub use render::{RenderError, ABI_VERSION_MAX, ABI_VERSION_MIN};
 
 static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
     RegexBuilder::new("^\\s*//.*")
@@ -93,6 +94,8 @@ pub enum GenerateError {
     VariableInfo(#[from] VariableInfoError),
     #[error(transparent)]
     BuildTables(#[from] ParseTableBuilderError),
+    #[error(transparent)]
+    Render(#[from] RenderError),
     #[cfg(feature = "load")]
     #[error(transparent)]
     ParseVersion(#[from] ParseVersionError),
@@ -106,6 +109,7 @@ pub struct IoError {
     pub path: Option<String>,
 }
 
+#[cfg(feature = "load")]
 impl IoError {
     fn new(error: &std::io::Error, path: Option<&Path>) -> Self {
         Self {
@@ -333,7 +337,7 @@ pub fn generate_parser_for_grammar(
         LANGUAGE_VERSION,
         semantic_version,
         None,
-        OptLevel::empty(),
+        OptLevel::default(),
     )?;
     Ok((input_grammar.name, parser.c_code))
 }
@@ -398,7 +402,7 @@ fn generate_parser_for_grammar_with_opts(
         abi_version,
         semantic_version,
         supertype_symbol_map,
-    );
+    )?;
     Ok(GeneratedParser {
         c_code,
         #[cfg(feature = "load")]

crates/generate/src/nfa.rs

@@ -280,6 +280,11 @@ impl CharacterSet {
 
     /// Produces a `CharacterSet` containing every character in `self` that is not present in
     /// `other`.
+    #[allow(
+        clippy::must_use_candidate,
+        clippy::return_self_not_must_use,
+        dead_code
+    )]
     pub fn difference(mut self, mut other: Self) -> Self {
         self.remove_intersection(&mut other);
         self

crates/generate/src/node_types.rs

@@ -1,4 +1,6 @@
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+#[cfg(feature = "load")]
+use std::collections::HashSet;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 
 use serde::Serialize;
 use thiserror::Error;
@@ -584,7 +586,13 @@ pub fn generate_node_types_json(
                 kind: node_type_json.kind.clone(),
                 named: true,
             };
-            subtype_map.push((supertype, subtypes.clone()));
+
+            // We only add to the subtype map if there are visible subtypes.
+            // A supertype may have zero subtypes if its children are all
+            // hidden (e.g., wrapping a hidden external token).
+            if !subtypes.is_empty() {
+                subtype_map.push((supertype, subtypes.clone()));
+            }
             node_type_json.subtypes = Some(subtypes);
         } else if !syntax_grammar.variables_to_inline.contains(&symbol) {
             // If a rule is aliased under multiple names, then its information
@@ -1256,6 +1264,49 @@ mod tests {
         );
     }
 
+    /// A supertype whose only child is a hidden external token
+    /// xgust not cause generation to panic. The subtype map must
+    /// skip entries with empty subtypes to avoid a lookup failure
+    /// in the topological sort.
+    #[test]
+    fn test_node_types_supertype_with_only_hidden_child() {
+        let node_types = get_node_types(&InputGrammar {
+            supertype_symbols: vec!["_type_a".to_string(), "_type_b".to_string()],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::seq(vec![Rule::named("_type_a"), Rule::named("_type_b")]),
+                },
+                // Supertype A: a normal choice of named subtypes
+                Variable {
+                    name: "_type_a".to_string(),
+                    kind: VariableType::Hidden,
+                    rule: Rule::choice(vec![Rule::named("v2"), Rule::named("v3")]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                Variable {
+                    name: "v3".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("y"),
+                },
+                // Supertype B: a hidden external token with no subtypes
+                Variable {
+                    name: "_type_b".to_string(),
+                    kind: VariableType::Hidden,
+                    rule: Rule::external(0),
+                },
+            ],
+            external_tokens: vec![Rule::named("_hidden_ext")],
+            ..Default::default()
+        });
+        assert!(node_types.is_ok());
+    }
+
     #[test]
     fn test_node_types_for_children_without_fields() {
         let node_types = get_node_types(&InputGrammar {

crates/generate/src/parse_grammar.rs

@@ -1,7 +1,6 @@
-use std::collections::HashSet;
-
 use log::warn;
 use regex::Regex;
+use rustc_hash::{FxHashMap, FxHashSet};
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 use thiserror::Error;
@@ -149,51 +148,150 @@ fn rule_is_referenced(rule: &Rule, target: &str, is_external: bool) -> bool {
     }
 }
 
-fn variable_is_used(
-    grammar_rules: &[(String, Rule)],
-    extras: &[Rule],
-    externals: &[Rule],
-    target_name: &str,
-    in_progress: &mut HashSet<String>,
-) -> bool {
-    let root = &grammar_rules.first().unwrap().0;
-    if target_name == root {
-        return true;
-    }
-
-    if extras
-        .iter()
-        .any(|rule| rule_is_referenced(rule, target_name, false))
-    {
-        return true;
-    }
-
-    if externals
-        .iter()
-        .any(|rule| rule_is_referenced(rule, target_name, true))
-    {
-        return true;
-    }
-
-    in_progress.insert(target_name.to_string());
-    let result = grammar_rules
-        .iter()
-        .filter(|(key, _)| *key != target_name)
-        .any(|(name, rule)| {
-            if !rule_is_referenced(rule, target_name, false) || in_progress.contains(name) {
-                return false;
+impl InputGrammar {
+    /// Strip unused rules from the grammar and clean up references to them
+    /// in the surrounding config. (conflicts, supertypes, inline, extras,
+    /// externals, precedences).
+    ///
+    /// A variable is "used" if it is the start rule, the word token, named in
+    /// `extras`/`externals`, or transitively reachable via rule references from
+    /// any of the above.
+    fn normalize(mut self) -> Self {
+        // Compute the used set via forward DFS from the implicit roots
+        // (start rule, word_token, refs in extras and externals).
+        //
+        // Extras count their top-level `NamedSymbol` as a use (so naming
+        // a rule directly in `extras` keeps it), but externals do not (the
+        // external entry is the rule itself, not a reference to one).
+        let used: FxHashSet<String> = {
+            let by_name: FxHashMap<&str, &Rule> = self
+                .variables
+                .iter()
+                .map(|v| (v.name.as_str(), &v.rule))
+                .collect();
+            let mut visited: FxHashSet<&str> = FxHashSet::default();
+            let mut stack: Vec<&str> = Vec::new();
+            if let Some(first) = self.variables.first() {
+                stack.push(first.name.as_str());
             }
-            variable_is_used(grammar_rules, extras, externals, name, in_progress)
-        });
-    in_progress.remove(target_name);
+            if let Some(word) = self.word_token.as_deref() {
+                stack.push(word);
+            }
+            for rule in &self.extra_symbols {
+                collect_referenced_names(rule, false, &mut stack);
+            }
+            for rule in &self.external_tokens {
+                collect_referenced_names(rule, true, &mut stack);
+            }
+            // Reserved-word entries are uses of the named rule (the entry
+            // names a token to reserve in some context). Top-level
+            // `NamedSymbol` counts, same as for extras.
+            for ctx in &self.reserved_words {
+                for rule in &ctx.reserved_words {
+                    collect_referenced_names(rule, false, &mut stack);
+                }
+            }
+            while let Some(name) = stack.pop() {
+                if !visited.insert(name) {
+                    continue;
+                }
+                if let Some(rule) = by_name.get(name) {
+                    collect_referenced_names(rule, false, &mut stack);
+                }
+            }
+            visited.into_iter().map(String::from).collect()
+        };
 
-    result
+        for v in &self.variables {
+            if !used.contains(v.name.as_str()) {
+                continue;
+            }
+            if !self
+                .extra_symbols
+                .iter()
+                .any(|r| rule_is_referenced(r, &v.name, false))
+            {
+                continue;
+            }
+            let inner_rule = match &v.rule {
+                Rule::Metadata { rule, .. } => rule.as_ref(),
+                other => other,
+            };
+            let matches_empty = match inner_rule {
+                Rule::String(s) => s.is_empty(),
+                Rule::Pattern(value, _) => Regex::new(value).is_ok_and(|reg| reg.is_match("")),
+                _ => false,
+            };
+            if matches_empty {
+                warn!(
+                    "Named extra rule `{name}` matches the empty string. \
+                     Inline this to avoid infinite loops while parsing.",
+                    name = v.name,
+                );
+            }
+        }
+
+        // Drop unused variables and clean up any references to them in the
+        // surrounding grammar config.
+        let dropped: Vec<String> = self
+            .variables
+            .iter()
+            .filter(|v| !used.contains(v.name.as_str()))
+            .map(|v| v.name.clone())
+            .collect();
+        self.variables.retain(|v| used.contains(v.name.as_str()));
+        for name in &dropped {
+            self.expected_conflicts.retain(|r| !r.contains(name));
+            self.supertype_symbols.retain(|r| r != name);
+            self.variables_to_inline.retain(|r| r != name);
+            self.extra_symbols
+                .retain(|r| !rule_is_referenced(r, name, true));
+            self.external_tokens
+                .retain(|r| !rule_is_referenced(r, name, true));
+            self.precedence_orderings.retain(|r| {
+                !r.iter()
+                    .any(|e| matches!(e, PrecedenceEntry::Symbol(s) if s == name))
+            });
+            // Prune entries but keep the context: an intentionally-empty
+            // reserved-word context is a meaningful marker that rule bodies
+            // may reference by name.
+            for ctx in &mut self.reserved_words {
+                ctx.reserved_words
+                    .retain(|r| !rule_is_referenced(r, name, false));
+            }
+        }
+
+        self
+    }
+}
+
+/// Append every `NamedSymbol` name reachable in `rule` to `out`. If
+/// `skip_top_level` is true, a `NamedSymbol` at the root of `rule` is
+/// ignored (used for externals entries, which name themselves).
+fn collect_referenced_names<'a>(rule: &'a Rule, skip_top_level: bool, out: &mut Vec<&'a str>) {
+    match rule {
+        Rule::NamedSymbol(name) => {
+            if !skip_top_level {
+                out.push(name.as_str());
+            }
+        }
+        Rule::Choice(rules) | Rule::Seq(rules) => {
+            for r in rules {
+                collect_referenced_names(r, false, out);
+            }
+        }
+        Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
+            collect_referenced_names(rule, skip_top_level, out);
+        }
+        Rule::Repeat(inner) => collect_referenced_names(inner, false, out),
+        Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => {}
+    }
 }
 
 pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
-    let mut grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
+    let grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
 
-    let mut extra_symbols =
+    let extra_symbols =
         grammar_json
             .extras
             .into_iter()
@@ -208,7 +306,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
                 ParseGrammarResult::Ok(acc)
             })?;
 
-    let mut external_tokens = grammar_json
+    let external_tokens = grammar_json
         .externals
         .into_iter()
         .map(|e| parse_rule(e, false))
@@ -227,74 +325,17 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         precedence_orderings.push(ordering);
     }
 
-    let mut variables = Vec::with_capacity(grammar_json.rules.len());
-
-    let rules = grammar_json
+    let variables = grammar_json
         .rules
         .into_iter()
-        .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?, false)?)))
-        .collect::<ParseGrammarResult<Vec<_>>>()?;
-
-    let mut in_progress = HashSet::new();
-
-    for (name, rule) in &rules {
-        if grammar_json.word.as_ref().is_none_or(|w| w != name)
-            && !variable_is_used(
-                &rules,
-                &extra_symbols,
-                &external_tokens,
+        .map(|(name, r)| {
+            Ok(Variable {
                 name,
-                &mut in_progress,
-            )
-        {
-            grammar_json.conflicts.retain(|r| !r.contains(name));
-            grammar_json.supertypes.retain(|r| r != name);
-            grammar_json.inline.retain(|r| r != name);
-            extra_symbols.retain(|r| !rule_is_referenced(r, name, true));
-            external_tokens.retain(|r| !rule_is_referenced(r, name, true));
-            precedence_orderings.retain(|r| {
-                !r.iter().any(|e| {
-                    let PrecedenceEntry::Symbol(s) = e else {
-                        return false;
-                    };
-                    s == name
-                })
-            });
-            continue;
-        }
-
-        if extra_symbols
-            .iter()
-            .any(|r| rule_is_referenced(r, name, false))
-        {
-            let inner_rule = if let Rule::Metadata { rule, .. } = rule {
-                rule
-            } else {
-                rule
-            };
-            let matches_empty = match inner_rule {
-                Rule::String(rule_str) => rule_str.is_empty(),
-                Rule::Pattern(ref value, _) => Regex::new(value)
-                    .map(|reg| reg.is_match(""))
-                    .unwrap_or(false),
-                _ => false,
-            };
-            if matches_empty {
-                warn!(
-                    concat!(
-                        "Named extra rule `{}` matches the empty string. ",
-                        "Inline this to avoid infinite loops while parsing."
-                    ),
-                    name
-                );
-            }
-        }
-        variables.push(Variable {
-            name: name.clone(),
-            kind: VariableType::Named,
-            rule: rule.clone(),
-        });
-    }
+                kind: VariableType::Named,
+                rule: parse_rule(serde_json::from_value(r)?, false)?,
+            })
+        })
+        .collect::<ParseGrammarResult<Vec<_>>>()?;
 
     let reserved_words = grammar_json
         .reserved
@@ -315,7 +356,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         })
         .collect::<ParseGrammarResult<Vec<_>>>()?;
 
-    Ok(InputGrammar {
+    let grammar = InputGrammar {
         name: grammar_json.name,
         word_token: grammar_json.word,
         expected_conflicts: grammar_json.conflicts,
@@ -326,7 +367,9 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
         extra_symbols,
         external_tokens,
         reserved_words,
-    })
+    }
+    .normalize();
+    Ok(grammar)
 }
 
 fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult<Rule> {
@@ -400,7 +443,7 @@ fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult<Rule> {
         }),
         RuleJSON::TOKEN { content } => parse_rule(*content, true).map(Rule::token),
         RuleJSON::IMMEDIATE_TOKEN { content } => {
-            parse_rule(*content, is_token).map(Rule::immediate_token)
+            parse_rule(*content, true).map(Rule::immediate_token)
         }
     }
 }

crates/generate/src/prepare_grammar.rs

@@ -4,6 +4,7 @@ mod extract_default_aliases;
 mod extract_tokens;
 mod flatten_grammar;
 mod intern_symbols;
+mod pattern;
 mod process_inlines;
 
 use std::{

crates/generate/src/prepare_grammar/expand_tokens.rs

@@ -1,7 +1,4 @@
-use regex_syntax::{
-    hir::{Class, Hir, HirKind},
-    ParserBuilder,
-};
+use regex_syntax::hir::{Class, Hir, HirKind};
 use serde::Serialize;
 use thiserror::Error;
 
@@ -9,6 +6,7 @@ use super::ExtractedLexicalGrammar;
 use crate::{
     grammars::{LexicalGrammar, LexicalVariable},
     nfa::{CharacterSet, Nfa, NfaState},
+    prepare_grammar::pattern,
     rules::{Precedence, Rule},
 };
 
@@ -175,13 +173,10 @@ impl NfaBuilder {
                     .replace(r"\W", r"[^0-9A-Za-z_]")
                     .replace(r"\S", r"[^\t-\r ]")
                     .replace(r"\D", r"[^0-9]");
-                let mut parser = ParserBuilder::new()
-                    .case_insensitive(f.contains('i'))
-                    .unicode(true)
-                    .utf8(false)
-                    .build();
-                let hir = parser
-                    .parse(&s)
+                // Parse WITHOUT case folding and fold ourselves (see
+                // `case_fold_ascii_safe`). Letting `regex_syntax` fold would pull the
+                // long s `ſ` and Kelvin sign `K` into ASCII `s`/`k`.
+                let hir = pattern::parse(&s, f.contains('i'))
                     .map_err(|e| ExpandRuleError::Parse(e.to_string()))?;
                 self.expand_regex(&hir, next_state_id)
                     .map_err(ExpandRuleError::ExpandRegex)
@@ -261,8 +256,7 @@ impl NfaBuilder {
                     .chars()
                     .rev()
                 {
-                    let char_set = CharacterSet::from_char(character);
-                    self.push_advance(char_set, next_state_id);
+                    self.push_advance(CharacterSet::from_char(character), next_state_id);
                     next_state_id = self.nfa.last_state_id();
                 }
 
@@ -274,21 +268,13 @@ impl NfaBuilder {
                     for c in class.ranges() {
                         chars = chars.add_range(c.start(), c.end());
                     }
-
-                    // For some reason, the long s `ſ` is included if the letter `s` is in a
-                    // pattern, so we remove it.
-                    if chars.range_count() == 3
-                        && chars
-                            .ranges()
-                            // exact check to ensure that `ſ` wasn't intentionally added.
-                            .all(|r| ['s'..='s', 'S'..='S', 'ſ'..='ſ'].contains(&r))
-                    {
-                        chars = chars.difference(CharacterSet::from_char('ſ'));
-                    }
                     self.push_advance(chars, next_state_id);
                     Ok(true)
                 }
                 Class::Bytes(bytes_class) => {
+                    // Byte classes only come from non-Unicode `(?-u:...)` groups, which
+                    // JS regex syntax can't express, so this is currently unreachable
+                    // from grammar patterns.
                     let mut chars = CharacterSet::default();
                     for c in bytes_class.ranges() {
                         chars = chars.add_range(c.start().into(), c.end().into());
@@ -727,6 +713,146 @@ mod tests {
                     ("\u{1000b}", Some((3, "\u{1000b}"))),
                 ],
             },
+            // Case-insensitive patterns must not fold in the two non-ASCII code
+            // points that Unicode simple case folding maps onto ASCII letters:
+            // `ſ` (U+017F) onto `s`, and the Kelvin sign `K` (U+212A) onto `k`.
+            Row {
+                rules: vec![Rule::pattern("[sk]+", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("sSkK.", Some((0, "sSkK"))),
+                    ("\u{017f}", None),              // long s, not matched by `s`
+                    ("\u{212a}", None),              // Kelvin sign, not matched by `k`
+                    ("sk\u{212a}", Some((0, "sk"))), // folded code point ends the token
+                ],
+            },
+            // A broad class carrying `/i` (a negated class, `\p{L}`, ...) keeps
+            // `ſ`/`K`: folding never *introduces* them here (the class already
+            // contains them), so there is nothing to drop.
+            Row {
+                rules: vec![Rule::pattern("[^\"]", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("\u{017f}", Some((0, "\u{017f}"))), // long s kept under /i
+                    ("\u{212a}", Some((0, "\u{212a}"))), // Kelvin sign kept under /i
+                    ("s", Some((0, "s"))),
+                ],
+            },
+            // An intentionally-written `ſ`/`K` is preserved: the stripping above
+            // only fires when the ASCII pair it folds with is also present.
+            Row {
+                rules: vec![Rule::pattern("[\u{017f}\u{212a}]+", "")],
+                separators: vec![],
+                examples: vec![("\u{017f}\u{212a}.", Some((0, "\u{017f}\u{212a}")))],
+            },
+            // Without the `i` flag nothing is folded, so a broad class such as
+            // `[^"]` must keep `ſ`/`K` instead of stripping them as fold artifacts.
+            Row {
+                rules: vec![Rule::pattern("[^\"]", "")],
+                separators: vec![],
+                examples: vec![
+                    ("a", Some((0, "a"))),
+                    ("\u{017f}", Some((0, "\u{017f}"))), // long s
+                    ("\u{212a}", Some((0, "\u{212a}"))), // Kelvin sign
+                    ("\"", None),                        // the one excluded character
+                ],
+            },
+            // `ſ`/`K` fold with nothing, so an explicit one pulls in no ASCII letter.
+            Row {
+                rules: vec![Rule::pattern("[\u{017f}\u{212a}]+", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("\u{017f}\u{212a}.", Some((0, "\u{017f}\u{212a}"))),
+                    ("s", None),
+                    ("k", None),
+                ],
+            },
+            // `\p{L}` already contains `ſ`/`K`, so folding adds nothing to strip.
+            Row {
+                rules: vec![Rule::pattern(r"\p{L}+", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("\u{017f}", Some((0, "\u{017f}"))),
+                    ("\u{212a}", Some((0, "\u{212a}"))),
+                    ("aA", Some((0, "aA"))),
+                    ("1", None),
+                ],
+            },
+            // Folding happens at the leaves, before negation: `[^a-z]` under `/i`
+            // must exclude `A-Z`, not re-admit it.
+            Row {
+                rules: vec![Rule::pattern("[^a-z]", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("!", Some((0, "!"))),
+                    ("a", None),
+                    ("A", None),
+                    ("\u{017f}", Some((0, "\u{017f}"))),
+                    ("\u{212a}", Some((0, "\u{212a}"))),
+                ],
+            },
+            // The same ordering through a nested class and through class-set algebra.
+            Row {
+                rules: vec![Rule::pattern("[^[a-c]]", "i")],
+                separators: vec![],
+                examples: vec![("d", Some((0, "d"))), ("a", None), ("A", None), ("C", None)],
+            },
+            Row {
+                rules: vec![Rule::pattern("[[a-z]--[b-d]]", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("a", Some((0, "a"))),
+                    ("A", Some((0, "A"))),
+                    ("b", None),
+                    ("B", None),
+                ],
+            },
+            Row {
+                rules: vec![Rule::pattern("[[a-z]&&[b-d]]", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("b", Some((0, "b"))),
+                    ("B", Some((0, "B"))),
+                    ("e", None),
+                    ("E", None),
+                ],
+            },
+            // Scoped flags apply only where they are in effect.
+            Row {
+                rules: vec![Rule::pattern("(?i)a(?-i)b", "")],
+                separators: vec![],
+                examples: vec![
+                    ("ab", Some((0, "ab"))),
+                    ("Ab", Some((0, "Ab"))),
+                    ("aB", None),
+                ],
+            },
+            // Case-insensitivity does not leak from one token to the next.
+            Row {
+                rules: vec![
+                    Rule::pattern("ab", "i"),
+                    Rule::pattern("cd", ""),
+                    Rule::string("ef"),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("AB", Some((0, "AB"))),
+                    ("cd", Some((1, "cd"))),
+                    ("CD", None),
+                    ("ef", Some((2, "ef"))),
+                    ("EF", None),
+                ],
+            },
+            Row {
+                rules: vec![Rule::pattern(r"\$\{[a-z0-9_\.]*[^a-z0-9_\.\}]", "i")],
+                separators: vec![],
+                examples: vec![
+                    ("${a}", None),
+                    ("${A}", None),
+                    ("${a!", Some((0, "${a!"))),
+                    ("${!", Some((0, "${!"))),
+                ],
+            },
             // Emojis
             Row {
                 rules: vec![Rule::pattern(r"\p{Emoji}+", "")],

crates/generate/src/prepare_grammar/extract_tokens.rs

@@ -21,6 +21,8 @@ unless they are used only as the grammar's start rule.
 "
     )]
     EmptyString(String),
+    #[error("Terminal rule '{0}' cannot be used as a supertype")]
+    SupertypeTerminal(String),
     #[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
     ExternalTokenNonTerminal(String),
     #[error("Non-symbol rules cannot be used as external tokens")]
@@ -128,11 +130,18 @@ pub(super) fn extract_tokens(
         })
         .collect();
 
-    let supertype_symbols = grammar
+    let supertype_symbols: Vec<Symbol> = grammar
         .supertype_symbols
         .into_iter()
         .map(|symbol| symbol_replacer.replace_symbol(symbol))
         .collect();
+    for supertype_symbol in &supertype_symbols {
+        if supertype_symbol.is_terminal() {
+            Err(ExtractTokensError::SupertypeTerminal(
+                lexical_variables[supertype_symbol.index].name.clone(),
+            ))?;
+        }
+    }
 
     let variables_to_inline = grammar
         .variables_to_inline

crates/generate/src/prepare_grammar/pattern.rs

@@ -0,0 +1,336 @@
+//! Parsing of token patterns.
+//!
+//! Unicode simple case folding maps two non-ASCII code points onto ASCII letters:
+//!     - the long s `ſ` onto `s`
+//!     - the Kelvin sign `K` onto `k`
+//!
+//! Taken as is, every case-insensitive ASCII token picks both up, which is virtually
+//! never intended and stops the token from being extracted as a keyword. To
+//! fix this, we expand the `i` flag ourselves.
+//!
+//! The [`regex_syntax`] crate parses a pattern into an [`ast::Ast`], then lowers
+//! it to an [`hir::Hir`] via a [`hir::translate::Translator`]. The translator is
+//! what applies `i`, and it folds each leaf of a class expression before applying
+//! that leaf's negation and the set algebra above it. This order is what makes
+//! `(?i)[^x]` exclude `X` rather than re-admit it, so we keep that order and only
+//! change the fold:
+//!     - Walk the AST
+//!     - Fold each leaf as if `ſ` and `K` were their own equivalence classes
+//!     - Hand the result to the translator with its own folding turned off
+//!
+//! Folding at the leaves is sufficient for correctness, as case folding sorts every
+//! character into a group of case variants (`{a, A}`, `{s, S, ſ}`, etc.), with
+//! each character in exactly one group. Folding a set adds (for each character
+//! in it) the rest of that character's group, so the result is always built out
+//! of whole groups. Negation and set algebra preserve that, because they treat
+//! every character in a group alike: a group ends up either wholly inside or wholly
+//! outside the result. Folding a set that is already whole groups adds nothing,
+//! so once leaves are folded, every fold the translator would still apply does
+//! nothing and it compiles exactly what is would have with our fold in place of
+//! its own.
+
+use regex_syntax::{
+    ast::{
+        self, parse::ParserBuilder, Ast, ClassBracketed, ClassSet, ClassSetItem, ClassSetRange,
+        ClassSetUnion, Flag, FlagsItem, FlagsItemKind, GroupKind, Span,
+    },
+    hir::{
+        self,
+        translate::{Translator, TranslatorBuilder},
+        Class, ClassUnicode, ClassUnicodeRange, Hir, HirKind,
+    },
+};
+
+/// The flags that decide whether we fold a scope's leaves.
+#[derive(Clone, Copy)]
+struct FoldMode {
+    case_insensitive: bool,
+    unicode: bool,
+}
+
+impl FoldMode {
+    /// Whether we fold this scope's leaves ourselves.
+    ///
+    /// Inside `(?-u:...)` the translator works on bytes and folds ASCII only, which
+    /// can never pull in `ſ` or `K`, so those scopes keep its folding.
+    const fn folds_here(self) -> bool {
+        self.case_insensitive && self.unicode
+    }
+}
+
+/// The state of the walk: The flags in effect, and one translator to resolve
+/// `\p{...}` and friends without rebuilding it per leaf.
+struct Expander<'a> {
+    translator: Translator,
+    pattern: &'a str,
+    mode: FoldMode,
+}
+
+/// Parse a token pattern into an [`Hir`], folding any `i` flag manually.
+pub(super) fn parse(
+    pattern: &str,
+    case_insensitive: bool,
+) -> Result<Hir, Box<regex_syntax::Error>> {
+    let mut ast = ParserBuilder::new()
+        .build()
+        .parse(pattern)
+        .map_err(|e| Box::new(e.into()))?;
+
+    let mut expander = Expander {
+        translator: TranslatorBuilder::new()
+            .case_insensitive(false)
+            .unicode(true)
+            .utf8(false)
+            .build(),
+        pattern,
+        mode: FoldMode {
+            case_insensitive,
+            unicode: true,
+        },
+    };
+    expander.expand(&mut ast).map_err(|e| Box::new(e.into()))?;
+    expander
+        .translator
+        .translate(pattern, &ast)
+        .map_err(|e| Box::new(e.into()))
+}
+
+impl Expander<'_> {
+    fn expand(&mut self, ast: &mut Ast) -> Result<(), hir::Error> {
+        match ast {
+            Ast::Flags(f) => self.set_flags(&mut f.flags),
+            Ast::Repetition(r) => self.expand(&mut r.ast)?,
+            Ast::Group(g) => {
+                let outer = self.mode;
+                if let GroupKind::NonCapturing(flags) = &mut g.kind {
+                    self.set_flags(flags);
+                }
+                self.expand(&mut g.ast)?;
+                self.mode = outer;
+            }
+            // Flags set in one branch carry into the next, as they do in the translator,
+            // which scopes flags to groups but not to these.
+            Ast::Alternation(a) => {
+                for branch in &mut a.asts {
+                    self.expand(branch)?;
+                }
+            }
+            Ast::Concat(c) => {
+                for element in &mut c.asts {
+                    self.expand(element)?;
+                }
+            }
+            Ast::ClassBracketed(b) if self.mode.folds_here() => {
+                self.expand_class_set(&mut b.kind)?;
+            }
+            // The remaining class-valued nodes are the same leaves as a bracket holds,
+            // so they fold through the same path.
+            _ if self.mode.folds_here() => {
+                let mut item = match ast {
+                    Ast::Literal(l) => ClassSetItem::Literal((**l).clone()),
+                    Ast::ClassUnicode(u) => ClassSetItem::Unicode((**u).clone()),
+                    Ast::ClassPerl(p) => ClassSetItem::Perl((**p).clone()),
+                    // `.` is fold-closed, nothing else matches a character
+                    _ => return Ok(()),
+                };
+                self.expand_class_item(&mut item)?;
+                if let ClassSetItem::Bracketed(class) = item {
+                    *ast = Ast::ClassBracketed(class);
+                }
+            }
+            // Left to the translator: either no `i` is in effect, or this is a
+            // `(?-u:...)` scope, where its own folding is ASCII-only and cannot
+            // pull in `ſ` or `K`.
+            #[rustfmt::skip]
+            Ast::Empty(_) | Ast::Dot(_) | Ast::Assertion(_) | Ast::Literal(_)
+            | Ast::ClassUnicode(_) | Ast::ClassPerl(_) | Ast::ClassBracketed(_) => {}
+        }
+        Ok(())
+    }
+
+    /// Apply a flag directive to the mode, then rewrite the directive to say what
+    /// the translator should still do about `i` in the scope it opens.
+    ///
+    /// `i` comes out everywhere, since folding a leaf we already folded would put
+    /// `ſ`/`K` back in, and goes back in only when we skipped the folding outselves
+    /// (inside `(?-u:...)`.
+    ///
+    /// `i` comes out everywhere, since folding a leaf we already folded would put
+    /// `ſ`/`K` straight back. It goes back in wherever [`FoldMode::folds_here`]
+    /// says we left the folding alone, which would otherwise lose case-insensitivity
+    /// in that scope entirely.
+    fn set_flags(&mut self, flags: &mut ast::Flags) {
+        let mut negated = false;
+        for item in &flags.items {
+            match item.kind {
+                FlagsItemKind::Negation => negated = true,
+                FlagsItemKind::Flag(Flag::CaseInsensitive) => self.mode.case_insensitive = !negated,
+                FlagsItemKind::Flag(Flag::Unicode) => self.mode.unicode = !negated,
+                FlagsItemKind::Flag(_) => {}
+            }
+        }
+        let item = |kind| FlagsItem {
+            span: flags.span,
+            kind,
+        };
+        flags
+            .items
+            .retain(|it| !matches!(it.kind, FlagsItemKind::Flag(Flag::CaseInsensitive)));
+        if self.mode.folds_here() {
+            flags.items.push(item(FlagsItemKind::Negation));
+            flags
+                .items
+                .push(item(FlagsItemKind::Flag(Flag::CaseInsensitive)));
+        } else if self.mode.case_insensitive {
+            flags
+                .items
+                .insert(0, item(FlagsItemKind::Flag(Flag::CaseInsensitive)));
+        }
+    }
+
+    /// Fold the leaves of a class expression, leaving its structure alone. The
+    /// translator still performs the negations and the `&&`/`--`/`~~` itself, on
+    /// operands that are already folded.
+    fn expand_class_set(&mut self, set: &mut ClassSet) -> Result<(), hir::Error> {
+        match set {
+            ClassSet::Item(item) => self.expand_class_item(item),
+            ClassSet::BinaryOp(op) => {
+                self.expand_class_set(&mut op.lhs)?;
+                self.expand_class_set(&mut op.rhs)
+            }
+        }
+    }
+
+    fn expand_class_item(&mut self, item: &mut ClassSetItem) -> Result<(), hir::Error> {
+        // Whether the leaf carries its own negation (`\P{L}, `[:^alpha:]`, `\D`, etc.).
+        let (span, negated) = match item {
+            ClassSetItem::Bracketed(b) => return self.expand_class_set(&mut b.kind),
+            ClassSetItem::Union(u) => {
+                return u
+                    .items
+                    .iter_mut()
+                    .try_for_each(|i| self.expand_class_item(i));
+            }
+            ClassSetItem::Empty(_) => return Ok(()),
+            ClassSetItem::Literal(l) => (l.span, false),
+            ClassSetItem::Range(r) => (r.span, false),
+            ClassSetItem::Ascii(a) => (a.span, a.negated),
+            ClassSetItem::Perl(p) => (p.span, p.negated),
+            ClassSetItem::Unicode(u) => (u.span, u.is_negated()),
+        };
+
+        // `leaf_set` applies the leaf's own negation, so undo it to recover the
+        // operand the translator would have folded, and hand the negation back
+        // to the replacement so it is reapplied after the fold.
+        let mut operand = self.leaf_set(item)?;
+        if negated {
+            operand.negate();
+        }
+
+        let mut folded = operand.clone();
+        Self::fold_ascii_safe(&mut folded);
+        // A leaf that is already fold-closed stays as written, which keeps most
+        // unicode properties out of the expansion.
+        if folded.ranges() != operand.ranges() {
+            *item = ClassSetItem::Bracketed(Box::new(ClassBracketed {
+                span,
+                negated,
+                kind: ClassSet::Item(Self::class_to_item(span, &folded)),
+            }));
+        }
+
+        Ok(())
+    }
+
+    /// The set a leaf denotes, with any negation of its own already applied.
+    ///
+    /// Literals and ranges are read directly from the AST. Everything else goes
+    /// back through the translator, which own the property and POSIX tables.
+    fn leaf_set(&mut self, item: &ClassSetItem) -> Result<ClassUnicode, hir::Error> {
+        let range = match item {
+            ClassSetItem::Literal(l) => ClassUnicodeRange::new(l.c, l.c),
+            ClassSetItem::Range(r) => ClassUnicodeRange::new(r.start.c, r.end.c),
+            _ => {
+                let ast = Ast::ClassBracketed(Box::new(ClassBracketed {
+                    span: *item.span(),
+                    negated: false,
+                    kind: ClassSet::Item(item.clone()),
+                }));
+                return Ok(Self::class_of(
+                    self.translator.translate(self.pattern, &ast)?,
+                ));
+            }
+        };
+        Ok(ClassUnicode::new([range]))
+    }
+
+    /// The class a single-class [`Hir`] denotes.
+    ///
+    /// We only ever translate one [`Ast::ClassBracketed`], so [`Hir::class`] built
+    /// the result: A class, or one of the two shapes it collapses to. The structural
+    /// kinds (`Empty`, `Look`, `Repetition`, `Capture`, `Concat`, `Alternation`)
+    /// need AST nodes we never construct in [`Self::leaf_set`].
+    fn class_of(hir: Hir) -> ClassUnicode {
+        match hir.into_kind() {
+            HirKind::Class(Class::Unicode(class)) => class,
+            HirKind::Literal(literal) => {
+                // A one character class, translated in unicode mode, so this must
+                // be that character's UTF-8.
+                let literal = std::str::from_utf8(&literal.0).unwrap();
+                ClassUnicode::new(literal.chars().map(|c| ClassUnicodeRange::new(c, c)))
+            }
+            // An empty class collapses to `Hir::fail` (an empty byte class).
+            _ => ClassUnicode::empty(),
+        }
+    }
+
+    /// Re-encode a class of the AST nodes the tranlator will read back, the inverse
+    /// of [`Self::leaf_set`].
+    ///
+    /// Every node reuses the leaf's own span to preserve error spans. The literal
+    /// kind is irrelevant, in unicode mode the translator only reads `c`.
+    fn class_to_item(span: Span, class: &ClassUnicode) -> ClassSetItem {
+        let literal = |c| ast::Literal {
+            span,
+            kind: ast::LiteralKind::Verbatim,
+            c,
+        };
+        ClassSetUnion {
+            span,
+            items: class
+                .ranges()
+                .iter()
+                .map(|r| {
+                    ClassSetItem::Range(ClassSetRange {
+                        span,
+                        start: literal(r.start()),
+                        end: literal(r.end()),
+                    })
+                })
+                .collect(),
+        }
+        .into_item()
+    }
+
+    /// Fold `class` in place, with `ſ` and `K` each in a group of their own rather
+    /// than the the `s`/`S` and `k`/`K` groups.
+    ///
+    /// Dropping them before the fold stops a `ſ` in the class from pulling in `s`/`S`.
+    /// Dropping them after removes the ones folding `s`/`S` introduced. The union
+    /// restores any the class genuinely held.
+    fn fold_ascii_safe(class: &mut ClassUnicode) {
+        // The code points that Unicode simple case folding maps onto ASCII letters:
+        // the long s `ſ` folds with `s`/`S`, and the Kelvin sign `K` with `k`/`K`.
+        const NON_ASCII_FOLDS: [char; 2] = ['\u{17f}', '\u{212a}'];
+        let exotic = ClassUnicode::new(NON_ASCII_FOLDS.map(|c| ClassUnicodeRange::new(c, c)));
+
+        // The ones the pattern asked for itself, which folding  must not drop
+        let mut asked_for = exotic.clone();
+        asked_for.intersect(class);
+
+        class.difference(&exotic);
+        class.case_fold_simple();
+        class.difference(&exotic);
+        class.union(&asked_for);
+    }
+}

crates/generate/src/prepare_grammar/process_inlines.rs

@@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
         let production_map = production_indices_by_step_id
             .into_iter()
             .map(|(step_id, production_indices)| {
-                let production = step_id.variable_index.map_or_else(
-                    || &productions[step_id.production_index],
-                    |variable_index| {
-                        &grammar.variables[variable_index].productions[step_id.production_index]
-                    },
-                ) as *const Production;
+                let production =
+                    core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
+                        || &productions[step_id.production_index],
+                        |variable_index| {
+                            &grammar.variables[variable_index].productions[step_id.production_index]
+                        },
+                    ));
                 ((production, step_id.step_index as u32), production_indices)
             })
             .collect();

crates/generate/src/render.rs

@@ -7,6 +7,8 @@ use std::{
 
 use crate::LANGUAGE_VERSION;
 use indoc::indoc;
+use serde::Serialize;
+use thiserror::Error;
 
 use super::{
     build_tables::Tables,
@@ -25,6 +27,16 @@ pub const ABI_VERSION_MIN: usize = 14;
 pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION;
 const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
 
+pub type RenderResult<T> = Result<T, RenderError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum RenderError {
+    #[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
+    ParseTable(usize),
+    #[error("This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {0}")]
+    ABI(usize),
+}
+
 #[clippy::format_args]
 macro_rules! add {
     ($this: tt, $($arg: tt)*) => {{
@@ -104,7 +116,7 @@ struct Metadata {
 }
 
 impl Generator {
-    fn generate(mut self) -> String {
+    fn generate(mut self) -> RenderResult<String> {
         self.init();
         self.add_header();
         self.add_includes();
@@ -161,7 +173,7 @@ impl Generator {
             self.add_reserved_word_sets();
         }
 
-        self.add_parse_table();
+        self.add_parse_table()?;
 
         if !self.syntax_grammar.external_tokens.is_empty() {
             self.add_external_token_enum();
@@ -171,7 +183,7 @@ impl Generator {
 
         self.add_parser_export();
 
-        self.buffer
+        Ok(self.buffer)
     }
 
     fn init(&mut self) {
@@ -325,7 +337,7 @@ impl Generator {
     }
 
     fn add_header(&mut self) {
-        add_line!(self, "/* Automatically @generated by tree-sitter */",);
+        add_line!(self, "/* Automatically @generated by tree-sitter */");
         add_line!(self, "");
     }
 
@@ -1273,7 +1285,7 @@ impl Generator {
         add_line!(self, "");
     }
 
-    fn add_parse_table(&mut self) {
+    fn add_parse_table(&mut self) -> RenderResult<()> {
         let mut parse_table_entries = HashMap::new();
         let mut next_parse_action_list_index = 0;
 
@@ -1443,6 +1455,9 @@ impl Generator {
             add_line!(self, "}};");
             add_line!(self, "");
         }
+        if next_parse_action_list_index >= usize::from(u16::MAX) {
+            Err(RenderError::ParseTable(next_parse_action_list_index))?;
+        }
 
         let mut parse_table_entries = parse_table_entries
             .into_iter()
@@ -1450,6 +1465,8 @@ impl Generator {
             .collect::<Vec<_>>();
         parse_table_entries.sort_by_key(|(index, _)| *index);
         self.add_parse_action_list(parse_table_entries);
+
+        Ok(())
     }
 
     fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
@@ -1942,11 +1959,10 @@ pub fn render_c_code(
     abi_version: usize,
     semantic_version: Option<(u8, u8, u8)>,
     supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
-) -> String {
-    assert!(
-        (ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version),
-        "This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {abi_version}",
-    );
+) -> RenderResult<String> {
+    if !(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version) {
+        Err(RenderError::ABI(abi_version))?;
+    }
 
     Generator {
         language_name: name.to_string(),

crates/generate/src/templates/array.h

@@ -50,69 +50,104 @@ extern "C" {
 /// memory allocated for the array's contents.
 #define array_clear(self) ((self)->size = 0)
 
+#ifdef __cplusplus
+#define _array__cast(self, expr) (decltype((self)->contents))(expr)
+#else
+#define _array__cast(self, expr) (expr)
+#endif
+
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity) \
-  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity)                 \
+  ((self)->contents = _array__cast(self, _array__reserve( \
+    (void *)(self)->contents, &(self)->capacity,          \
+    array_elem_size(self), new_capacity))                 \
+  )
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((Array *)(self))
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                            \
-  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
-   (self)->contents[(self)->size++] = (element))
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__cast(self, _array__grow(           \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    ));                                                           \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count) \
-  do { \
-    if ((count) == 0) break; \
-    _array__grow((Array *)(self), count, array_elem_size(self)); \
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__cast(self, _array__grow(                          \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    ));                                                                          \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count); \
+    (self)->size += (count);                                                     \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other)                                       \
+#define array_push_all(self, other) \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, contents)                    \
-  _array__splice(                                               \
-    (Array *)(self), array_elem_size(self), (self)->size, \
-    0, count,  contents                                        \
-  )
+#define array_extend(self, count, other_contents)                 \
+  ((self)->contents = _array__cast(self, _array__splice(          \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
+  )))
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents)  \
-  _array__splice(                                                       \
-    (Array *)(self), array_elem_size(self), _index,                \
-    old_count, new_count, new_contents                                 \
-  )
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  ((self)->contents = _array__cast(self, _array__splice(              \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
+  )))
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element) \
-  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+#define array_insert(self, _index, element)                     \
+  ((self)->contents = _array__cast(self, _array__splice(        \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )))
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((Array *)(self), array_elem_size(self), _index)
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other) \
-  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+#define array_assign(self, other)                                   \
+  ((self)->contents = _array__cast(self, _array__assign(            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )))
 
 /// Swap one array with another
-#define array_swap(self, other) \
-  _array__swap((Array *)(self), (Array *)(other))
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array__cast(other, _array_swap_tmp);       \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
+  } while (0)
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +192,90 @@ extern "C" {
 
 // Private
 
-typedef Array(void) Array;
-
-/// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(Array *self) {
-  if (self->contents) {
-    ts_free(self->contents);
-    self->contents = NULL;
-    self->size = 0;
-    self->capacity = 0;
-  }
-}
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(Array *self, size_t element_size,
-                                uint32_t index) {
-  assert(index < self->size);
-  char *contents = (char *)self->contents;
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  assert(index < *size);
+  char *contents = (char *)self_contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (self->size - index - 1) * element_size);
-  self->size--;
+          (*size - index - 1) * element_size);
+  (*size)--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
-  if (new_capacity > self->capacity) {
-    if (self->contents) {
-      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
     } else {
-      self->contents = ts_malloc(new_capacity * element_size);
+      new_contents = ts_malloc(new_capacity * element_size);
     }
-    self->capacity = new_capacity;
+    *capacity = new_capacity;
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
-  _array__reserve(self, element_size, other->size);
-  self->size = other->size;
-  memcpy(self->contents, other->contents, self->size * element_size);
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(Array *self, Array *other) {
-  Array swap = *other;
-  *other = *self;
-  *self = swap;
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
-  uint32_t new_size = self->size + count;
-  if (new_size > self->capacity) {
-    uint32_t new_capacity = self->capacity * 2;
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    _array__reserve(self, element_size, new_capacity);
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void _array__splice(Array *self, size_t element_size,
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t new_size = *size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  assert(old_end <= self->size);
+  assert(old_end <= *size);
 
-  _array__reserve(self, element_size, new_size);
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
 
-  char *contents = (char *)self->contents;
-  if (self->size > old_end) {
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (self->size - old_end) * element_size
+      (*size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -250,7 +293,9 @@ static inline void _array__splice(Array *self, size_t element_size,
       );
     }
   }
-  self->size += new_count - old_count;
+  *size += new_count - old_count;
+
+  return new_contents;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

crates/highlight/src/highlight.rs

@@ -1274,28 +1274,22 @@ fn injection_for_match<'a>(
             // In addition to specifying the language name via the text of a
             // captured node, it can also be hard-coded via a `#set!` predicate
             // that sets the injection.language key.
-            "injection.language" => {
-                if language_name.is_none() {
-                    language_name = prop.value.as_ref().map(std::convert::AsRef::as_ref);
-                }
+            "injection.language" if language_name.is_none() => {
+                language_name = prop.value.as_ref().map(std::convert::AsRef::as_ref);
             }
 
             // Setting the `injection.self` key can be used to specify that the
             // language name should be the same as the language of the current
             // layer.
-            "injection.self" => {
-                if language_name.is_none() {
-                    language_name = Some(config.language_name.as_str());
-                }
+            "injection.self" if language_name.is_none() => {
+                language_name = Some(config.language_name.as_str());
             }
 
             // Setting the `injection.parent` key can be used to specify that
             // the language name should be the same as the language of the
             // parent layer
-            "injection.parent" => {
-                if language_name.is_none() {
-                    language_name = parent_name;
-                }
+            "injection.parent" if language_name.is_none() => {
+                language_name = parent_name;
             }
 
             // By default, injections do not include the *children* of an

crates/language/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-language"
 description = "The tree-sitter Language type, used by the library and by language implementations"
-version = "0.1.6"
+version = "0.1.7"
 authors.workspace = true
 edition.workspace = true
 rust-version = "1.77"

crates/language/wasm/include/stdint.h

@@ -23,9 +23,15 @@ typedef long unsigned int size_t;
 
 typedef long unsigned int uintptr_t;
 
-#define UINT16_MAX 65535
+#define INT8_MAX 127
+#define INT16_MAX 32767
+#define INT32_MAX 2147483647L
+#define INT64_MAX 9223372036854775807LL
 
+#define UINT8_MAX 255
+#define UINT16_MAX 65535
 #define UINT32_MAX 4294967295U
+#define UINT64_MAX 18446744073709551615ULL
 
 #if defined(__wasm32__)
 

crates/language/wasm/include/string.h

@@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
 
 int strncmp(const char *left, const char *right, size_t n);
 
+size_t strlen(const char *str);
+
 #endif // TREE_SITTER_WASM_STRING_H_

crates/language/wasm/src/stdio.c

@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <string.h>
 
 typedef struct {
   bool left_justify;   // -
@@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
   return 2 + len;
 }
 
-size_t strlen(const char *str) {
-  const char *s = str;
-  while (*s) s++;
-  return s - str;
-}
-
 char *strncpy(char *dest, const char *src, size_t n) {
   char *d = dest;
   const char *s = src;

crates/language/wasm/src/stdlib.c

@@ -48,6 +48,19 @@ static int grow_heap(size_t size) {
   return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX;
 }
 
+// Grows the heap if necessary to fit a region at the _end_ of the heap
+// ending at `region_end` by `size` bytes.
+//
+// Returns 0 if the heap could not be grown, 1 otherwise.
+static inline int grow_heap_for_region(Region *region_end, size_t size) {
+  if (region_end > heap_end) {
+    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) return 0;
+    if (!grow_heap(size)) return 0;
+    heap_end = get_heap_end();
+  }
+  return 1;
+}
+
 // Clear out the heap, and move it to the given address.
 void reset_heap(void *new_heap_start) {
   heap_start = new_heap_start;
@@ -76,13 +89,7 @@ void *malloc(size_t size) {
 
   Region *region_end = region_after(next, size);
 
-  if (region_end > heap_end) {
-    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) {
-      return NULL;
-    }
-    if (!grow_heap(size)) return NULL;
-    heap_end = get_heap_end();
-  }
+  if (!grow_heap_for_region(region_end, size)) return NULL;
 
   void *result = &next->data;
   next->size = size;
@@ -109,6 +116,7 @@ void free(void *ptr) {
 
 void *calloc(size_t count, size_t size) {
   void *result = malloc(count * size);
+  if (!result) return NULL;
   memset(result, 0, count * size);
   return result;
 }
@@ -117,19 +125,36 @@ void *realloc(void *ptr, size_t new_size) {
   if (ptr == NULL) {
     return malloc(new_size);
   }
+  if (new_size == 0) {
+    free(ptr);
+    return NULL;
+  }
+
 
   Region *region = region_for_ptr(ptr);
   Region *region_end = region_after(region, region->size);
 
-  // When reallocating the last allocated region, return
-  // the same pointer, and skip copying the data.
+  // When reallocating the last allocated region, resize
+  // in place if possible, return the same pointer, and
+  // skip copying the data.
   if (region_end == next) {
-    next = region;
-    return malloc(new_size);
+    Region *new_region_end = region_after(region, new_size);
+
+    size_t additional_size = (char *)new_region_end - (char *)heap_end;
+    if (!grow_heap_for_region(new_region_end, additional_size)) return NULL;
+
+    region->size = new_size;
+    next = new_region_end;
+    return &region->data;
   }
 
   void *result = malloc(new_size);
-  memcpy(result, &region->data, region->size);
+  if (!result) return NULL;
+
+  size_t copy_size = region->size < new_size ? region->size : new_size;
+  memcpy(result, &region->data, copy_size);
+
+  free(ptr);
   return result;
 }
 

crates/language/wasm/src/string.c

@@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
   }
   return 0;
 }
+
+size_t strlen(const char *str) {
+  const char *s = str;
+  while (*s) s++;
+  return s - str;
+}

crates/loader/src/loader.rs

@@ -1,6 +1,8 @@
 #![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 
+#[cfg(unix)]
+use std::fmt::Write as _;
 #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))]
 use std::ops::Range;
 #[cfg(feature = "tree-sitter-highlight")]
@@ -8,6 +10,7 @@ use std::sync::Mutex;
 use std::{
     collections::HashMap,
     env, fs,
+    hash::{Hash as _, Hasher as _},
     io::{BufRead, BufReader},
     marker::PhantomData,
     mem,
@@ -21,7 +24,7 @@ use etcetera::BaseStrategy as _;
 use fs4::fs_std::FileExt;
 use libloading::{Library, Symbol};
 use log::{error, info, warn};
-use once_cell::unsync::OnceCell;
+use once_cell::sync::OnceCell;
 use regex::{Regex, RegexBuilder};
 use semver::Version;
 use serde::{Deserialize, Deserializer, Serialize};
@@ -75,8 +78,6 @@ pub enum LoaderError {
     NoLanguage,
     #[error(transparent)]
     Query(LoaderQueryError),
-    #[error(transparent)]
-    ScannerSymbols(ScannerSymbolError),
     #[error("Failed to load language for scope '{0}':\n{1}")]
     ScopeLoad(String, Box<Self>),
     #[error(transparent)]
@@ -199,28 +200,6 @@ impl std::fmt::Display for SymbolError {
     }
 }
 
-#[derive(Debug, Error)]
-pub struct ScannerSymbolError {
-    pub missing: Vec<String>,
-}
-
-impl std::fmt::Display for ScannerSymbolError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(
-            f,
-            "Missing required functions in the external scanner, parsing won't work without these!\n"
-        )?;
-        for symbol in &self.missing {
-            writeln!(f, "  `{symbol}`")?;
-        }
-        writeln!(
-            f,
-            "You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n"
-        )?;
-        Ok(())
-    }
-}
-
 #[derive(Debug, Error)]
 pub struct WasiSDKClangError {
     pub wasi_sdk_dir: String,
@@ -654,8 +633,6 @@ impl<'a> CompileConfig<'a> {
     }
 }
 
-unsafe impl Sync for Loader {}
-
 impl Loader {
     pub fn new() -> LoaderResult<Self> {
         let parser_lib_path = if let Ok(path) = env::var("TREE_SITTER_LIBDIR") {
@@ -831,8 +808,11 @@ impl Loader {
                     path = PathBuf::from(path.file_stem()?.to_os_string());
                 }
                 extensions.reverse();
-                self.language_configuration_ids_by_file_type
-                    .get(&extensions.join("."))
+                // Try longest extension suffixs first (e.g. "foo.bar.baz"->"bar.baz"->"baz"),
+                // stopping at the first match.
+                (0..extensions.len())
+                    .map(|i| extensions[i..].join("."))
+                    .find_map(|key| self.language_configuration_ids_by_file_type.get(&key))
             });
 
         if let Some(configuration_ids) = configuration_ids {
@@ -1025,20 +1005,26 @@ impl Loader {
             return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
         }
 
+        // Create a unique lock path based on the output path hash to prevent
+        // interference when multiple processes build the same grammar (by name)
+        // to different output locations
+        let lock_hash = {
+            let mut hasher = std::hash::DefaultHasher::new();
+            output_path.hash(&mut hasher);
+            format!("{:x}", hasher.finish())
+        };
+
         let lock_path = if env::var("CROSS_RUNNER").is_ok() {
             tempfile::tempdir()
-                .unwrap()
+                .expect("create a temp dir")
                 .path()
-                .join("tree-sitter")
-                .join("lock")
-                .join(format!("{}.lock", config.name))
+                .to_path_buf()
         } else {
-            etcetera::choose_base_strategy()?
-                .cache_dir()
-                .join("tree-sitter")
-                .join("lock")
-                .join(format!("{}.lock", config.name))
-        };
+            etcetera::choose_base_strategy()?.cache_dir()
+        }
+        .join("tree-sitter")
+        .join("lock")
+        .join(format!("{}-{lock_hash}.lock", config.name));
 
         if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
             recompile = false;
@@ -1085,10 +1071,30 @@ impl Loader {
             self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?;
 
             if config.scanner_path.is_some() {
-                self.check_external_scanner(&config.name, &output_path)?;
+                self.check_external_scanner(&output_path)?;
             }
         }
 
+        // Ensure the dynamic library exists before trying to load it. This can
+        // happen in race conditions where we couldn't acquire the lock because
+        // another process was compiling but it still hasn't finished by the
+        // time we reach this point, so the output file still doesn't exist.
+        //
+        // Instead of allowing the `load_language` call below to fail, return a
+        // clearer error to the user here.
+        if !output_path.exists() {
+            let msg = format!(
+                "Dynamic library `{}` not found after build attempt. \
+                Are you running multiple processes building to the same output location?",
+                output_path.display()
+            );
+
+            Err(LoaderError::IO(IoError::new(
+                std::io::Error::new(std::io::ErrorKind::NotFound, msg),
+                Some(output_path.as_path()),
+            )))?;
+        }
+
         Self::load_language(&output_path, &language_fn_name)
     }
 
@@ -1187,6 +1193,9 @@ impl Loader {
                 command.arg("-UTREE_SITTER_REUSE_ALLOCATOR");
             } else {
                 command.arg("-shared");
+                command.arg("-Wl,--no-undefined");
+                #[cfg(target_os = "openbsd")]
+                command.arg("-lc");
             }
             command.args(cc_config.get_files());
             command.arg("-o").arg(output_path);
@@ -1221,25 +1230,15 @@ impl Loader {
     }
 
     #[cfg(unix)]
-    fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> {
-        let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) {
-            "_"
+    fn check_external_scanner(&self, library_path: &Path) -> LoaderResult<()> {
+        let section = " T ";
+        // Older ppc toolchains incorrectly report functions in the Data section. This bug has been
+        // fixed, but we still need to account for older systems.
+        let old_ppc_section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
+            Some(" D ")
         } else {
-            ""
+            None
         };
-        let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
-            " D "
-        } else {
-            " T "
-        };
-        let mut must_have = vec![
-            format!("{prefix}tree_sitter_{name}_external_scanner_create"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_destroy"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_serialize"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_deserialize"),
-            format!("{prefix}tree_sitter_{name}_external_scanner_scan"),
-        ];
-
         let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned());
         let command = Command::new(nm_cmd)
             .arg("--defined-only")
@@ -1247,54 +1246,41 @@ impl Loader {
             .output();
         if let Ok(output) = command {
             if output.status.success() {
-                let mut found_non_static = false;
+                let mut non_static_symbols = String::new();
                 for line in String::from_utf8_lossy(&output.stdout).lines() {
-                    if line.contains(section) {
+                    if line.contains(section) || old_ppc_section.is_some_and(|s| line.contains(s)) {
                         if let Some(function_name) =
                             line.split_whitespace().collect::<Vec<_>>().get(2)
                         {
                             if !line.contains("tree_sitter_") {
-                                if !found_non_static {
-                                    found_non_static = true;
-                                    warn!("Found non-static non-tree-sitter functions in the external scanner");
-                                }
-                                warn!("  `{function_name}`");
-                            } else {
-                                must_have.retain(|f| f != function_name);
+                                writeln!(&mut non_static_symbols, "  `{function_name}`").unwrap();
                             }
                         }
                     }
                 }
-                if found_non_static {
-                    warn!(concat!(
-                        "Consider making these functions static, they can cause conflicts ",
-                        "when another tree-sitter project uses the same function name."
-                    ));
-                }
-
-                if !must_have.is_empty() {
-                    return Err(LoaderError::ScannerSymbols(ScannerSymbolError {
-                        missing: must_have,
-                    }));
+                if !non_static_symbols.is_empty() {
+                    warn!(
+                        "Found non-static non-tree-sitter functions in the external scanner\n{non_static_symbols}\n{}",
+                        concat!(
+                            "Consider making these functions static, they can cause conflicts ",
+                            "when another tree-sitter project uses the same function name."
+                        )
+                    );
                 }
             }
+        } else {
+            warn!(
+                "Failed to run `nm` to verify symbols in {}",
+                library_path.display()
+            );
         }
 
         Ok(())
     }
 
     #[cfg(windows)]
-    fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> {
+    fn check_external_scanner(&self, _library_path: &Path) -> LoaderResult<()> {
         // TODO: there's no nm command on windows, whoever wants to implement this can and should :)
-
-        // let mut must_have = vec![
-        //     format!("tree_sitter_{name}_external_scanner_create"),
-        //     format!("tree_sitter_{name}_external_scanner_destroy"),
-        //     format!("tree_sitter_{name}_external_scanner_serialize"),
-        //     format!("tree_sitter_{name}_external_scanner_deserialize"),
-        //     format!("tree_sitter_{name}_external_scanner_scan"),
-        // ];
-
         Ok(())
     }
 
@@ -1309,6 +1295,7 @@ impl Loader {
 
         let mut command = Command::new(&clang_executable);
         command.current_dir(src_path).args([
+            "--target=wasm32-unknown-wasi",
             "-o",
             output_path.to_str().unwrap(),
             "-fPIC",
@@ -1487,7 +1474,7 @@ impl Loader {
     ) -> Option<&'a HighlightConfiguration> {
         match self.language_configuration_for_injection_string(string) {
             Err(e) => {
-                error!("Failed to load language for injection string '{string}': {e}",);
+                error!("Failed to load language for injection string '{string}': {e}");
                 None
             }
             Ok(None) => None,

crates/tags/src/tags.rs

@@ -518,13 +518,10 @@ where
                         // reuse results from the previous tag.
                         let mut prev_utf16_column = 0;
                         let mut prev_utf8_byte = name_range.start - span.start.column;
-                        let line_info = self.prev_line_info.as_ref().and_then(|info| {
-                            if info.utf8_position.row == span.start.row {
-                                Some(info)
-                            } else {
-                                None
-                            }
-                        });
+                        let line_info = self
+                            .prev_line_info
+                            .as_ref()
+                            .filter(|&info| info.utf8_position.row == span.start.row);
                         let line_range = if let Some(line_info) = line_info {
                             if line_info.utf8_position.column <= span.start.column {
                                 prev_utf8_byte = line_info.utf8_byte;

crates/xtask/src/build_wasm.rs

@@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
         "-D", "NDEBUG=",
         "-D", "_POSIX_C_SOURCE=200112L",
         "-D", "_DEFAULT_SOURCE=",
+        "-D", "_BSD_SOURCE=",
         "-D", "_DARWIN_C_SOURCE=",
         "-I", "lib/src",
         "-I", "lib/include",

crates/xtask/src/fetch.rs

@@ -1,19 +1,74 @@
-use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION};
+use crate::{bail_on_err, root_dir, EMSCRIPTEN_VERSION};
 use anyhow::Result;
-use std::{fs, process::Command};
+use std::{fs, path::Path, process::Command};
 
-pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
+enum FixtureRef<'a> {
+    Tag(&'a str),
+    Branch(&'a str),
+}
+
+impl<'a> FixtureRef<'a> {
+    #[allow(clippy::use_self)]
+    const fn new(tag: &'a str, branch: Option<&'a str>) -> FixtureRef<'a> {
+        if let Some(b) = branch {
+            Self::Branch(b)
+        } else {
+            Self::Tag(tag)
+        }
+    }
+
+    const fn ref_type(&self) -> &'static str {
+        match self {
+            FixtureRef::Tag(_) => "tag",
+            FixtureRef::Branch(_) => "branch",
+        }
+    }
+}
+
+impl std::fmt::Display for FixtureRef<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FixtureRef::Tag(tag) => write!(f, "{tag}"),
+            FixtureRef::Branch(branch) => write!(f, "{branch}"),
+        }
+    }
+}
+
+fn current_ref_name(grammar_dir: &Path) -> Result<(String, Option<&'static str>)> {
+    let tag_args = ["describe", "--tags", "--exact-match", "HEAD"];
+    let branch_args = ["rev-parse", "--abbrev-ref", "HEAD"];
+
+    for (args, ref_type) in [tag_args.as_ref(), branch_args.as_ref()]
+        .iter()
+        .zip(&["tag", "branch"])
+    {
+        let name_cmd = Command::new("git")
+            .current_dir(grammar_dir)
+            .args(*args)
+            .output()?;
+        let name = String::from_utf8_lossy(&name_cmd.stdout);
+        let name = name.trim();
+        if !name.is_empty() {
+            return Ok((name.to_string(), Some(ref_type)));
+        }
+    }
+
+    Ok(("<unknown>".to_string(), None))
+}
+
+pub fn run_fixtures() -> Result<()> {
     let fixtures_dir = root_dir().join("test").join("fixtures");
     let grammars_dir = fixtures_dir.join("grammars");
     let fixtures_path = fixtures_dir.join("fixtures.json");
 
-    // grammar name, tag
-    let mut fixtures: Vec<(String, String)> =
+    // grammar name, tag, [branch]
+    let fixtures: Vec<(String, String, Option<String>)> =
         serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?;
 
-    for (grammar, tag) in &mut fixtures {
-        let grammar_dir = grammars_dir.join(&grammar);
+    for (grammar, tag, branch) in &fixtures {
+        let grammar_dir = grammars_dir.join(grammar);
         let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}");
+        let target_ref = FixtureRef::new(tag, branch.as_deref());
 
         println!("Fetching the {grammar} grammar...");
 
@@ -24,7 +79,7 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 "--depth",
                 "1",
                 "--branch",
-                tag,
+                &target_ref.to_string(),
                 &grammar_url,
                 &grammar_dir.to_string_lossy(),
             ]);
@@ -33,31 +88,71 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 &format!("Failed to clone the {grammar} grammar"),
             )?;
         } else {
-            let mut describe_command = Command::new("git");
-            describe_command.current_dir(&grammar_dir).args([
-                "describe",
-                "--tags",
-                "--exact-match",
-                "HEAD",
-            ]);
+            let (current_ref, current_ref_type) = current_ref_name(&grammar_dir)?;
+            if current_ref != target_ref.to_string() {
+                println!(
+                    "Updating {grammar} grammar from {} {current_ref} to {} {target_ref}...",
+                    current_ref_type.unwrap_or("<unknown>"),
+                    target_ref.ref_type(),
+                );
 
-            let output = describe_command.output()?;
-            let current_tag = String::from_utf8_lossy(&output.stdout);
-            let current_tag = current_tag.trim();
-
-            if current_tag != tag {
-                println!("Updating {grammar} grammar from {current_tag} to {tag}...");
-
-                let mut fetch_command = Command::new("git");
-                fetch_command.current_dir(&grammar_dir).args([
-                    "fetch",
-                    "origin",
-                    &format!("refs/tags/{tag}:refs/tags/{tag}"),
-                ]);
-                bail_on_err(
-                    &fetch_command.spawn()?.wait_with_output()?,
-                    &format!("Failed to fetch tag {tag} for {grammar} grammar"),
-                )?;
+                match target_ref {
+                    FixtureRef::Branch(branch) => {
+                        let mut fetch_cmd = Command::new("git");
+                        fetch_cmd.current_dir(&grammar_dir).args([
+                            "fetch",
+                            "--update-shallow",
+                            "origin",
+                            &format!("+refs/heads/{branch}:refs/remotes/origin/{branch}"),
+                        ]);
+                        bail_on_err(
+                            &fetch_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to fetch branch {branch}"),
+                        )?;
+                        let mut switch_cmd = Command::new("git");
+                        switch_cmd
+                            .current_dir(&grammar_dir)
+                            .args(["switch", branch]);
+                        bail_on_err(
+                            &switch_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to checkout branch {branch}"),
+                        )?;
+                        let mut set_upstream_cmd = Command::new("git");
+                        set_upstream_cmd.current_dir(&grammar_dir).args([
+                            "branch",
+                            "--set-upstream-to",
+                            &format!("origin/{branch}"),
+                            branch,
+                        ]);
+                        bail_on_err(
+                            &set_upstream_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to set upstream for branch {branch}"),
+                        )?;
+                        let mut pull_cmd = Command::new("git");
+                        pull_cmd
+                            .current_dir(&grammar_dir)
+                            .args(["pull", "origin", branch]);
+                        bail_on_err(
+                            &pull_cmd.spawn()?.wait_with_output()?,
+                            &format!("Failed to pull latest from branch {branch}"),
+                        )?;
+                    }
+                    FixtureRef::Tag(tag) => {
+                        let mut fetch_command = Command::new("git");
+                        fetch_command.current_dir(&grammar_dir).args([
+                            "fetch",
+                            "origin",
+                            &format!("refs/tags/{tag}:refs/tags/{tag}"),
+                        ]);
+                        bail_on_err(
+                            &fetch_command.spawn()?.wait_with_output()?,
+                            &format!(
+                                "Failed to fetch {} {target_ref} for {grammar} grammar",
+                                target_ref.ref_type()
+                            ),
+                        )?;
+                    }
+                }
 
                 let mut reset_command = Command::new("git");
                 reset_command
@@ -71,29 +166,23 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
                 let mut checkout_command = Command::new("git");
                 checkout_command
                     .current_dir(&grammar_dir)
-                    .args(["checkout", tag]);
+                    .args(["checkout", &target_ref.to_string()]);
                 bail_on_err(
                     &checkout_command.spawn()?.wait_with_output()?,
-                    &format!("Failed to checkout tag {tag} for {grammar} grammar"),
+                    &format!(
+                        "Failed to checkout {} {target_ref} for {grammar} grammar",
+                        target_ref.ref_type()
+                    ),
                 )?;
             } else {
-                println!("{grammar} grammar is already at tag {tag}");
+                println!(
+                    "{grammar} grammar is already at {} {target_ref}",
+                    target_ref.ref_type()
+                );
             }
         }
     }
 
-    if args.update {
-        println!("Updating the fixtures lock file");
-        fs::write(
-            &fixtures_path,
-            // format the JSON without extra newlines
-            serde_json::to_string(&fixtures)?
-                .replace("[[", "[\n  [")
-                .replace("],", "],\n  ")
-                .replace("]]", "]\n]"),
-        )?;
-    }
-
     Ok(())
 }
 

crates/xtask/src/generate.rs

@@ -68,9 +68,11 @@ pub fn run_bindings() -> Result<()> {
     let output = Command::new("cargo")
         .args(["metadata", "--format-version", "1"])
         .output()
-        .unwrap();
+        .context("Failed to execute cargo metadata")?;
+    bail_on_err(&output, "Failed to run cargo metadata")?;
 
-    let metadata = serde_json::from_slice::<serde_json::Value>(&output.stdout).unwrap();
+    let metadata = serde_json::from_slice::<serde_json::Value>(&output.stdout)
+        .context("Failed to parse cargo metadata output")?;
 
     let Some(rust_version) = metadata
         .get("packages")
@@ -118,7 +120,7 @@ pub fn run_bindings() -> Result<()> {
 
     bindings
         .write_to_file("lib/binding_rust/bindings.rs")
-        .with_context(|| "Failed to write bindings")
+        .context("Failed to write bindings")
 }
 
 pub fn run_wasm_exports() -> Result<()> {

crates/xtask/src/main.rs

@@ -36,7 +36,7 @@ enum Commands {
     /// Fetches emscripten.
     FetchEmscripten,
     /// Fetches the fixtures for testing tree-sitter.
-    FetchFixtures(FetchFixtures),
+    FetchFixtures,
     /// Generate the Rust bindings from the C library.
     GenerateBindings,
     /// Generates the fixtures for testing tree-sitter.
@@ -118,13 +118,6 @@ struct Clippy {
     package: Option<String>,
 }
 
-#[derive(Args)]
-struct FetchFixtures {
-    /// Update all fixtures to the latest tag
-    #[arg(long, short)]
-    update: bool,
-}
-
 #[derive(Args)]
 struct GenerateFixtures {
     /// Generates the parser to Wasm
@@ -232,8 +225,8 @@ fn run() -> Result<()> {
         Commands::CheckWasmExports(check_options) => check_wasm_exports::run(&check_options)?,
         Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?,
         Commands::FetchEmscripten => fetch::run_emscripten()?,
-        Commands::FetchFixtures(fetch_fixture_options) => {
-            fetch::run_fixtures(&fetch_fixture_options)?;
+        Commands::FetchFixtures => {
+            fetch::run_fixtures()?;
         }
         Commands::GenerateBindings => generate::run_bindings()?,
         Commands::GenerateFixtures(generate_fixtures_options) => {

crates/xtask/src/upgrade_wasmtime.rs

@@ -3,7 +3,7 @@ use std::process::Command;
 use anyhow::{Context, Result};
 use semver::Version;
 
-use crate::{create_commit, UpgradeWasmtime};
+use crate::{bail_on_err, create_commit, UpgradeWasmtime};
 
 const WASMTIME_RELEASE_URL: &str = "https://github.com/bytecodealliance/wasmtime/releases/download";
 
@@ -22,11 +22,13 @@ fn update_cargo(version: &Version) -> Result<()> {
 
     std::fs::write("lib/Cargo.toml", new_lines.join("\n") + "\n")?;
 
-    Command::new("cargo")
+    let output = Command::new("cargo")
         .arg("update")
-        .status()
-        .map(|_| ())
-        .with_context(|| "Failed to execute cargo update")
+        .spawn()?
+        .wait_with_output()?;
+    bail_on_err(&output, "Failed to run cargo update")?;
+
+    Ok(())
 }
 
 fn zig_fetch(lines: &mut Vec<String>, version: &Version, url_suffix: &str) -> Result<()> {
@@ -39,9 +41,15 @@ fn zig_fetch(lines: &mut Vec<String>, version: &Version, url_suffix: &str) -> Re
         .arg(url)
         .output()
         .with_context(|| format!("Failed to execute zig fetch {url}"))?;
+    bail_on_err(&output, &format!("`zig fetch {url}` failed"))?;
 
-    let hash = String::from_utf8_lossy(&output.stdout);
-    lines.push(format!("            .hash = \"{}\",", hash.trim_end()));
+    let hash = std::str::from_utf8(&output.stdout)
+        .with_context(|| format!("`zig fetch {url}` produced non-UTF-8 output"))?
+        .trim_end();
+    if hash.is_empty() {
+        anyhow::bail!("`zig fetch {url}` produced an empty hash");
+    }
+    lines.push(format!("            .hash = \"{hash}\","));
 
     Ok(())
 }

docs/src/assets/js/playground.js

@@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
 }
 
 window.initializePlayground = async (opts) => {
-  const { Parser, Language } = window.TreeSitter;
+  const { Parser, Language, Query } = window.TreeSitter;
 
   const { local } = opts;
   if (local) {
@@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
       marks.forEach((m) => m.clear());
 
       if (tree && query) {
-        const captures = query.captures(
-          tree.rootNode,
-          { row: startRow, column: 0 },
-          { row: endRow, column: 0 },
-        );
+        const captures = query.captures(tree.rootNode, {
+          startPosition: { row: startRow, column: 0 },
+          endPosition: { row: endRow, column: 0 },
+        });
         let lastNodeId;
         for (const { name, node } of captures) {
           if (node.id === lastNodeId) continue;
@@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
       const queryText = queryEditor.getValue();
 
       try {
-        query = parser.language.query(queryText);
+        query = new Query(parser.language, queryText);
         let match;
 
         let row = 0;

docs/src/cli/fuzz.md

@@ -31,11 +31,13 @@ If `--lib-path` is used, the name of the language used to extract the library's
 
 ### `--edits <EDITS>`
 
-The maximum number of edits to perform. The default is 3.
+The maximum number of edits to perform. The default is 3. This value can also be set via the `TREE_SITTER_EDITS` environment
+variable.
 
 ### `--iterations <ITERATIONS>`
 
-The number of iterations to run. The default is 10.
+The number of iterations to run. The default is 10. This value can also be set via the `TREE_SITTER_ITERATIONS` environment
+variable.
 
 ### `-i/--include <INCLUDE>`
 

docs/src/creating-parsers/1-getting-started.md

@@ -41,6 +41,10 @@ cd tree-sitter-${LOWER_PARSER_NAME}
 The `LOWER_` prefix here means the "lowercase" name of the language.
 ```
 
+```admonish warning
+Dashes are not permitted via the CLI's `init` command and should not be used in parser names.
+```
+
 ### Init
 
 Once you've installed the `tree-sitter` CLI tool, you can start setting up your project, which will allow your parser to

docs/src/using-parsers/queries/2-operators.md

@@ -110,6 +110,69 @@ This pattern would match a set of possible keyword tokens, capturing them as `@k
 ] @keyword
 ```
 
+Alternations can have quantified alternants, and then can have their own
+quantifiers as well. See the following examples for an illustration of how these
+cases work:
+
+```query
+;;; SOURCE CODE ;;;
+; #include <foo>
+; #include <bar>
+; #include <baz>
+; // comment
+;;;;;;;;;;;;;;;;;;;
+
+[
+  (preproc_include)
+  (comment)
+]+ @capture
+; ^ Produces one match with four captures:
+; [
+;   "#include <foo>\n",
+;   "#include <bar>\n",
+;   "#include <baz>\n",
+;   "// comment",
+; ]
+;
+; Regex equivalent: [ab]+
+
+[
+  (preproc_include)+
+  (comment)
+] @capture
+; ^ Produces two matches; one with three captures, and one with one capture:
+; [
+;   "#include <foo>\n",
+;   "#include <bar>\n",
+;   "#include <baz>\n",
+; ],
+; [
+;   "// comment",
+; ]
+;
+; Regex equivalent: a+|b
+
+[
+  (preproc_include)
+  (comment)
+] @capture
+; ^ Produces four matches, each with one capture:
+; [
+;   "#include <foo>\n",
+; ],
+; [
+;   "#include <bar>\n",
+; ],
+; [
+;   "#include <baz>\n",
+; ],
+; [
+;   "// comment",
+; ]
+;
+; Regex equivalent: [ab]
+```
+
 ## Anchors
 
 The anchor operator, `.`, is used to constrain the ways in which child patterns are matched. It has different behaviors
@@ -117,7 +180,7 @@ depending on where it's placed inside a query.
 
 When `.` is placed before the _first_ child within a parent pattern, the child will only match when it is the first named
 node in the parent. For example, the below pattern matches a given `array` node at most once, assigning the `@the-element`
-capture to the first `identifier` node in the parent `array`:
+capture to the first node in the parent `array`, only if it's an `identifier` node:
 
 ```query
 (array . (identifier) @the-element)
@@ -148,4 +211,52 @@ Without the anchor, non-consecutive pairs like `a, c` and `b, d` would also be m
 
 The restrictions placed on a pattern by an anchor operator ignore anonymous nodes.
 
+### Anchors with Quantifiers and Groups
+
+When an anchor is next to a quantified node (`*`, `+`, `?`), its meaning depends on whether the
+anchor sits _between two patterns_ or _at the edge of a parent node_.
+
+An anchor _between two child patterns_ constrains the two matched nodes to be immediate siblings.
+If one of those patterns is a quantifier that matches zero nodes, there is no node on that side,
+so the anchor imposes no constraint. For example, given
+
+```query
+(translation_unit (comment)* @doc . (function_definition) @function)
+```
+
+a `function_definition` with no preceding `comment` still matches, with `@doc` capturing nothing.
+When comments are present, they must immediately precede the function.
+
+An anchor at the _start or end of a node pattern_ (a leading or trailing `.`) constrains the
+matched sequence to begin at the parent's first, or end at its last, named child. If the pattern
+element next to that edge is a quantifier that matches zero nodes, the constraint applies to the
+nearest node that the pattern _does_ match. For example, given
+
+```query
+(preproc_if (preproc_def)+ @def . (preproc_else)? @else .)
+```
+
+the trailing anchor requires the last matched node to be the parent's last named child: when a
+`preproc_else` is present it must be last. When it is absent, the last `preproc_def` must be last.
+
+Similarly, if an optionally quantified node is anchored between two siblings and matches zero nodes,
+both sibling anchors collapse into one, constraining the outer nodes together. For example, given
+
+```query
+(translation_unit
+  (declaration) @a
+  .
+  (comment)*
+  .
+  (function_definition) @b)
+```
+
+If there are no comments, `(declaration)` and `(function_definition)` must be immediate siblings
+in order for the query to match.
+
+An anchor may not appear at the first or last position inside a group `(...)` or an alternation
+`[...]`. A group or alternation is not a node, so it has no first or last child to anchor against,
+and there is no sibling on that side to anchor to. For example, write `(comment)* @doc . (function)`
+rather than `((comment)+ @doc .)? (function)`.
+
 [regex]: https://en.wikipedia.org/wiki/Regular_expression#Basic_concepts

docs/src/using-parsers/queries/4-api.md

@@ -59,3 +59,27 @@ bool ts_query_cursor_next_match(TSQueryCursor *, TSQueryMatch *match);
 
 This function will return `false` when there are no more matches. Otherwise, it will populate the `match` with data about
 which pattern matched and which nodes were captured.
+
+## Restricting the Query Range
+
+You can restrict the range in which the query is executed using byte offsets or point (row, column) positions:
+
+```c
+bool ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
+bool ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
+```
+
+These functions return matches that *intersect* with the given range. A match may be returned even if only part of it overlaps
+with the range.
+
+There are also "containing" variants that only return matches where all captured nodes are fully within the range:
+
+```c
+bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
+bool ts_query_cursor_set_containing_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
+```
+
+```admonish note
+For all of these functions, an end value of zero is treated as unbounded (the maximum possible value).
+This means passing a byte range of `(0, 0)` (or a point range of `{0, 0}, {0, 0}`) will match the entire tree, not an empty range.
+```

flake.nix

@@ -17,7 +17,7 @@
       eachSystem = lib.genAttrs systems;
       pkgsFor = inputs.nixpkgs.legacyPackages;
 
-      version = "0.26.3";
+          version = "0.26.13";
 
       fs = lib.fileset;
       src = fs.toSource {
@@ -333,6 +333,7 @@
               pkg-config
               llvm
               clang
+              clang-tools
               libclang
 
               nodejs_22
@@ -348,6 +349,7 @@
             ];
 
             shellHook = ''
+              export PATH="${pkgs.clang-tools}/bin:$PATH"
               echo "Tree-sitter Dev Environment"
               echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
               echo ""

lib/Cargo.toml

@@ -53,7 +53,7 @@ tree-sitter-language.workspace = true
 streaming-iterator = "0.1.9"
 
 [dependencies.wasmtime-c-api]
-version = "33.0.2"
+version = "36.0.14"
 optional = true
 package = "wasmtime-c-api-impl"
 default-features = false

lib/binding_rust/bindings.rs

@@ -106,6 +106,7 @@ pub struct TSLogger {
         ),
     >,
 }
+#[doc = " A summary of a change to a text document.\n\n The `start_byte` and `start_point` values must be less than or equal to the\n `old_end_byte` and `old_end_point` values, respectively. Passing an edit\n that violates these invariants may produce nonsensical results."]
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct TSInputEdit {
@@ -218,7 +219,7 @@ extern "C" {
     pub fn ts_parser_included_ranges(self_: *const TSParser, count: *mut u32) -> *const TSRange;
 }
 extern "C" {
-    #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n    and (row, column) position. The function should return a pointer to the\n    text and write its length to the [`bytes_read`] pointer. The parser does\n    not take ownership of this buffer; it just borrows it until it has\n    finished reading it. The function should write a zero value to the\n    [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n    of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n    `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to the progress callback returning true. This callback\n    is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read"]
+    #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n    and (row, column) position. The function should return a pointer to the\n    text and write its length to the [`bytes_read`] pointer. The parser does\n    not take ownership of this buffer; it just borrows it until it has\n    finished reading it. The function should write a zero value to the\n    [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n    of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n    `TSInputEncodingUTF8`, `TSInputEncodingUTF16LE`, `TSInputEncoding16BE`,\n    or `TSInputEncodingCustom`.\n 4. [`decode`]: A function to read one code point from the given input. This\n    function should return the number of bytes consumed and write the code point\n    to the [`code_point`] pointer, or write -1 if the input is invalid.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to the progress callback returning true. This callback\n    is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read\n [`decode`]: TSInput::decode\n [`code_point`]: TSDecodeFunction::code_point"]
     pub fn ts_parser_parse(
         self_: *mut TSParser,
         old_tree: *const TSTree,
@@ -298,7 +299,7 @@ extern "C" {
     pub fn ts_tree_included_ranges(self_: *const TSTree, length: *mut u32) -> *mut TSRange;
 }
 extern "C" {
-    #[doc = " Edit the syntax tree to keep it in sync with source code that has been\n edited.\n\n You must describe the edit both in terms of byte offsets and in terms of\n (row, column) coordinates."]
+    #[doc = " Edit the syntax tree to keep it in sync with source code that has been\n edited.\n\n You must describe the edit both in terms of byte offsets and in terms of\n (row, column) coordinates.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_tree_edit(self_: *mut TSTree, edit: *const TSInputEdit);
 }
 extern "C" {
@@ -488,7 +489,7 @@ extern "C" {
     ) -> TSNode;
 }
 extern "C" {
-    #[doc = " Edit the node to keep it in-sync with source code that has been edited.\n\n This function is only rarely needed. When you edit a syntax tree with the\n [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree\n afterward will already reflect the edit. You only need to use [`ts_node_edit`]\n when you have a [`TSNode`] instance that you want to keep and continue to use\n after an edit."]
+    #[doc = " Edit the node to keep it in-sync with source code that has been edited.\n\n This function is only rarely needed. When you edit a syntax tree with the\n [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree\n afterward will already reflect the edit. You only need to use [`ts_node_edit`]\n when you have a [`TSNode`] instance that you want to keep and continue to use\n after an edit.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_node_edit(self_: *mut TSNode, edit: *const TSInputEdit);
 }
 extern "C" {
@@ -496,11 +497,11 @@ extern "C" {
     pub fn ts_node_eq(self_: TSNode, other: TSNode) -> bool;
 }
 extern "C" {
-    #[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance."]
+    #[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_point_edit(point: *mut TSPoint, point_byte: *mut u32, edit: *const TSInputEdit);
 }
 extern "C" {
-    #[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance."]
+    #[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
     pub fn ts_range_edit(range: *mut TSRange, edit: *const TSInputEdit);
 }
 extern "C" {
@@ -697,7 +698,7 @@ extern "C" {
     pub fn ts_query_cursor_set_match_limit(self_: *mut TSQueryCursor, limit: u32);
 }
 extern "C" {
-    #[doc = " Set the range of bytes in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."]
+    #[doc = " Set the range of bytes in which the query will be executed.\n\n The query cursor will return matches that intersect with the given byte range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range\n unbounded.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."]
     pub fn ts_query_cursor_set_byte_range(
         self_: *mut TSQueryCursor,
         start_byte: u32,
@@ -705,7 +706,7 @@ extern "C" {
     ) -> bool;
 }
 extern "C" {
-    #[doc = " Set the range of (row, column) positions in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start point is greater than the end point, otherwise\n it will return `true`."]
+    #[doc = " Set the range of (row, column) positions in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the\n range unbounded.\n\n This will return `false` if the start point is greater than the end point, otherwise\n it will return `true`."]
     pub fn ts_query_cursor_set_point_range(
         self_: *mut TSQueryCursor,
         start_point: TSPoint,
@@ -713,7 +714,7 @@ extern "C" {
     ) -> bool;
 }
 extern "C" {
-    #[doc = " Set the byte range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"]
+    #[doc = " Set the byte range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500\n\n NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range\n unbounded."]
     pub fn ts_query_cursor_set_containing_byte_range(
         self_: *mut TSQueryCursor,
         start_byte: u32,
@@ -721,7 +722,7 @@ extern "C" {
     ) -> bool;
 }
 extern "C" {
-    #[doc = " Set the point range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"]
+    #[doc = " Set the point range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500\n\n NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the\n range unbounded."]
     pub fn ts_query_cursor_set_containing_point_range(
         self_: *mut TSQueryCursor,
         start_point: TSPoint,

lib/binding_rust/build.rs

@@ -39,7 +39,7 @@ fn main() {
     }
 
     config
-        .flag_if_supported("-std=c11")
+        .std("c11")
         .flag_if_supported("-fvisibility=hidden")
         .flag_if_supported("-Wshadow")
         .flag_if_supported("-Wno-unused-parameter")
@@ -49,6 +49,7 @@ fn main() {
         .include(&include_path)
         .define("_POSIX_C_SOURCE", "200112L")
         .define("_DEFAULT_SOURCE", None)
+        .define("_BSD_SOURCE", None)
         .define("_DARWIN_C_SOURCE", None)
         .warnings(false)
         .file(src_path.join("lib.c"))

lib/binding_rust/lib.rs

@@ -1451,7 +1451,6 @@ impl Tree {
     /// functions. Call it on the old tree that was passed to parse, and
     /// pass the new tree that was returned from `parse`.
     #[doc(alias = "ts_tree_get_changed_ranges")]
-    #[must_use]
     pub fn changed_ranges(&self, other: &Self) -> impl ExactSizeIterator<Item = Range> {
         let mut count = 0u32;
         unsafe {
@@ -2398,9 +2397,7 @@ impl Query {
         }
         let column = offset - line_start;
 
-        let kind;
-        let message;
-        match error_type {
+        let (message, kind) = match error_type {
             // Error types that report names
             ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => {
                 let suffix = source.split_at(offset).1;
@@ -2423,27 +2420,29 @@ impl Query {
                         }
                     })
                     .unwrap_or(suffix.len());
-                message = format!("\"{}\"", suffix.split_at(end_offset).0);
-                kind = match error_type {
-                    ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType,
-                    ffi::TSQueryErrorField => QueryErrorKind::Field,
-                    ffi::TSQueryErrorCapture => QueryErrorKind::Capture,
-                    _ => unreachable!(),
-                };
+                (
+                    format!("\"{}\"", suffix.split_at(end_offset).0),
+                    match error_type {
+                        ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType,
+                        ffi::TSQueryErrorField => QueryErrorKind::Field,
+                        ffi::TSQueryErrorCapture => QueryErrorKind::Capture,
+                        _ => unreachable!(),
+                    },
+                )
             }
 
             // Error types that report positions
-            _ => {
-                message = line_containing_error.map_or_else(
+            _ => (
+                line_containing_error.map_or_else(
                     || "Unexpected EOF".to_string(),
                     |line| line.to_string() + "\n" + &" ".repeat(offset - line_start) + "^",
-                );
-                kind = match error_type {
+                ),
+                match error_type {
                     ffi::TSQueryErrorStructure => QueryErrorKind::Structure,
                     _ => QueryErrorKind::Syntax,
-                };
-            }
-        }
+                },
+            ),
+        };
 
         Err(QueryError {
             row,
@@ -2471,7 +2470,7 @@ impl Query {
         let pattern_count = unsafe { ffi::ts_query_pattern_count(ptr.0) as usize };
 
         let mut capture_names = Vec::with_capacity(capture_count as usize);
-        let mut capture_quantifiers_vec = Vec::with_capacity(pattern_count as usize);
+        let mut capture_quantifiers_vec = Vec::with_capacity(pattern_count);
         let mut text_predicates_vec = Vec::with_capacity(pattern_count);
         let mut property_predicates_vec = Vec::with_capacity(pattern_count);
         let mut property_settings_vec = Vec::with_capacity(pattern_count);
@@ -2924,7 +2923,7 @@ impl Query {
         } else {
             Err(predicate_error(
                 row,
-                format!("Invalid arguments to {function_name} predicate. Missing key argument",),
+                format!("Invalid arguments to {function_name} predicate. Missing key argument"),
             ))
         }
     }
@@ -3746,11 +3745,11 @@ impl fmt::Display for QueryError {
 #[must_use]
 pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
     let mut indent_level = initial_indent_level;
-    let mut formatted = String::new();
+    let mut formatted = String::with_capacity(sexp.len());
     let mut has_field = false;
 
     let mut c_iter = sexp.chars().peekable();
-    let mut s = String::with_capacity(sexp.len());
+    let mut scratch = String::with_capacity(sexp.len());
     let mut quote = '\0';
     let mut saw_paren = false;
     let mut did_last = false;
@@ -3796,12 +3795,12 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
         Some(())
     };
 
-    while fetch_next_str(&mut s).is_some() {
-        if s.is_empty() && indent_level > 0 {
+    while fetch_next_str(&mut scratch).is_some() {
+        if scratch.is_empty() && indent_level > 0 {
             // ")"
             indent_level -= 1;
             write!(formatted, ")").unwrap();
-        } else if s.starts_with('(') {
+        } else if scratch.starts_with('(') {
             if has_field {
                 has_field = false;
             } else {
@@ -3815,27 +3814,27 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
             }
 
             // "(node_name"
-            write!(formatted, "{s}").unwrap();
+            write!(formatted, "{scratch}").unwrap();
 
             // "(MISSING node_name" or "(UNEXPECTED 'x'"
-            if s.starts_with("(MISSING") || s.starts_with("(UNEXPECTED") {
-                fetch_next_str(&mut s).unwrap();
-                if s.is_empty() {
+            if scratch.starts_with("(MISSING") || scratch.starts_with("(UNEXPECTED") {
+                fetch_next_str(&mut scratch).unwrap();
+                if scratch.is_empty() {
                     while indent_level > 0 {
                         indent_level -= 1;
                         write!(formatted, ")").unwrap();
                     }
                 } else {
-                    write!(formatted, " {s}").unwrap();
+                    write!(formatted, " {scratch}").unwrap();
                 }
             }
-        } else if s.ends_with(':') {
+        } else if scratch.ends_with(':') {
             // "field:"
             writeln!(formatted).unwrap();
             for _ in 0..indent_level {
                 write!(formatted, "  ").unwrap();
             }
-            write!(formatted, "{s} ").unwrap();
+            write!(formatted, "{scratch} ").unwrap();
             has_field = true;
             indent_level += 1;
         }

lib/binding_web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.26.3",
+  "version": "0.26.13",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "web-tree-sitter",
-      "version": "0.26.3",
+      "version": "0.26.13",
       "license": "MIT",
       "devDependencies": {
         "@eslint/js": "^9.39.1",

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.26.3",
+  "version": "0.26.13",
   "description": "Tree-sitter bindings for the web",
   "repository": {
     "type": "git",

lib/binding_web/script/build.js

@@ -44,6 +44,9 @@ async function build() {
     keepNames: true,
     external: ['fs/*', 'fs/promises'],
     resolveExtensions: ['.ts', '.js', format === 'esm' ? '.mjs' : '.cjs'],
+    ...(format === 'cjs' ? {
+      footer: { js: 'module.exports.default = module.exports;' },
+    } : {}),
   });
 
   // Copy the Wasm files to the appropriate spot, as esbuild doesn't "bundle" Wasm files

lib/include/tree_sitter/api.h

@@ -114,6 +114,13 @@ typedef struct TSLogger {
   void (*log)(void *payload, TSLogType log_type, const char *buffer);
 } TSLogger;
 
+/**
+ * A summary of a change to a text document.
+ *
+ * The `start_byte` and `start_point` values must be less than or equal to the
+ * `old_end_byte` and `old_end_point` values, respectively. Passing an edit
+ * that violates these invariants may produce nonsensical results.
+ */
 typedef struct TSInputEdit {
   uint32_t start_byte;
   uint32_t old_end_byte;
@@ -293,7 +300,11 @@ const TSRange *ts_parser_included_ranges(
  * 2. [`payload`]: An arbitrary pointer that will be passed to each invocation
  *    of the [`read`] function.
  * 3. [`encoding`]: An indication of how the text is encoded. Either
- *    `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.
+ *    `TSInputEncodingUTF8`, `TSInputEncodingUTF16LE`, `TSInputEncoding16BE`,
+ *    or `TSInputEncodingCustom`.
+ * 4. [`decode`]: A function to read one code point from the given input. This
+ *    function should return the number of bytes consumed and write the code point
+ *    to the [`code_point`] pointer, or write -1 if the input is invalid.
  *
  * This function returns a syntax tree on success, and `NULL` on failure. There
  * are four possible reasons for failure:
@@ -306,6 +317,8 @@ const TSRange *ts_parser_included_ranges(
  * [`payload`]: TSInput::payload
  * [`encoding`]: TSInput::encoding
  * [`bytes_read`]: TSInput::read
+ * [`decode`]: TSInput::decode
+ * [`code_point`]: TSDecodeFunction::code_point
  */
 TSTree *ts_parser_parse(
   TSParser *self,
@@ -437,6 +450,9 @@ TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length);
  *
  * You must describe the edit both in terms of byte offsets and in terms of
  * (row, column) coordinates.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_tree_edit(TSTree *self, const TSInputEdit *edit);
 
@@ -700,6 +716,9 @@ TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPo
  * afterward will already reflect the edit. You only need to use [`ts_node_edit`]
  * when you have a [`TSNode`] instance that you want to keep and continue to use
  * after an edit.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_node_edit(TSNode *self, const TSInputEdit *edit);
 
@@ -714,6 +733,9 @@ bool ts_node_eq(TSNode self, TSNode other);
  * This function updates a single point's byte offset and row/column position
  * based on an edit operation. This is useful for editing points without
  * requiring a tree or node instance.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit);
 
@@ -723,6 +745,9 @@ void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit
  * This function updates a range's start and end positions based on an edit
  * operation. This is useful for editing ranges without requiring a tree
  * or node instance.
+ *
+ * The edit's `start_byte` must be less than or equal to its `old_end_byte`,
+ * and its `start_point` must be less than or equal to its `old_end_point`.
  */
 void ts_range_edit(TSRange *range, const TSInputEdit *edit);
 
@@ -1070,7 +1095,7 @@ void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit);
 /**
  * Set the range of bytes in which the query will be executed.
  *
- * The query cursor will return matches that intersect with the given point range.
+ * The query cursor will return matches that intersect with the given byte range.
  * This means that a match may be returned even if some of its captures fall
  * outside the specified range, as long as at least part of the match
  * overlaps with the range.
@@ -1079,6 +1104,9 @@ void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit);
  * than the specified range, but part of that node intersects with the range,
  * the entire match will be returned.
  *
+ * NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range
+ * unbounded.
+ *
  * This will return `false` if the start byte is greater than the end byte, otherwise
  * it will return `true`.
  */
@@ -1096,6 +1124,9 @@ bool ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, ui
  * than the specified range, but part of that node intersects with the range,
  * the entire match will be returned.
  *
+ * NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the
+ * range unbounded.
+ *
  * This will return `false` if the start point is greater than the end point, otherwise
  * it will return `true`.
  */
@@ -1109,6 +1140,9 @@ bool ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, T
  * matches where _all_ nodes are _fully_ contained within the given range. Both functions
  * can be used together, e.g. to search for any matches that intersect line 5000, as
  * long as they are fully contained within lines 4500-5500
+ *
+ * NOTE: An `end_byte` of zero is interpreted as `UINT32_MAX`, making the range
+ * unbounded.
  */
 bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
 
@@ -1120,6 +1154,9 @@ bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t sta
  * matches where _all_ nodes are _fully_ contained within the given range. Both functions
  * can be used together, e.g. to search for any matches that intersect line 5000, as
  * long as they are fully contained within lines 4500-5500
+ *
+ * NOTE: An `end_point` of `(0, 0)` is interpreted as `POINT_MAX`, making the
+ * range unbounded.
  */
 bool ts_query_cursor_set_containing_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
 

lib/src/alloc.h

@@ -9,7 +9,7 @@ extern "C" {
 #include <stdio.h>
 #include <stdlib.h>
 
-#if defined(TREE_SITTER_HIDDEN_SYMBOLS) || defined(_WIN32)
+#if defined(TREE_SITTER_HIDE_SYMBOLS) || defined(_WIN32)
 #define TS_PUBLIC
 #else
 #define TS_PUBLIC __attribute__((visibility("default")))

lib/src/array.h

@@ -50,69 +50,104 @@ extern "C" {
 /// memory allocated for the array's contents.
 #define array_clear(self) ((self)->size = 0)
 
+#ifdef __cplusplus
+#define _array__cast(self, expr) (decltype((self)->contents))(expr)
+#else
+#define _array__cast(self, expr) (expr)
+#endif
+
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity) \
-  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+#define array_reserve(self, new_capacity)                 \
+  ((self)->contents = _array__cast(self, _array__reserve( \
+    (void *)(self)->contents, &(self)->capacity,          \
+    array_elem_size(self), new_capacity))                 \
+  )
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self) _array__delete((Array *)(self))
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                            \
-  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
-   (self)->contents[(self)->size++] = (element))
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__cast(self, _array__grow(           \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    ));                                                           \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count) \
-  do { \
-    if ((count) == 0) break; \
-    _array__grow((Array *)(self), count, array_elem_size(self)); \
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__cast(self, _array__grow(                          \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    ));                                                                          \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count); \
+    (self)->size += (count);                                                     \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other)                                       \
+#define array_push_all(self, other) \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, contents)                    \
-  _array__splice(                                               \
-    (Array *)(self), array_elem_size(self), (self)->size, \
-    0, count,  contents                                        \
-  )
+#define array_extend(self, count, other_contents)                 \
+  ((self)->contents = _array__cast(self, _array__splice(          \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
+  )))
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents)  \
-  _array__splice(                                                       \
-    (Array *)(self), array_elem_size(self), _index,                \
-    old_count, new_count, new_contents                                 \
-  )
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  ((self)->contents = _array__cast(self, _array__splice(              \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
+  )))
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element) \
-  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+#define array_insert(self, _index, element)                     \
+  ((self)->contents = _array__cast(self, _array__splice(        \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )))
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((Array *)(self), array_elem_size(self), _index)
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other) \
-  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+#define array_assign(self, other)                                   \
+  ((self)->contents = _array__cast(self, _array__assign(            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )))
 
 /// Swap one array with another
-#define array_swap(self, other) \
-  _array__swap((Array *)(self), (Array *)(other))
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array__cast(other, _array_swap_tmp);       \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
+  } while (0)
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +192,90 @@ extern "C" {
 
 // Private
 
-typedef Array(void) Array;
-
-/// This is not what you're looking for, see `array_delete`.
-static inline void _array__delete(Array *self) {
-  if (self->contents) {
-    ts_free(self->contents);
-    self->contents = NULL;
-    self->size = 0;
-    self->capacity = 0;
-  }
-}
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(Array *self, size_t element_size,
-                                uint32_t index) {
-  ts_assert(index < self->size);
-  char *contents = (char *)self->contents;
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  ts_assert(index < *size);
+  char *contents = (char *)self_contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (self->size - index - 1) * element_size);
-  self->size--;
+          (*size - index - 1) * element_size);
+  (*size)--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
-  if (new_capacity > self->capacity) {
-    if (self->contents) {
-      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
     } else {
-      self->contents = ts_malloc(new_capacity * element_size);
+      new_contents = ts_malloc(new_capacity * element_size);
     }
-    self->capacity = new_capacity;
+    *capacity = new_capacity;
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
-  _array__reserve(self, element_size, other->size);
-  self->size = other->size;
-  memcpy(self->contents, other->contents, self->size * element_size);
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(Array *self, Array *other) {
-  Array swap = *other;
-  *other = *self;
-  *self = swap;
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
-  uint32_t new_size = self->size + count;
-  if (new_size > self->capacity) {
-    uint32_t new_capacity = self->capacity * 2;
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    _array__reserve(self, element_size, new_capacity);
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
   }
+  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void _array__splice(Array *self, size_t element_size,
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t new_size = *size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  ts_assert(old_end <= self->size);
+  ts_assert(old_end <= *size);
 
-  _array__reserve(self, element_size, new_size);
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
 
-  char *contents = (char *)self->contents;
-  if (self->size > old_end) {
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (self->size - old_end) * element_size
+      (*size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -250,7 +293,9 @@ static inline void _array__splice(Array *self, size_t element_size,
       );
     }
   }
-  self->size += new_count - old_count;
+  *size += new_count - old_count;
+
+  return new_contents;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

lib/src/parser.c

@@ -795,7 +795,13 @@ static Subtree ts_parser__reuse_node(
       reason = "is_missing";
     } else if (ts_subtree_is_fragile(result)) {
       reason = "is_fragile";
-    } else if (ts_parser__has_included_range_difference(self, byte_offset, end_byte_offset)) {
+    } else if (ts_parser__has_included_range_difference(
+                 self,
+                 byte_offset,
+                 ts_subtree_is_eof(result)
+                   ? end_byte_offset
+                   : end_byte_offset + ts_subtree_lookahead_bytes(result)
+               )) {
       reason = "contains_different_included_range";
     }
 
@@ -1219,10 +1225,17 @@ static bool ts_parser__recover_to_state(
       Subtree error_tree = *array_get(&error_trees, 0);
       uint32_t error_child_count = ts_subtree_child_count(error_tree);
       if (error_child_count > 0) {
-        array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree));
+        SubtreeArray nested = array_new();
+        array_reserve(&nested, error_child_count);
         for (unsigned j = 0; j < error_child_count; j++) {
-          ts_subtree_retain(*array_get(&slice.subtrees, j));
+          Subtree child = ts_subtree_children(error_tree)[j];
+          ts_subtree_retain(child);
+          array_push(&nested, child);
         }
+        Subtree nested_error = ts_subtree_from_mut(ts_subtree_new_node(
+          ts_builtin_sym_error_repeat, &nested, 0, self->language
+        ));
+        array_insert(&slice.subtrees, 0, nested_error);
       }
       ts_subtree_array_delete(&self->tree_pool, &error_trees);
     }
@@ -1741,6 +1754,13 @@ static bool ts_parser__advance(
       }
     }
 
+    // If the current lookahead token is not valid and the parser is
+    // already in the error state, restart the error recovery process.
+    if (state == ERROR_STATE) {
+      ts_parser__recover(self, version, lookahead);
+      return true;
+    }
+
     // If the current lookahead token is not valid and the previous subtree on
     // the stack was reused from an old tree, then it wasn't actually valid to
     // reuse that previous subtree. Remove it from the stack, and in its place,
@@ -2179,7 +2199,7 @@ balance:
   ts_assert(self->finished_tree.ptr);
   if (!ts_parser__balance_subtree(self)) {
     self->canceled_balancing = true;
-    return false;
+    return NULL;
   }
   self->canceled_balancing = false;
   LOG("done");

lib/src/portable/endian.h

@@ -20,6 +20,7 @@
     defined(__GNU__) || \
     defined(__HAIKU__) || \
     defined(__illumos__) || \
+    defined(__redox__) || \
     defined(__NetBSD__) || \
     defined(__OpenBSD__) || \
     defined(__CYGWIN__) || \

lib/src/query.c

@@ -18,6 +18,11 @@
 
 // #define DEBUG_ANALYZE_QUERY
 // #define DEBUG_EXECUTE_QUERY
+// #define DEBUG_QUERY_STEPS
+
+#if defined(DEBUG_QUERY_STEPS) || defined(DEBUG_ANALYZE_QUERY) || defined(DEBUG_EXECUTE_QUERY)
+#define DEBUG_DUMP_STEPS
+#endif
 
 #define MAX_STEP_CAPTURE_COUNT 3
 #define MAX_NEGATED_FIELD_COUNT 8
@@ -56,15 +61,15 @@ typedef struct {
  * Steps have some additional fields in order to handle the `.` (or "anchor") operator,
  * which forbids additional child nodes:
  * - `is_immediate` - Indicates that the node matching this step cannot be preceded
- *   by other sibling nodes that weren't specified in the pattern.
+ *    by other sibling nodes that weren't specified in the pattern.
  * - `is_last_child` - Indicates that the node matching this step cannot have any
- *   subsequent named siblings.
+ *    subsequent named siblings.
  *
  * For simple patterns, steps are matched in sequential order. But in order to
  * handle alternative/repeated/optional sub-patterns, query steps are not always
  * structured as a linear sequence; they sometimes need to split and merge. This
  * is done using the following fields:
- *  - `alternative_index` - The index of a different query step that serves as
+ * - `alternative_index` - The index of a different query step that serves as
  *    an alternative to this step. A `NONE` value represents no alternative.
  *    When a query state reaches a step with an alternative index, the state
  *    is duplicated, with one copy remaining at the original step, and one copy
@@ -75,21 +80,26 @@ typedef struct {
  * - `is_pass_through` - Indicates that state has no matching logic of its own,
  *    and exists only to split a state. One copy of the state advances immediately
  *    to the next step, and one moves to the alternative step.
- * - `alternative_is_immediate` - Indicates that this step's alternative step
- *    should be treated as if `is_immediate` is true.
+ * - `alternative_is_skip` - Indicates that this step's `alternative_index` is the
+ *    forward skip introduced by a `?` or `*` quantifier (the branch taken when the
+ *    quantifier matches zero occurrences). For a state that follows it, an
+ *    immediately-following anchor is vacuous.
+ * - `is_inside_alternation` - Indicates that state is inside an alternation.
+ *    Currently only written to quantifier steps, read by logic that maintains
+ *    correctness for quantifiers inside alternations.
  *
  * Steps also store some derived state that summarizes how they relate to other
  * steps within the same pattern. This is used to optimize the matching process:
- *  - `contains_captures` - Indicates that this step or one of its child steps
- *     has a non-empty `capture_ids` list.
- *  - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
- *     it and all of its subsequent sibling steps within the same parent pattern
- *     are guaranteed to match.
- *  - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
- *     for the entire top-level pattern. When iterating through a query's
- *     captures using `ts_query_cursor_next_capture`, this field is used to
- *     detect that a capture can safely be returned from a match that has not
- *     even completed yet.
+ * - `contains_captures` - Indicates that this step or one of its child steps
+ *    has a non-empty `capture_ids` list.
+ * - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
+ *    it and all of its subsequent sibling steps within the same parent pattern
+ *    are guaranteed to match.
+ * - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
+ *    for the entire top-level pattern. When iterating through a query's
+ *    captures using `ts_query_cursor_next_capture`, this field is used to
+ *    detect that a capture can safely be returned from a match that has not
+ *    even completed yet.
  */
 typedef struct {
   TSSymbol symbol;
@@ -104,11 +114,12 @@ typedef struct {
   bool is_last_child: 1;
   bool is_pass_through: 1;
   bool is_dead_end: 1;
-  bool alternative_is_immediate: 1;
+  bool is_inside_alternation: 1;
   bool contains_captures: 1;
   bool root_pattern_guaranteed: 1;
   bool parent_pattern_guaranteed: 1;
   bool is_missing: 1;
+  bool alternative_is_skip: 1;
 } QueryStep;
 
 /*
@@ -180,6 +191,8 @@ typedef struct {
  *    have already been returned.
  * - `capture_list_id` - A numeric id that can be used to retrieve the state's
  *    list of captures from the `CaptureListPool`.
+ * - `heap_insert_order` - A sequence number used to preserve discovery order
+ *    among finished states with the same capture position and pattern.
  * - `seeking_immediate_match` - A flag that indicates that the state's next
  *    step must be matched by the very next sibling. This is used when
  *    processing repetitions, or when processing a wildcard node followed by
@@ -193,6 +206,7 @@ typedef struct {
 typedef struct {
   uint32_t id;
   uint32_t capture_list_id;
+  uint32_t heap_insert_order;
   uint16_t start_depth;
   uint16_t step_index;
   uint16_t pattern_index;
@@ -201,8 +215,10 @@ typedef struct {
   bool has_in_progress_alternatives: 1;
   bool dead: 1;
   bool needs_parent: 1;
+  bool skipped_quantifier: 1;
 } QueryState;
 
+typedef Array(QueryState) QueryStateList;
 typedef Array(TSQueryCapture) CaptureList;
 
 /*
@@ -313,14 +329,19 @@ struct TSQuery {
 struct TSQueryCursor {
   const TSQuery *query;
   TSTreeCursor cursor;
-  Array(QueryState) states;
-  Array(QueryState) finished_states;
+  QueryStateList states;
+  QueryStateList finished_states;
+  // Tracks how much of finished_states is in heap order. Elements at indices
+  // < this value satisfy the min-heap property; elements >= this value are
+  // newly pushed and need to be sifted into place. Only used by `next_capture`.
+  uint32_t finished_states_heap_size;
   CaptureListPool capture_list_pool;
   uint32_t depth;
   uint32_t max_start_depth;
   TSRange included_range;
   TSRange containing_range;
   uint32_t next_state_id;
+  uint32_t next_finished_state_id;
   const TSQueryCursorOptions *query_options;
   TSQueryCursorState query_state;
   unsigned operation_count;
@@ -333,6 +354,7 @@ struct TSQueryCursor {
 static const TSQueryError PARENT_DONE = -1;
 static const uint16_t PATTERN_DONE_MARKER = UINT16_MAX;
 static const uint16_t NONE = UINT16_MAX;
+static const uint32_t CAPTURE_LIST_NONE = UINT32_MAX;
 static const TSSymbol WILDCARD_SYMBOL = 0;
 static const unsigned OP_COUNT_PER_QUERY_CALLBACK_CHECK = 100;
 
@@ -428,7 +450,7 @@ static CaptureListPool capture_list_pool_new(void) {
 }
 
 static void capture_list_pool_reset(CaptureListPool *self) {
-  for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
+  for (uint32_t i = 0; i < self->list.size; i++) {
     // This invalid size means that the list is not in use.
     array_get(&self->list, i)->size = UINT32_MAX;
   }
@@ -436,18 +458,18 @@ static void capture_list_pool_reset(CaptureListPool *self) {
 }
 
 static void capture_list_pool_delete(CaptureListPool *self) {
-  for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
+  for (uint32_t i = 0; i < self->list.size; i++) {
     array_delete(array_get(&self->list, i));
   }
   array_delete(&self->list);
 }
 
-static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) {
+static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint32_t id) {
   if (id >= self->list.size) return &self->empty_list;
   return array_get(&self->list, id);
 }
 
-static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) {
+static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint32_t id) {
   ts_assert(id < self->list.size);
   return array_get(&self->list, id);
 }
@@ -458,10 +480,10 @@ static bool capture_list_pool_is_empty(const CaptureListPool *self) {
   return self->free_capture_list_count == 0 && self->list.size >= self->max_capture_list_count;
 }
 
-static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
+static uint32_t capture_list_pool_acquire(CaptureListPool *self) {
   // First see if any already allocated capture list is currently unused.
   if (self->free_capture_list_count > 0) {
-    for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) {
+    for (uint32_t i = 0; i < self->list.size; i++) {
       if (array_get(&self->list, i)->size == UINT32_MAX) {
         array_clear(array_get(&self->list, i));
         self->free_capture_list_count--;
@@ -474,7 +496,7 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
   // doesn't put us over the requested maximum.
   uint32_t i = self->list.size;
   if (i >= self->max_capture_list_count) {
-    return NONE;
+    return CAPTURE_LIST_NONE;
   }
   CaptureList list;
   array_init(&list);
@@ -482,12 +504,146 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) {
   return i;
 }
 
-static void capture_list_pool_release(CaptureListPool *self, uint16_t id) {
+static void capture_list_pool_release(CaptureListPool *self, uint32_t id) {
   if (id >= self->list.size) return;
   array_get(&self->list, id)->size = UINT32_MAX;
   self->free_capture_list_count++;
 }
 
+/********************
+ * FinishedStateHeap
+ *
+ * A min-heap of finished query states, ordered by (byte offset of next
+ * unconsumed capture, pattern_index, insertion order). This allows
+ * ts_query_cursor_next_capture to find the earliest capture in O(1) instead
+ * of scanning all finished states. The heap is maintained lazily -
+ * ts_query_cursor__advance uses plain array_push, and next_capture sifts
+ * new elements into place via a tracked heap_size boundary.
+ ********************/
+
+static void finished_state_swap(QueryStateList *states, uint32_t a, uint32_t b) {
+  QueryState tmp = *array_get(states, a);
+  *array_get(states, a) = *array_get(states, b);
+  *array_get(states, b) = tmp;
+}
+
+// Compare two finished states by (byte offset of next unconsumed capture,
+// pattern_index, insertion order).
+static inline bool finished_state_precedes(
+  const QueryState *a,
+  const QueryState *b,
+  const CaptureListPool *pool
+) {
+  const CaptureList *a_caps = capture_list_pool_get(pool, a->capture_list_id);
+  const CaptureList *b_caps = capture_list_pool_get(pool, b->capture_list_id);
+  if (a->consumed_capture_count >= a_caps->size) return false;
+  if (b->consumed_capture_count >= b_caps->size) return true;
+  uint32_t a_byte = ts_node_start_byte(a_caps->contents[a->consumed_capture_count].node);
+  uint32_t b_byte = ts_node_start_byte(b_caps->contents[b->consumed_capture_count].node);
+  if (a_byte != b_byte) return a_byte < b_byte;
+  if (a->pattern_index != b->pattern_index) return a->pattern_index < b->pattern_index;
+  return a->heap_insert_order < b->heap_insert_order;
+}
+
+static void finished_state_sift_down(
+  QueryStateList *states,
+  uint32_t index,
+  const CaptureListPool *pool
+) {
+  uint32_t size = states->size;
+  while (true) {
+    uint32_t smallest = index;
+    uint32_t left = 2 * index + 1;
+    uint32_t right = 2 * index + 2;
+    if (left < size && finished_state_precedes(
+      array_get(states, left),
+      array_get(states, smallest),
+      pool
+    )) {
+      smallest = left;
+    }
+    if (right < size && finished_state_precedes(
+      array_get(states, right),
+      array_get(states, smallest),
+      pool
+    )) {
+      smallest = right;
+    }
+    if (smallest == index) break;
+    finished_state_swap(states, index, smallest);
+    index = smallest;
+  }
+}
+
+static void finished_state_sift_up(
+  QueryStateList *states,
+  uint32_t index,
+  const CaptureListPool *pool
+) {
+  while (index > 0) {
+    uint32_t parent = (index - 1) / 2;
+    if (finished_state_precedes(
+      array_get(states, index),
+      array_get(states, parent),
+      pool
+    )) {
+      finished_state_swap(states, index, parent);
+      index = parent;
+    } else {
+      break;
+    }
+  }
+}
+
+static inline void finished_state_pop(QueryStateList *states, const CaptureListPool *pool) {
+  if (states->size > 1) *array_front(states) = *array_back(states);
+  states->size--;
+  if (states->size > 0) finished_state_sift_down(states, 0, pool);
+}
+
+// Remove an element at an arbitrary index and restore heap order.
+static void finished_state_erase(
+  QueryStateList *states,
+  uint32_t index,
+  const CaptureListPool *pool
+) {
+  if (index == states->size - 1) {
+    states->size--;
+    return;
+  }
+  *array_get(states, index) = *array_back(states);
+  states->size--;
+  // The replacement element may need to go up or down.
+  if (index > 0 && finished_state_precedes(
+    array_get(states, index),
+    array_get(states, (index - 1) / 2),
+    pool
+  )) {
+    finished_state_sift_up(states, index, pool);
+  } else {
+    finished_state_sift_down(states, index, pool);
+  }
+}
+
+static void ts_query_cursor__push_finished_state(
+  TSQueryCursor *self,
+  QueryState *state
+) {
+  state->heap_insert_order = self->next_finished_state_id++;
+  array_push(&self->finished_states, *state);
+}
+
+static void ts_query_cursor__heapify_finished_states(TSQueryCursor *self) {
+  while (self->finished_states_heap_size < self->finished_states.size) {
+    finished_state_sift_up(
+      &self->finished_states,
+      self->finished_states_heap_size,
+      &self->capture_list_pool
+    );
+    self->finished_states_heap_size++;
+  }
+}
+
 /**************
  * Quantifiers
  **************/
@@ -816,17 +972,8 @@ static QueryStep query_step__new(
   QueryStep step = {
     .symbol = symbol,
     .depth = depth,
-    .field = 0,
     .alternative_index = NONE,
-    .negated_field_list_id = 0,
-    .contains_captures = false,
-    .is_last_child = false,
-    .is_named = false,
-    .is_pass_through = false,
-    .is_dead_end = false,
-    .root_pattern_guaranteed = false,
     .is_immediate = is_immediate,
-    .alternative_is_immediate = false,
   };
   for (unsigned i = 0; i < MAX_STEP_CAPTURE_COUNT; i++) {
     step.capture_ids[i] = NONE;
@@ -1465,6 +1612,44 @@ static void ts_query__perform_analysis(
   }
 }
 
+#ifdef DEBUG_DUMP_STEPS
+static void ts_query__dump_steps(const TSQuery *self, const char *label) {
+  printf("=== STEPS (%s) ===\n", label);
+  for (unsigned i = 0; i < self->steps.size; i++) {
+    const QueryStep *s = array_get(&self->steps, i);
+    if (s->depth == PATTERN_DONE_MARKER) {
+      printf("%3u: DONE\n", i);
+      continue;
+    }
+    printf("%3u: depth=%u sym=%s", i, s->depth,
+      s->symbol == WILDCARD_SYMBOL ? "_" : ts_language_symbol_name(self->language, s->symbol));
+    if (s->supertype_symbol) {
+      printf(" super=%s", ts_language_symbol_name(self->language, s->supertype_symbol));
+    }
+    if (s->field) {
+      printf(" field=%s", ts_language_field_name_for_id(self->language, s->field));
+    }
+    if (s->alternative_index != NONE) printf(" alt=%u", s->alternative_index);
+    if (s->is_immediate) printf(" IMM");
+    if (s->is_pass_through) printf(" PASS");
+    if (s->is_dead_end) printf(" DEAD");
+    if (s->is_last_child) printf(" LAST");
+    if (s->is_named) printf(" NAMED");
+    if (s->is_missing) printf(" MISSING");
+    if (s->is_inside_alternation) printf(" INALT");
+    if (s->contains_captures) printf(" HASCAP");
+    if (s->parent_pattern_guaranteed) printf(" PPG");
+    if (s->root_pattern_guaranteed) printf(" RPG");
+    for (unsigned c = 0; c < MAX_STEP_CAPTURE_COUNT && s->capture_ids[c] != NONE; c++) {
+      uint32_t cap_len;
+      const char *cap_name = symbol_table_name_for_id(&self->captures, s->capture_ids[c], &cap_len);
+      printf(" @%.*s", (int)cap_len, cap_name);
+    }
+    printf("\n");
+  }
+}
+#endif
+
 static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   Array(uint16_t) non_rooted_pattern_start_steps = array_new();
   for (unsigned i = 0; i < self->pattern_map.size; i++) {
@@ -1902,25 +2087,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) {
   }
 
   #ifdef DEBUG_ANALYZE_QUERY
-    printf("Steps:\n");
-    for (unsigned i = 0; i < self->steps.size; i++) {
-      QueryStep *step = array_get(&self->steps, i);
-      if (step->depth == PATTERN_DONE_MARKER) {
-        printf("  %u: DONE\n", i);
-      } else {
-        printf(
-          "  %u: {symbol: %s, field: %s, depth: %u, parent_pattern_guaranteed: %d, root_pattern_guaranteed: %d}\n",
-          i,
-          (step->symbol == WILDCARD_SYMBOL)
-            ? "ANY"
-            : ts_language_symbol_name(self->language, step->symbol),
-          (step->field ? ts_language_field_name_for_id(self->language, step->field) : "-"),
-          step->depth,
-          step->parent_pattern_guaranteed,
-          step->root_pattern_guaranteed
-        );
-      }
-    }
+    ts_query__dump_steps(self, "analysis");
   #endif
 
   // Determine which repetition symbols in this language have the possibility
@@ -2231,6 +2398,7 @@ static TSQueryError ts_query__parse_pattern(
   Stream *stream,
   uint32_t depth,
   bool is_immediate,
+  bool is_inside_alternation,
   CaptureQuantifiers *capture_quantifiers
 ) {
   if (stream->next == 0) return TSQueryErrorSyntax;
@@ -2264,6 +2432,7 @@ static TSQueryError ts_query__parse_pattern(
         stream,
         depth,
         is_immediate,
+        true,
         &branch_capture_quantifiers
       );
 
@@ -2322,15 +2491,24 @@ static TSQueryError ts_query__parse_pattern(
       CaptureQuantifiers child_capture_quantifiers = capture_quantifiers_new();
       for (;;) {
         if (stream->next == '.') {
+          const char *anchor_start = stream->input;
           child_is_immediate = true;
           stream_advance(stream);
           stream_skip_whitespace(stream);
+          // A `.` at a group's end has no sibling to anchor, and a group is not a
+          // node, so there is no last child to anchor against.
+          if (stream->next == ')') {
+            stream_reset(stream, anchor_start);
+            capture_quantifiers_delete(&child_capture_quantifiers);
+            return TSQueryErrorSyntax;
+          }
         }
         TSQueryError e = ts_query__parse_pattern(
           self,
           stream,
           depth,
           child_is_immediate,
+          is_inside_alternation,
           &child_capture_quantifiers
         );
         if (e == PARENT_DONE) {
@@ -2373,7 +2551,7 @@ static TSQueryError ts_query__parse_pattern(
         // Parse the wildcard symbol
         if (length == 1 && node_name[0] == '_') {
           symbol = WILDCARD_SYMBOL;
-        } else if (!strncmp(node_name, "MISSING", length)) {
+        } else if (length == 7 && !strncmp(node_name, "MISSING", length)) {
           is_missing = true;
           stream_skip_whitespace(stream);
 
@@ -2569,6 +2747,7 @@ static TSQueryError ts_query__parse_pattern(
           stream,
           depth + 1,
           child_is_immediate,
+          is_inside_alternation,
           &child_capture_quantifiers
         );
         // In the event we only parsed a predicate, meaning no new steps were added,
@@ -2680,6 +2859,7 @@ static TSQueryError ts_query__parse_pattern(
       stream,
       depth,
       is_immediate,
+      is_inside_alternation,
       &field_capture_quantifiers
     );
     if (e) {
@@ -2798,16 +2978,16 @@ static TSQueryError ts_query__parse_pattern(
   switch (quantifier) {
     case TSQuantifierOneOrMore:
       repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false);
+      repeat_step.is_inside_alternation = is_inside_alternation;
       repeat_step.alternative_index = starting_step_index;
       repeat_step.is_pass_through = true;
-      repeat_step.alternative_is_immediate = true;
       array_push(&self->steps, repeat_step);
       break;
     case TSQuantifierZeroOrMore:
       repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false);
+      repeat_step.is_inside_alternation = is_inside_alternation;
       repeat_step.alternative_index = starting_step_index;
       repeat_step.is_pass_through = true;
-      repeat_step.alternative_is_immediate = true;
       array_push(&self->steps, repeat_step);
 
       // Stop when `step->alternative_index` is `NONE` or it points to
@@ -2818,6 +2998,7 @@ static TSQueryError ts_query__parse_pattern(
         step = array_get(&self->steps, step->alternative_index);
       }
       step->alternative_index = self->steps.size;
+      step->alternative_is_skip = true;
       break;
     case TSQuantifierZeroOrOne:
       step = array_get(&self->steps, starting_step_index);
@@ -2825,6 +3006,7 @@ static TSQueryError ts_query__parse_pattern(
         step = array_get(&self->steps, step->alternative_index);
       }
       step->alternative_index = self->steps.size;
+      step->alternative_is_skip = true;
       break;
     default:
       break;
@@ -2884,7 +3066,7 @@ TSQuery *ts_query_new(
       .is_non_local = false,
     }));
     CaptureQuantifiers capture_quantifiers = capture_quantifiers_new();
-    *error_type = ts_query__parse_pattern(self, &stream, 0, false, &capture_quantifiers);
+    *error_type = ts_query__parse_pattern(self, &stream, 0, false, false, &capture_quantifiers);
     array_push(&self->steps, query_step__new(0, PATTERN_DONE_MARKER, false));
 
     QueryPattern *pattern = array_back(&self->patterns);
@@ -2958,14 +3140,69 @@ TSQuery *ts_query_new(
         break;
       }
     }
+
+    // Fix up quantifier loop-backs within alternations. When a branch of an
+    // alternation has a + or * quantifier, the quantifier's pass_through step
+    // loops back to the branch's first step. However, the alternation linking
+    // assigns that same step's `alternative_index` to point to the _next_ branch.
+    // This causes the quantifier loop to incorrectly explore other alternation branches,
+    // when a quantified branch matches, loops back, and then fails to match. To correct
+    // this, we create "clean" copies of the branches' first steps without the link to the
+    // next branch. After a quantified branch matches, it loops back to the cleaned copy.
+    {
+      uint32_t pat_start = pattern->steps.offset;
+      uint32_t pat_end = pat_start + pattern->steps.length - 1; // exclude DONE
+
+      for (uint32_t i = pat_start; i < pat_end; i++) {
+        QueryStep *s = array_get(&self->steps, i);
+        // Ensure this step is a pass_through with a _backward_ alternative (a quantifier loop-back)
+        if (!s->is_pass_through || !s->is_inside_alternation
+            || s->alternative_index == NONE || s->alternative_index >= i) continue;
+
+        uint32_t target_idx = s->alternative_index;
+        QueryStep *target = array_get(&self->steps, target_idx);
+
+        // Check if the target has a forward alternative from alternation linking
+        uint16_t target_alt_index = target->alternative_index;
+        if (target_alt_index == NONE
+            || target_alt_index <= target_idx || target_alt_index >= pat_end) continue;
+
+        // Create a clean copy of the target step without the alternation alternative.
+        uint32_t copy_idx = self->steps.size;
+        QueryStep copy = *target;
+        copy.alternative_index = NONE;
+        uint16_t target_depth = target->depth;
+        array_push(&self->steps, copy);
+
+        // Add a dead_end that redirects to the pass through step after the target,
+        // so the pattern continues correctly after the cleaned copy matches.
+        QueryStep redirect = query_step__new(0, target_depth, false);
+        redirect.is_dead_end = true;
+        redirect.alternative_index = target_idx + 1;
+        array_push(&self->steps, redirect);
+
+        // Update the pass_through to loop back to the copy. Reacquire `s` since
+        // `self->steps` may have been reallocated.
+        s = array_get(&self->steps, i);
+        s->alternative_index = copy_idx;
+      }
+    }
   }
 
+  #ifdef DEBUG_DUMP_STEPS
+    ts_query__dump_steps(self, "post-parse");
+  #endif
+
   if (!ts_query__analyze_patterns(self, error_offset)) {
     *error_type = TSQueryErrorStructure;
     ts_query_delete(self);
     return NULL;
   }
 
+  #ifdef DEBUG_DUMP_STEPS
+    ts_query__dump_steps(self, "post-analysis");
+  #endif
+
   array_delete(&self->string_buffer);
   return self;
 }
@@ -3099,12 +3336,18 @@ bool ts_query__step_is_fallible(
   const TSQuery *self,
   uint16_t step_index
 ) {
-  ts_assert((uint32_t)step_index + 1 < self->steps.size);
+  unsigned i = 1;
   QueryStep *step = array_get(&self->steps, step_index);
-  QueryStep *next_step = array_get(&self->steps, step_index + 1);
+  QueryStep *next_step;
+  do {
+    ts_assert((uint32_t)step_index + i < self->steps.size);
+    next_step = array_get(&self->steps, step_index + i);
+    i++;
+  } while (next_step->is_pass_through);
   return (
     next_step->depth != PATTERN_DONE_MARKER &&
-    next_step->depth > step->depth &&
+    (next_step->depth > step->depth ||
+        (next_step->depth == step->depth && next_step->is_immediate)) &&
     (!next_step->parent_pattern_guaranteed || step->symbol == WILDCARD_SYMBOL)
   );
 }
@@ -3232,10 +3475,12 @@ void ts_query_cursor_exec(
 
   array_clear(&self->states);
   array_clear(&self->finished_states);
+  self->finished_states_heap_size = 0;
   ts_tree_cursor_reset(&self->cursor, node);
   capture_list_pool_reset(&self->capture_list_pool);
   self->on_visible_node = true;
   self->next_state_id = 0;
+  self->next_finished_state_id = 0;
   self->depth = 0;
   self->ascending = false;
   self->halted = false;
@@ -3458,6 +3703,46 @@ void ts_query_cursor__compare_captures(
   }
 }
 
+// Order two in-progress states for the longest-match dedup pass. Within a
+// (start_depth, pattern_index) group, states with no captures sort first, as they are a
+// subset of every other state (so the dedup pass must always compare them). The rest
+// sort by the start byte of their first capture.
+static bool ts_query_cursor__state_precedes(
+  const TSQueryCursor *self,
+  const QueryState *a,
+  const QueryState *b
+) {
+  if (a->start_depth != b->start_depth) return a->start_depth < b->start_depth;
+  if (a->pattern_index != b->pattern_index) return a->pattern_index < b->pattern_index;
+  const CaptureList *a_caps = capture_list_pool_get(&self->capture_list_pool, a->capture_list_id);
+  const CaptureList *b_caps = capture_list_pool_get(&self->capture_list_pool, b->capture_list_id);
+  if ((a_caps->size == 0) != (b_caps->size == 0)) return a_caps->size == 0;
+  if (a_caps->size == 0) return false;
+  return
+    ts_node_start_byte(array_get(a_caps, 0)->node) <
+    ts_node_start_byte(array_get(b_caps, 0)->node);
+}
+
+// Stable-sort the in-progress states with the order dictated by `ts_query_cursor__state_precedes`.
+// This runs once per node, right before the dedup pass.
+static void ts_query_cursor__sort_states_by_capture(TSQueryCursor *self) {
+  QueryStateList *states = &self->states;
+  for (uint32_t i = 1; i < states->size; i++) {
+    // Fast+common path: this state is already ordered after its predecessor, so it does not need
+    // to move.
+    if (!ts_query_cursor__state_precedes(
+      self, array_get(states, i), array_get(states, i - 1)
+    )) continue;
+    QueryState key = *array_get(states, i);
+    uint32_t j = i;
+    do {
+      *array_get(states, j) = *array_get(states, j - 1);
+      j--;
+    } while (j > 0 && ts_query_cursor__state_precedes(self, &key, array_get(states, j - 1)));
+    *array_get(states, j) = key;
+  }
+}
+
 static void ts_query_cursor__add_state(
   TSQueryCursor *self,
   const PatternEntry *pattern
@@ -3507,7 +3792,8 @@ static void ts_query_cursor__add_state(
   );
   array_insert(&self->states, index, ((QueryState) {
     .id = UINT32_MAX,
-    .capture_list_id = NONE,
+    .capture_list_id = CAPTURE_LIST_NONE,
+    .heap_insert_order = UINT32_MAX,
     .step_index = pattern->step_index,
     .pattern_index = pattern->pattern_index,
     .start_depth = start_depth,
@@ -3516,6 +3802,7 @@ static void ts_query_cursor__add_state(
     .has_in_progress_alternatives = false,
     .needs_parent = step->depth == 1,
     .dead = false,
+    .skipped_quantifier = false,
   }));
 }
 
@@ -3527,13 +3814,13 @@ static CaptureList *ts_query_cursor__prepare_to_capture(
   QueryState *state,
   unsigned state_index_to_preserve
 ) {
-  if (state->capture_list_id == NONE) {
+  if (state->capture_list_id == CAPTURE_LIST_NONE) {
     state->capture_list_id = capture_list_pool_acquire(&self->capture_list_pool);
 
     // If there are no capture lists left in the pool, then terminate whichever
     // state has captured the earliest node in the document, and steal its
     // capture list.
-    if (state->capture_list_id == NONE) {
+    if (state->capture_list_id == CAPTURE_LIST_NONE) {
       self->did_exceed_match_limit = true;
       uint32_t state_index, byte_offset, pattern_index;
       if (
@@ -3552,7 +3839,7 @@ static CaptureList *ts_query_cursor__prepare_to_capture(
         );
         QueryState *other_state = array_get(&self->states, state_index);
         state->capture_list_id = other_state->capture_list_id;
-        other_state->capture_list_id = NONE;
+        other_state->capture_list_id = CAPTURE_LIST_NONE;
         other_state->dead = true;
         CaptureList *list = capture_list_pool_get_mut(
           &self->capture_list_pool,
@@ -3606,10 +3893,10 @@ static QueryState *ts_query_cursor__copy_state(
   const QueryState *state = *state_ref;
   uint32_t state_index = (uint32_t)(state - self->states.contents);
   QueryState copy = *state;
-  copy.capture_list_id = NONE;
+  copy.capture_list_id = CAPTURE_LIST_NONE;
 
   // If the state has captures, copy its capture list.
-  if (state->capture_list_id != NONE) {
+  if (state->capture_list_id != CAPTURE_LIST_NONE) {
     CaptureList *new_captures = ts_query_cursor__prepare_to_capture(self, &copy, state_index);
     if (!new_captures) return NULL;
     const CaptureList *old_captures = capture_list_pool_get(
@@ -3765,7 +4052,7 @@ static inline bool ts_query_cursor__advance(
             (state->start_depth > self->depth || self->depth == 0)
           ) {
             LOG("  finish pattern %u\n", state->pattern_index);
-            array_push(&self->finished_states, *state);
+            ts_query_cursor__push_finished_state(self, state);
             did_match = true;
             deleted_count++;
           }
@@ -3956,7 +4243,7 @@ static inline bool ts_query_cursor__advance(
             node_does_match = symbol == step->symbol && (!step->is_missing || is_missing);
           }
           bool later_sibling_can_match = has_later_siblings;
-          if ((step->is_immediate && is_named) || state->seeking_immediate_match) {
+          if ((step->is_immediate && is_named && !state->skipped_quantifier) || state->seeking_immediate_match) {
             later_sibling_can_match = false;
           }
           if (step->is_last_child && has_later_named_siblings) {
@@ -4099,6 +4386,9 @@ static inline bool ts_query_cursor__advance(
           } else {
               state->seeking_immediate_match = false;
           }
+          // The zero-skip's vacuous-anchor exemption only covers the immediate
+          // step it lands on. Once the state advances, a later anchor is normal.
+          state->skipped_quantifier = false;
 
           if (stop_on_definite_step && next_step->root_pattern_guaranteed) did_match = true;
 
@@ -4127,27 +4417,68 @@ static inline bool ts_query_cursor__advance(
                 k--;
               }
 
+              // A `?`/`*` zero-skip past a step that carries a trailing last-child
+              // anchor transfers that requirement to the last matched node. The
+              // skip is only valid if that node really is the last named child.
+              if (
+                child_step->alternative_is_skip &&
+                child_step->is_last_child &&
+                has_later_named_siblings
+              ) {
+                continue;
+              }
+
               QueryState *copy = ts_query_cursor__copy_state(self, &child_state);
               if (copy) {
                 LOG(
-                  "  split state for branch. pattern:%u, from_step:%u, to_step:%u, immediate:%d, capture_count: %u\n",
+                  "  split state for branch. pattern:%u, from_step:%u, to_step:%u, pass_through:%d, capture_count:%u\n",
                   copy->pattern_index,
                   copy->step_index,
                   next_step->alternative_index,
-                  next_step->alternative_is_immediate,
+                  next_step->is_pass_through,
                   capture_list_pool_get(&self->capture_list_pool, copy->capture_list_id)->size
                 );
                 end_index++;
                 copy_count++;
                 copy->step_index = child_step->alternative_index;
-                if (child_step->alternative_is_immediate) {
+                if (child_step->is_pass_through) {
                   copy->seeking_immediate_match = true;
                 }
+                // Taking a `?`/`*` zero-skip means the quantified subpattern matched
+                // nothing. How an adjacent anchor behaves then depends on where it sat:
+                if (child_step->alternative_is_skip) {
+                  if (!child_step->is_immediate) {
+                    QueryStep *skip_target = array_get(
+                      &self->query->steps,
+                      child_step->alternative_index
+                    );
+                    // No leading anchor on the skipped step, so an immediately-following
+                    // anchor on the skip target is vacuous (`Q* . B` with zero `Q` lets
+                    // `B` match anywhere).
+                    copy->skipped_quantifier = skip_target->depth == child_step->depth;
+                  } else if (
+                    array_get(&self->query->steps, child_state->step_index - 1)->depth <
+                    child_step->depth
+                  ) {
+                    // The skipped step was the parent's first child pattern and carried a
+                    // leading *boundary* anchor (`(P . Q* Y)`). Transfer the first-child
+                    // requirement to the skip target so it survives the empty run: `Y`
+                    // must still be the parent's first named child.
+                    copy->seeking_immediate_match = true;
+                  }
+                  // Otherwise the skipped step carried a leading *between* anchor
+                  // (`A . Q* ...`): with zero `Q` that adjacency vanishes, while the skip
+                  // target's own anchor, if any, still applies (`A . Q* . B` stays adjacent).
+                }
               }
             }
           }
         }
 
+        // Order states by capture position so the dedup pass below can stop scanning a
+        // group once the remaining states are disjoint from the current one.
+        ts_query_cursor__sort_states_by_capture(self);
+
         for (unsigned j = 0; j < self->states.size; j++) {
           QueryState *state = array_get(&self->states, j);
           if (state->dead) {
@@ -4163,7 +4494,10 @@ static inline bool ts_query_cursor__advance(
           for (unsigned k = j + 1; k < self->states.size; k++) {
             QueryState *other_state = array_get(&self->states, k);
 
-            // Query states are kept in ascending order of start_depth and pattern_index.
+            // Query states are kept in ascending order of start_depth and pattern_index, and
+            // (via the above call to `ts_query_cursor__sort_states_by_capture`) in ascending
+            // order of first-capture position within each such group.
+            //
             // Since the longest-match criteria is only used for deduping matches of the same
             // pattern and root node, we only need to perform pairwise comparisons within a
             // small slice of the states array.
@@ -4172,6 +4506,25 @@ static inline bool ts_query_cursor__advance(
               other_state->pattern_index != state->pattern_index
             ) break;
 
+            // States in a group acquire their first capture in tree-traversal order, so the
+            // group is ordered by first-capture position. Once `other_state`'s captures begin
+            // at or after where `state`'s captures end, `other_state` (and every state after
+            // it in the group) is disjoint from `state`: neither can be a capture-subset of
+            // the other, so there is nothing to drop and no longest-match alternative to
+            // record. Stop scanning `state` against the rest of the group.
+            const CaptureList *state_captures =
+              capture_list_pool_get(&self->capture_list_pool, state->capture_list_id);
+            const CaptureList *other_captures =
+              capture_list_pool_get(&self->capture_list_pool, other_state->capture_list_id);
+            if (
+              state_captures->size > 0 &&
+              other_captures->size > 0 &&
+              ts_node_start_byte(array_get(other_captures, 0)->node) >=
+                ts_node_end_byte(array_get(state_captures, state_captures->size - 1)->node)
+            ) {
+              break;
+            }
+
             bool left_contains_right, right_contains_left;
             ts_query_cursor__compare_captures(
               self,
@@ -4181,7 +4534,10 @@ static inline bool ts_query_cursor__advance(
               &right_contains_left
             );
             if (left_contains_right) {
-              if (state->step_index == other_state->step_index) {
+              if (
+                state->step_index == other_state->step_index &&
+                (other_state->seeking_immediate_match || !state->seeking_immediate_match)
+              ) {
                 LOG(
                   "  drop shorter state. pattern: %u, step_index: %u\n",
                   state->pattern_index,
@@ -4195,7 +4551,10 @@ static inline bool ts_query_cursor__advance(
               other_state->has_in_progress_alternatives = true;
             }
             if (right_contains_left) {
-              if (state->step_index == other_state->step_index) {
+              if (
+                state->step_index == other_state->step_index &&
+                (state->seeking_immediate_match || !other_state->seeking_immediate_match)
+              ) {
                 LOG(
                   "  drop shorter state. pattern: %u, step_index: %u\n",
                   state->pattern_index,
@@ -4227,7 +4586,7 @@ static inline bool ts_query_cursor__advance(
                 LOG("  defer finishing pattern %u\n", state->pattern_index);
               } else {
                 LOG("  finish pattern %u\n", state->pattern_index);
-                array_push(&self->finished_states, *state);
+                ts_query_cursor__push_finished_state(self, state);
                 array_erase(&self->states, (uint32_t)(state - self->states.contents));
                 did_match = true;
                 j--;
@@ -4265,8 +4624,22 @@ bool ts_query_cursor_next_match(
       return false;
     }
   }
+  if (self->finished_states_heap_size > 0) {
+    ts_query_cursor__heapify_finished_states(self);
+  }
 
-  QueryState *state = array_get(&self->finished_states, 0);
+  uint32_t state_index = 0;
+  if (self->finished_states_heap_size > 0) {
+    for (uint32_t i = 1; i < self->finished_states.size; i++) {
+      QueryState *state = array_get(&self->finished_states, i);
+      QueryState *earliest_state = array_get(&self->finished_states, state_index);
+      if (state->heap_insert_order < earliest_state->heap_insert_order) {
+        state_index = i;
+      }
+    }
+  }
+
+  QueryState *state = array_get(&self->finished_states, state_index);
   if (state->id == UINT32_MAX) state->id = self->next_state_id++;
   match->id = state->id;
   match->pattern_index = state->pattern_index;
@@ -4277,7 +4650,12 @@ bool ts_query_cursor_next_match(
   match->captures = captures->contents;
   match->capture_count = captures->size;
   capture_list_pool_release(&self->capture_list_pool, state->capture_list_id);
-  array_erase(&self->finished_states, 0);
+  if (self->finished_states_heap_size > 0) {
+    finished_state_erase(&self->finished_states, state_index, &self->capture_list_pool);
+    self->finished_states_heap_size = self->finished_states.size;
+  } else {
+    array_erase(&self->finished_states, state_index);
+  }
   return true;
 }
 
@@ -4285,6 +4663,10 @@ void ts_query_cursor_remove_match(
   TSQueryCursor *self,
   uint32_t match_id
 ) {
+  if (self->finished_states_heap_size > 0) {
+    ts_query_cursor__heapify_finished_states(self);
+  }
+
   for (unsigned i = 0; i < self->finished_states.size; i++) {
     const QueryState *state = array_get(&self->finished_states, i);
     if (state->id == match_id) {
@@ -4292,7 +4674,12 @@ void ts_query_cursor_remove_match(
         &self->capture_list_pool,
         state->capture_list_id
       );
-      array_erase(&self->finished_states, i);
+      if (self->finished_states_heap_size > 0) {
+        finished_state_erase(&self->finished_states, i, &self->capture_list_pool);
+        self->finished_states_heap_size = self->finished_states.size;
+      } else {
+        array_erase(&self->finished_states, i);
+      }
       return;
     }
   }
@@ -4321,6 +4708,9 @@ bool ts_query_cursor_next_capture(
   // be discovered in order, because patterns can overlap. Search for matches
   // until there is a finished capture that is before any unfinished capture.
   for (;;) {
+    // Sift any newly pushed finished states into the heap.
+    ts_query_cursor__heapify_finished_states(self);
+
     // First, find the earliest capture in an unfinished match.
     uint32_t first_unfinished_capture_byte;
     uint32_t first_unfinished_pattern_index;
@@ -4334,13 +4724,14 @@ bool ts_query_cursor_next_capture(
       &first_unfinished_state_is_definite
     );
 
-    // Then find the earliest capture in a finished match. It must occur
-    // before the first capture in an *unfinished* match.
+    // Then find the earliest capture in a finished match. The finished_states
+    // array is maintained as a min-heap, so the earliest is always at index 0.
+    // Clean up fully-consumed and out-of-range states from the heap root first.
     QueryState *first_finished_state = NULL;
     uint32_t first_finished_capture_byte = first_unfinished_capture_byte;
     uint32_t first_finished_pattern_index = first_unfinished_pattern_index;
-    for (unsigned i = 0; i < self->finished_states.size;) {
-      QueryState *state = array_get(&self->finished_states, i);
+    while (self->finished_states.size > 0) {
+      QueryState *state = array_get(&self->finished_states, 0);
       const CaptureList *captures = capture_list_pool_get(
         &self->capture_list_pool,
         state->capture_list_id
@@ -4352,7 +4743,8 @@ bool ts_query_cursor_next_capture(
           &self->capture_list_pool,
           state->capture_list_id
         );
-        array_erase(&self->finished_states, i);
+        finished_state_pop(&self->finished_states, &self->capture_list_pool);
+        self->finished_states_heap_size = self->finished_states.size;
         continue;
       }
 
@@ -4371,6 +4763,7 @@ bool ts_query_cursor_next_capture(
       // Skip captures that are outside of the cursor's range.
       if (node_outside_of_range) {
         state->consumed_capture_count++;
+        finished_state_sift_down(&self->finished_states, 0, &self->capture_list_pool);
         continue;
       }
 
@@ -4386,7 +4779,7 @@ bool ts_query_cursor_next_capture(
         first_finished_capture_byte = node_start_byte;
         first_finished_pattern_index = state->pattern_index;
       }
-      i++;
+      break;
     }
 
     // If there is finished capture that is clearly before any unfinished
@@ -4413,6 +4806,11 @@ bool ts_query_cursor_next_capture(
       match->capture_count = captures->size;
       *capture_index = state->consumed_capture_count;
       state->consumed_capture_count++;
+      // If this state is in the finished_states heap, its sort key has changed
+      // (next capture is now later in the document). Restore heap order.
+      if (state == first_finished_state) {
+        finished_state_sift_down(&self->finished_states, 0, &self->capture_list_pool);
+      }
       return true;
     }
 

lib/src/subtree.c

@@ -335,6 +335,15 @@ void ts_subtree_compress(
   }
 }
 
+// The part of an error node's cost that penalizes the extent it spans, as
+// opposed to the cost of its contents.
+static inline uint32_t ts_subtree__error_extent_cost(Length size) {
+  return
+    ERROR_COST_PER_RECOVERY +
+    ERROR_COST_PER_SKIPPED_CHAR * size.bytes +
+    ERROR_COST_PER_SKIPPED_LINE * size.extent.row;
+}
+
 // Assign all of the node's properties that depend on its children.
 void ts_subtree_summarize_children(
   MutableSubtree self,
@@ -386,20 +395,25 @@ void ts_subtree_summarize_children(
       lookahead_end_byte = child_lookahead_end_byte;
     }
 
-    if (ts_subtree_symbol(child) != ts_builtin_sym_error_repeat) {
-      self.ptr->error_cost += ts_subtree_error_cost(child);
-    }
-
     uint32_t grandchild_count = ts_subtree_child_count(child);
-    if (
-      self.ptr->symbol == ts_builtin_sym_error ||
-      self.ptr->symbol == ts_builtin_sym_error_repeat
-    ) {
-      if (!ts_subtree_extra(child) && !(ts_subtree_is_error(child) && grandchild_count == 0)) {
-        if (ts_subtree_visible(child)) {
-          self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE;
-        } else if (grandchild_count > 0) {
-          self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE * child.ptr->visible_child_count;
+    if (ts_subtree_symbol(child) == ts_builtin_sym_error_repeat) {
+      // Refund an `_ERROR` child's extent penalty, which this node re-charges
+      // as part of its own extent below, so that the grouping is cost-neutral.
+      uint32_t extent_cost = ts_subtree__error_extent_cost(ts_subtree_size(child));
+      ts_assert(ts_subtree_error_cost(child) >= extent_cost);
+      self.ptr->error_cost += ts_subtree_error_cost(child) - extent_cost;
+    } else {
+      self.ptr->error_cost += ts_subtree_error_cost(child);
+      if (
+        self.ptr->symbol == ts_builtin_sym_error ||
+        self.ptr->symbol == ts_builtin_sym_error_repeat
+      ) {
+        if (!ts_subtree_extra(child) && !(ts_subtree_is_error(child) && grandchild_count == 0)) {
+          if (ts_subtree_visible(child)) {
+            self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE;
+          } else if (grandchild_count > 0) {
+            self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE * child.ptr->visible_child_count;
+          }
         }
       }
     }
@@ -443,10 +457,7 @@ void ts_subtree_summarize_children(
     self.ptr->symbol == ts_builtin_sym_error ||
     self.ptr->symbol == ts_builtin_sym_error_repeat
   ) {
-    self.ptr->error_cost +=
-      ERROR_COST_PER_RECOVERY +
-      ERROR_COST_PER_SKIPPED_CHAR * self.ptr->size.bytes +
-      ERROR_COST_PER_SKIPPED_LINE * self.ptr->size.extent.row;
+    self.ptr->error_cost += ts_subtree__error_extent_cost(self.ptr->size);
   }
 
   if (self.ptr->child_count > 0) {

lib/src/subtree.h

@@ -252,7 +252,7 @@ static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
 // Get a subtree's children, which are allocated immediately before the
 // tree's own heap data.
 #define ts_subtree_children(self) \
-  ((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
+  ((self).data.is_inline ? (Subtree *)NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
 
 static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) {
   if (self->data.is_inline) {

lib/src/tree_cursor.c

@@ -153,8 +153,10 @@ static inline bool ts_tree_cursor_child_iterator_previous(
 // TSTreeCursor - lifecycle
 
 TSTreeCursor ts_tree_cursor_new(TSNode node) {
-  TSTreeCursor self = {NULL, NULL, {0, 0, 0}};
-  ts_tree_cursor_init((TreeCursor *)&self, node);
+  TreeCursor cursor = {0};
+  ts_tree_cursor_init(&cursor, node);
+  TSTreeCursor self = {0};
+  memcpy(&self, &cursor, sizeof(cursor));
   return self;
 }
 
@@ -550,8 +552,10 @@ void ts_tree_cursor_current_status(
       (*supertype_count)++;
     }
 
-    // Determine if the current node has later siblings.
-    if (!*has_later_siblings) {
+    // Determine if the current node has later siblings. A later *anonymous*
+    // sibling settles `has_later_siblings` but says nothing about later *named*
+    // siblings.
+    if (!*has_later_named_siblings) {
       unsigned sibling_count = parent_entry->subtree->ptr->child_count;
       unsigned structural_child_index = entry->structural_child_index;
       if (!ts_subtree_extra(*entry->subtree)) structural_child_index++;
@@ -563,14 +567,12 @@ void ts_tree_cursor_current_status(
         );
         if (sibling_metadata.visible) {
           *has_later_siblings = true;
-          if (*has_later_named_siblings) break;
           if (sibling_metadata.named) {
             *has_later_named_siblings = true;
             break;
           }
         } else if (ts_subtree_visible_child_count(sibling) > 0) {
           *has_later_siblings = true;
-          if (*has_later_named_siblings) break;
           if (sibling.ptr->named_child_count > 0) {
             *has_later_named_siblings = true;
             break;
@@ -697,12 +699,13 @@ const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
 
 TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
   const TreeCursor *cursor = (const TreeCursor *)_cursor;
-  TSTreeCursor res = {NULL, NULL, {0, 0}};
-  TreeCursor *copy = (TreeCursor *)&res;
-  copy->tree = cursor->tree;
-  copy->root_alias_symbol = cursor->root_alias_symbol;
-  array_init(&copy->stack);
-  array_push_all(&copy->stack, &cursor->stack);
+  TreeCursor copy = {0};
+  copy.tree = cursor->tree;
+  copy.root_alias_symbol = cursor->root_alias_symbol;
+  array_init(&copy.stack);
+  array_push_all(&copy.stack, &cursor->stack);
+  TSTreeCursor res = {0};
+  memcpy(&res, &copy, sizeof(copy));
   return res;
 }
 

lib/src/unicode.h

@@ -53,8 +53,14 @@ static inline uint32_t ts_decode_utf16_le(
   uint32_t length,
   int32_t *code_point
 ) {
+  if (length < 2) {
+    *code_point = TS_DECODE_ERROR;
+    return length;
+  }
   uint32_t i = 0;
-  U16_NEXT_LE(((uint16_t *)string), i, length, *code_point);
+  // length is in bytes; U16_NEXT indexes into uint16_t*, so its length
+  // parameter must be in code units (length / 2), not bytes.
+  U16_NEXT_LE(((uint16_t *)string), i, length / 2, *code_point);
   return i * 2;
 }
 
@@ -63,8 +69,14 @@ static inline uint32_t ts_decode_utf16_be(
   uint32_t length,
   int32_t *code_point
 ) {
+  if (length < 2) {
+    *code_point = TS_DECODE_ERROR;
+    return length;
+  }
   uint32_t i = 0;
-  U16_NEXT_BE(((uint16_t *)string), i, length, *code_point);
+  // length is in bytes; U16_NEXT indexes into uint16_t*, so its length
+  // parameter must be in code units (length / 2), not bytes.
+  U16_NEXT_BE(((uint16_t *)string), i, length / 2, *code_point);
   return i * 2;
 }
 

lib/src/wasm_store.c

@@ -80,15 +80,15 @@ typedef struct {
 } LanguageWasmInstance;
 
 typedef struct {
-  uint32_t reset_heap;
-  uint32_t proc_exit;
-  uint32_t abort;
-  uint32_t assert_fail;
-  uint32_t notify_memory_growth;
-  uint32_t debug_message;
-  uint32_t at_exit;
-  uint32_t args_get;
-  uint32_t args_sizes_get;
+  wasmtime_func_t reset_heap;
+  wasmtime_func_t proc_exit;
+  wasmtime_func_t abort;
+  wasmtime_func_t assert_fail;
+  wasmtime_func_t notify_memory_growth;
+  wasmtime_func_t debug_message;
+  wasmtime_func_t at_exit;
+  wasmtime_func_t args_get;
+  wasmtime_func_t args_sizes_get;
 } BuiltinFunctionIndices;
 
 // TSWasmStore - A struct that allows a given `Parser` to use Wasm-backed
@@ -104,7 +104,7 @@ struct TSWasmStore {
   Array(LanguageWasmInstance) language_instances;
   uint32_t current_memory_offset;
   uint32_t current_function_table_offset;
-  uint32_t *stdlib_fn_indices;
+  wasmtime_func_t *stdlib_fn_indices;
   BuiltinFunctionIndices builtin_fn_indices;
   wasmtime_global_t stack_pointer_global;
   wasm_globaltype_t *const_i32_type;
@@ -255,7 +255,7 @@ static bool wasm_dylink_info__parse(
  * Native callbacks exposed to Wasm modules
  *******************************************/
 
- static wasm_trap_t *callback__abort(
+static wasm_trap_t *callback__abort(
   void *env,
   wasmtime_caller_t* caller,
   wasmtime_val_raw_t *args_and_results,
@@ -360,23 +360,65 @@ static wasm_trap_t *callback__lexer_eof(
 }
 
 typedef struct {
-  uint32_t *storage_location;
+  void *storage_location;
   wasmtime_func_unchecked_callback_t callback;
   wasm_functype_t *type;
 } FunctionDefinition;
 
-static void *copy(const void *data, size_t size) {
+typedef struct {
+  const uint8_t *data;
+  size_t size;
+} WasmMemory;
+
+static bool wasm_memory__contains(const WasmMemory *memory, int32_t address, size_t size) {
+  if (address < 0) return false;
+  size_t start = (size_t)address;
+  return start <= memory->size && size <= memory->size - start;
+}
+
+static bool wasm_memory__read(const WasmMemory *memory, int32_t address, void *result, size_t size) {
+  if (!wasm_memory__contains(memory, address, size)) return false;
+  memcpy(result, &memory->data[address], size);
+  return true;
+}
+
+static bool wasm_memory__string_length(const WasmMemory *memory, int32_t address, size_t *length) {
+  if (address < 0 || (size_t)address >= memory->size) return false;
+  const uint8_t *data = &memory->data[address];
+  size_t limit = memory->size - (size_t)address;
+  for (size_t i = 0; i < limit; i++) {
+    if (data[i] == 0) {
+      *length = i;
+      return true;
+    }
+  }
+  return false;
+}
+
+static void *copy(const WasmMemory *memory, int32_t address, size_t size, bool *ok) {
+  if (!*ok || size == 0) return NULL;
+  if (!wasm_memory__contains(memory, address, size)) {
+    *ok = false;
+    return NULL;
+  }
   void *result = ts_malloc(size);
-  memcpy(result, data, size);
+  memcpy(result, &memory->data[address], size);
   return result;
 }
 
 static void *copy_unsized_static_array(
-  const uint8_t *data,
+  const WasmMemory *memory,
   int32_t start_address,
   const int32_t all_addresses[],
-  size_t address_count
+  size_t address_count,
+  bool *ok
 ) {
+  if (!*ok || start_address == 0) return NULL;
+  if (start_address < 0) {
+    *ok = false;
+    return NULL;
+  }
+
   int32_t end_address = 0;
   for (unsigned i = 0; i < address_count; i++) {
     if (all_addresses[i] > start_address) {
@@ -388,28 +430,48 @@ static void *copy_unsized_static_array(
 
   if (!end_address) return NULL;
   size_t size = end_address - start_address;
+  if (!wasm_memory__contains(memory, start_address, size)) {
+    *ok = false;
+    return NULL;
+  }
   void *result = ts_malloc(size);
-  memcpy(result, &data[start_address], size);
+  memcpy(result, &memory->data[start_address], size);
   return result;
 }
 
 static void *copy_strings(
-  const uint8_t *data,
+  const WasmMemory *memory,
   int32_t array_address,
   size_t count,
-  StringData *string_data
+  StringData *string_data,
+  bool *ok
 ) {
+  if (!*ok) return NULL;
+  if (count > SIZE_MAX / sizeof(char *)) {
+    *ok = false;
+    return NULL;
+  }
+  if (count > (SIZE_MAX / sizeof(int32_t)) ||
+      !wasm_memory__contains(memory, array_address, count * sizeof(int32_t))) {
+    *ok = false;
+    return NULL;
+  }
+
   const char **result = ts_malloc(count * sizeof(char *));
   for (unsigned i = 0; i < count; i++) {
     int32_t address;
-    memcpy(&address, &data[array_address + i * sizeof(address)], sizeof(address));
+    memcpy(&address, &memory->data[array_address + i * sizeof(address)], sizeof(address));
     if (address == 0) {
       result[i] = (const char *)-1;
     } else {
-      const uint8_t *string = &data[address];
-      uint32_t len = strlen((const char *)string);
+      size_t len;
+      if (!wasm_memory__string_length(memory, address, &len) || len > UINT32_MAX) {
+        ts_free(result);
+        *ok = false;
+        return NULL;
+      }
       result[i] = (const char *)(uintptr_t)string_data->size;
-      array_extend(string_data, len + 1, string);
+      array_extend(string_data, len + 1, &memory->data[address]);
     }
   }
   for (unsigned i = 0; i < count; i++) {
@@ -423,16 +485,54 @@ static void *copy_strings(
 }
 
 static void *copy_string(
-  const uint8_t *data,
-  int32_t address
+  const WasmMemory *memory,
+  int32_t address,
+  bool *ok
 ) {
-  const char *string = (const char *)&data[address];
-  size_t len = strlen(string);
+  if (!*ok) return NULL;
+  size_t len;
+  if (!wasm_memory__string_length(memory, address, &len)) {
+    *ok = false;
+    return NULL;
+  }
+  const char *string = (const char *)&memory->data[address];
   char *result = ts_malloc(len + 1);
   memcpy(result, string, len + 1);
   return result;
 }
 
+static void delete_partially_loaded_language(
+  TSLanguage *language,
+  StringData *symbol_name_buffer,
+  StringData *field_name_buffer
+) {
+  if (language) {
+    ts_free((void *)language->alias_map);
+    ts_free((void *)language->alias_sequences);
+    ts_free((void *)language->external_scanner.symbol_map);
+    ts_free((void *)language->field_map_entries);
+    ts_free((void *)language->field_map_slices);
+    ts_free((void *)language->field_names);
+    ts_free((void *)language->lex_modes);
+    ts_free((void *)language->name);
+    ts_free((void *)language->parse_actions);
+    ts_free((void *)language->parse_table);
+    ts_free((void *)language->primary_state_ids);
+    ts_free((void *)language->public_symbol_map);
+    ts_free((void *)language->reserved_words);
+    ts_free((void *)language->small_parse_table);
+    ts_free((void *)language->small_parse_table_map);
+    ts_free((void *)language->supertype_map_entries);
+    ts_free((void *)language->supertype_map_slices);
+    ts_free((void *)language->supertype_symbols);
+    ts_free((void *)language->symbol_metadata);
+    ts_free((void *)language->symbol_names);
+    ts_free(language);
+  }
+  array_delete(symbol_name_buffer);
+  array_delete(field_name_buffer);
+}
+
 static bool name_eq(const wasm_name_t *name, const char *string) {
   return strncmp(string, name->data, name->size) == 0;
 }
@@ -476,15 +576,11 @@ void language_id_delete(WasmLanguageId *self) {
 }
 
 static wasmtime_extern_t get_builtin_extern(
-  wasmtime_table_t *table,
-  unsigned index
+  wasmtime_func_t *func
 ) {
   return (wasmtime_extern_t) {
     .kind = WASMTIME_EXTERN_FUNC,
-    .of.func = (wasmtime_func_t) {
-      .store_id = table->store_id,
-      .__private = index
-    }
+    .of.func = *func
   };
 }
 
@@ -519,21 +615,21 @@ static bool ts_wasm_store__provide_builtin_import(
 
   // Builtin functions
   else if (name_eq(import_name, "__assert_fail")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.assert_fail);
+    *import = get_builtin_extern(&self->builtin_fn_indices.assert_fail);
   } else if (name_eq(import_name, "__cxa_atexit")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.at_exit);
+    *import = get_builtin_extern(&self->builtin_fn_indices.at_exit);
   } else if (name_eq(import_name, "args_get")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_get);
+    *import = get_builtin_extern(&self->builtin_fn_indices.args_get);
   } else if (name_eq(import_name, "args_sizes_get")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_sizes_get);
+    *import = get_builtin_extern(&self->builtin_fn_indices.args_sizes_get);
   } else if (name_eq(import_name, "abort")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.abort);
+    *import = get_builtin_extern(&self->builtin_fn_indices.abort);
   } else if (name_eq(import_name, "proc_exit")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.proc_exit);
+    *import = get_builtin_extern(&self->builtin_fn_indices.proc_exit);
   } else if (name_eq(import_name, "emscripten_notify_memory_growth")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.notify_memory_growth);
+    *import = get_builtin_extern(&self->builtin_fn_indices.notify_memory_growth);
   } else if (name_eq(import_name, "tree_sitter_debug_message")) {
-    *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.debug_message);
+    *import = get_builtin_extern(&self->builtin_fn_indices.debug_message);
   } else {
     return false;
   }
@@ -575,6 +671,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   wasmtime_module_t *stdlib_module = NULL;
   wasm_memorytype_t *memory_type = NULL;
   wasm_tabletype_t *table_type = NULL;
+  wasmtime_func_t *lexer_funcs = NULL;
 
   // Define functions called by scanners via function pointers on the lexer.
   LexerInWasmMemory lexer = {
@@ -583,34 +680,34 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   };
   FunctionDefinition lexer_definitions[] = {
     {
-      (uint32_t *)&lexer.advance,
+      &lexer.advance,
       callback__lexer_advance,
       wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.mark_end,
+      &lexer.mark_end,
       callback__lexer_mark_end,
       wasm_functype_new_1_0(wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.get_column,
+      &lexer.get_column,
       callback__lexer_get_column,
       wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.is_at_included_range_start,
+      &lexer.is_at_included_range_start,
       callback__lexer_is_at_included_range_start,
       wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
     {
-      (uint32_t *)&lexer.eof,
+      &lexer.eof,
       callback__lexer_eof,
       wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
     },
   };
 
   // Define builtin functions that can be imported by scanners.
-  BuiltinFunctionIndices builtin_fn_indices;
+  BuiltinFunctionIndices builtin_fn_indices = {0};
   FunctionDefinition builtin_definitions[] = {
     {
       &builtin_fn_indices.proc_exit,
@@ -657,18 +754,16 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   // Create all of the Wasm functions.
   unsigned builtin_definitions_len = array_len(builtin_definitions);
   unsigned lexer_definitions_len = array_len(lexer_definitions);
+  lexer_funcs = ts_calloc(lexer_definitions_len, sizeof(wasmtime_func_t));
   for (unsigned i = 0; i < builtin_definitions_len; i++) {
     FunctionDefinition *definition = &builtin_definitions[i];
-    wasmtime_func_t func;
-    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
-    *definition->storage_location = func.__private;
+    wasmtime_func_t *func = (wasmtime_func_t *)definition->storage_location;
+    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, func);
     wasm_functype_delete(definition->type);
   }
   for (unsigned i = 0; i < lexer_definitions_len; i++) {
     FunctionDefinition *definition = &lexer_definitions[i];
-    wasmtime_func_t func;
-    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
-    *definition->storage_location = func.__private;
+    wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &lexer_funcs[i]);
     wasm_functype_delete(definition->type);
   }
 
@@ -763,7 +858,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
     .memory = memory,
     .function_table = function_table,
     .language_instances = array_new(),
-    .stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(uint32_t)),
+    .stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(wasmtime_func_t)),
     .builtin_fn_indices = builtin_fn_indices,
     .stack_pointer_global = stack_pointer_global,
     .current_memory_offset = 0,
@@ -816,7 +911,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
 
   // Process the stdlib module's exports.
   for (unsigned i = 0; i < stdlib_symbols_len; i++) {
-    self->stdlib_fn_indices[i] = UINT32_MAX;
+    self->stdlib_fn_indices[i] = (wasmtime_func_t){.store_id = 0};
   }
   wasmtime_module_exports(stdlib_module, &export_types);
   for (unsigned i = 0; i < export_types.size; i++) {
@@ -851,20 +946,20 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
       }
 
       if (name_eq(name, "reset_heap")) {
-        self->builtin_fn_indices.reset_heap = export.of.func.__private;
+        self->builtin_fn_indices.reset_heap = export.of.func;
         continue;
       }
 
       for (unsigned j = 0; j < stdlib_symbols_len; j++) {
         if (name_eq(name, STDLIB_SYMBOLS[j])) {
-          self->stdlib_fn_indices[j] = export.of.func.__private;
+          self->stdlib_fn_indices[j] = export.of.func;
           break;
         }
       }
     }
   }
 
-  if (self->builtin_fn_indices.reset_heap == UINT32_MAX) {
+  if (self->builtin_fn_indices.reset_heap.store_id == 0) {
     wasm_error->kind = TSWasmErrorKindInstantiate;
     format(
       &wasm_error->message,
@@ -874,7 +969,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   }
 
   for (unsigned i = 0; i < stdlib_symbols_len; i++) {
-    if (self->stdlib_fn_indices[i] == UINT32_MAX) {
+    if (self->stdlib_fn_indices[i].store_id == 0) {
       wasm_error->kind = TSWasmErrorKindInstantiate;
       format(
         &wasm_error->message,
@@ -904,13 +999,13 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
   }
   for (unsigned i = 0; i < lexer_definitions_len; i++) {
     FunctionDefinition *definition = &lexer_definitions[i];
-    wasmtime_func_t func = {function_table.store_id, *definition->storage_location};
-    wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = func};
+    wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = lexer_funcs[i]};
     error = wasmtime_table_set(context, &function_table, table_index, &func_val);
     ts_assert(!error);
     *(int32_t *)(definition->storage_location) = table_index;
     table_index++;
   }
+  ts_free(lexer_funcs);
 
   self->current_function_table_offset = table_index;
   self->lexer_address = initial_memory_pages * MEMORY_PAGE_SIZE;
@@ -937,6 +1032,7 @@ error:
   if (message.size) wasm_byte_vec_delete(&message);
   if (export_types.size) wasm_exporttype_vec_delete(&export_types);
   if (imports) ts_free(imports);
+  ts_free(lexer_funcs);
   return NULL;
 }
 
@@ -1016,8 +1112,6 @@ static bool ts_wasm_store__instantiate(
   // Construct the language function name as string.
   format(&language_function_name, "tree_sitter_%s", language_name);
 
-  const uint64_t store_id = self->function_table.store_id;
-
   // Build the imports list for the module.
   wasm_importtype_vec_t import_types = WASM_EMPTY_VEC;
   wasmtime_module_imports(module, &import_types);
@@ -1038,8 +1132,7 @@ static bool ts_wasm_store__instantiate(
     bool defined_in_stdlib = false;
     for (unsigned j = 0; j < array_len(STDLIB_SYMBOLS); j++) {
       if (name_eq(import_name, STDLIB_SYMBOLS[j])) {
-        uint16_t address = self->stdlib_fn_indices[j];
-        imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = {store_id, address}};
+        imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = self->stdlib_fn_indices[j]};
         defined_in_stdlib = true;
         break;
       }
@@ -1179,6 +1272,9 @@ const TSLanguage *ts_wasm_store_load_language(
   WasmDylinkInfo dylink_info;
   wasmtime_module_t *module = NULL;
   wasmtime_error_t *error = NULL;
+  TSLanguage *language = NULL;
+  StringData symbol_name_buffer = array_new();
+  StringData field_name_buffer = array_new();
   wasm_error->kind = TSWasmErrorKindNone;
 
   if (!wasm_dylink_info__parse((const unsigned char *)wasm, wasm_len, &dylink_info)) {
@@ -1219,10 +1315,17 @@ const TSLanguage *ts_wasm_store_load_language(
   LanguageInWasmMemory wasm_language;
   wasmtime_context_t *context = wasmtime_store_context(self->store);
   const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
-  memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory));
+  WasmMemory wasm_memory = {
+    .data = memory,
+    .size = wasmtime_memory_data_size(context, &self->memory),
+  };
+  bool valid_wasm_memory = true;
+  if (!wasm_memory__read(&wasm_memory, language_address, &wasm_language, sizeof(LanguageInWasmMemory))) {
+    goto invalid_language_memory;
+  }
 
   bool has_supertypes =
-    wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS &&
+    wasm_language.abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS &&
     wasm_language.supertype_count > 0;
 
   int32_t addresses[] = {
@@ -1259,10 +1362,7 @@ const TSLanguage *ts_wasm_store_load_language(
   };
   uint32_t address_count = array_len(addresses);
 
-  TSLanguage *language = ts_calloc(1, sizeof(TSLanguage));
-  StringData symbol_name_buffer = array_new();
-  StringData field_name_buffer = array_new();
-
+  language = ts_calloc(1, sizeof(TSLanguage));
   *language = (TSLanguage) {
     .abi_version = wasm_language.abi_version,
     .symbol_count = wasm_language.symbol_count,
@@ -1278,40 +1378,54 @@ const TSLanguage *ts_wasm_store_load_language(
     .keyword_capture_token = wasm_language.keyword_capture_token,
     .metadata = wasm_language.metadata,
     .parse_table = copy(
-      &memory[wasm_language.parse_table],
-      wasm_language.large_state_count * wasm_language.symbol_count * sizeof(uint16_t)
+      &wasm_memory,
+      wasm_language.parse_table,
+      wasm_language.large_state_count * wasm_language.symbol_count * sizeof(uint16_t),
+      &valid_wasm_memory
     ),
     .parse_actions = copy_unsized_static_array(
-      memory,
+      &wasm_memory,
       wasm_language.parse_actions,
       addresses,
-      address_count
+      address_count,
+      &valid_wasm_memory
     ),
     .symbol_names = copy_strings(
-      memory,
+      &wasm_memory,
       wasm_language.symbol_names,
       wasm_language.symbol_count + wasm_language.alias_count,
-      &symbol_name_buffer
+      &symbol_name_buffer,
+      &valid_wasm_memory
     ),
     .symbol_metadata = copy(
-      &memory[wasm_language.symbol_metadata],
-      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbolMetadata)
+      &wasm_memory,
+      wasm_language.symbol_metadata,
+      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbolMetadata),
+      &valid_wasm_memory
     ),
     .public_symbol_map = copy(
-      &memory[wasm_language.public_symbol_map],
-      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.public_symbol_map,
+      (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbol),
+      &valid_wasm_memory
     ),
     .lex_modes = copy(
-      &memory[wasm_language.lex_modes],
-      wasm_language.state_count * sizeof(TSLexerMode)
+      &wasm_memory,
+      wasm_language.lex_modes,
+      wasm_language.state_count * sizeof(TSLexerMode),
+      &valid_wasm_memory
     ),
   };
+  if (!valid_wasm_memory) goto invalid_language_memory;
 
   if (language->field_count > 0 && language->production_id_count > 0) {
     language->field_map_slices = copy(
-      &memory[wasm_language.field_map_slices],
-      wasm_language.production_id_count * sizeof(TSMapSlice)
+      &wasm_memory,
+      wasm_language.field_map_slices,
+      wasm_language.production_id_count * sizeof(TSMapSlice),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
 
     // Determine the number of field map entries by finding the greatest index
     // in any of the slices.
@@ -1325,22 +1439,29 @@ const TSLanguage *ts_wasm_store_load_language(
     }
 
     language->field_map_entries = copy(
-      &memory[wasm_language.field_map_entries],
-      field_map_entry_count * sizeof(TSFieldMapEntry)
+      &wasm_memory,
+      wasm_language.field_map_entries,
+      field_map_entry_count * sizeof(TSFieldMapEntry),
+      &valid_wasm_memory
     );
     language->field_names = copy_strings(
-      memory,
+      &wasm_memory,
       wasm_language.field_names,
       wasm_language.field_count + 1,
-      &field_name_buffer
+      &field_name_buffer,
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (has_supertypes) {
     language->supertype_symbols = copy(
-      &memory[wasm_language.supertype_symbols],
-      wasm_language.supertype_count * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.supertype_symbols,
+      wasm_language.supertype_count * sizeof(TSSymbol),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
 
     // Determine the number of supertype map slices by finding the greatest
     // supertype ID.
@@ -1353,18 +1474,24 @@ const TSLanguage *ts_wasm_store_load_language(
     }
 
     language->supertype_map_slices = copy(
-      &memory[wasm_language.supertype_map_slices],
-      (largest_supertype + 1) * sizeof(TSMapSlice)
+      &wasm_memory,
+      wasm_language.supertype_map_slices,
+      (largest_supertype + 1) * sizeof(TSMapSlice),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
 
     TSSymbol last_supertype = language->supertype_symbols[language->supertype_count - 1];
     TSMapSlice last_slice = language->supertype_map_slices[last_supertype];
     uint32_t supertype_map_entry_count = last_slice.index + last_slice.length;
 
     language->supertype_map_entries = copy(
-      &memory[wasm_language.supertype_map_entries],
-      supertype_map_entry_count * sizeof(char *)
+      &wasm_memory,
+      wasm_language.supertype_map_entries,
+      supertype_map_entry_count * sizeof(char *),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->max_alias_sequence_length > 0 && language->production_id_count > 0) {
@@ -1372,59 +1499,95 @@ const TSLanguage *ts_wasm_store_load_language(
     int32_t alias_map_size = 0;
     for (;;) {
       TSSymbol symbol;
-      memcpy(&symbol, &memory[wasm_language.alias_map + alias_map_size], sizeof(symbol));
+      if (!wasm_memory__read(&wasm_memory, wasm_language.alias_map + alias_map_size, &symbol, sizeof(symbol))) {
+        goto invalid_language_memory;
+      }
       alias_map_size += sizeof(TSSymbol);
       if (symbol == 0) break;
       uint16_t value_count;
-      memcpy(&value_count, &memory[wasm_language.alias_map + alias_map_size], sizeof(value_count));
+      if (!wasm_memory__read(&wasm_memory, wasm_language.alias_map + alias_map_size, &value_count, sizeof(value_count))) {
+        goto invalid_language_memory;
+      }
       alias_map_size += sizeof(uint16_t);
       alias_map_size += value_count * sizeof(TSSymbol);
     }
     language->alias_map = copy(
-      &memory[wasm_language.alias_map],
-      alias_map_size
+      &wasm_memory,
+      wasm_language.alias_map,
+      alias_map_size,
+      &valid_wasm_memory
     );
     language->alias_sequences = copy(
-      &memory[wasm_language.alias_sequences],
-      wasm_language.production_id_count * wasm_language.max_alias_sequence_length * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.alias_sequences,
+      wasm_language.production_id_count * wasm_language.max_alias_sequence_length * sizeof(TSSymbol),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->state_count > language->large_state_count) {
     uint32_t small_state_count = wasm_language.state_count - wasm_language.large_state_count;
     language->small_parse_table_map = copy(
-      &memory[wasm_language.small_parse_table_map],
-      small_state_count * sizeof(uint32_t)
+      &wasm_memory,
+      wasm_language.small_parse_table_map,
+      small_state_count * sizeof(uint32_t),
+      &valid_wasm_memory
     );
     language->small_parse_table = copy_unsized_static_array(
-      memory,
+      &wasm_memory,
       wasm_language.small_parse_table,
       addresses,
-      address_count
+      address_count,
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) {
     language->primary_state_ids = copy(
-      &memory[wasm_language.primary_state_ids],
-      wasm_language.state_count * sizeof(TSStateId)
+      &wasm_memory,
+      wasm_language.primary_state_ids,
+      wasm_language.state_count * sizeof(TSStateId),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
   }
 
   if (language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) {
-    language->name = copy_string(memory, wasm_language.name);
-    language->reserved_words = copy(
-        &memory[wasm_language.reserved_words],
-        wasm_language.max_reserved_word_set_size * sizeof(TSSymbol)
-    );
+    language->name = copy_string(&wasm_memory, wasm_language.name, &valid_wasm_memory);
+    if (!valid_wasm_memory) goto invalid_language_memory;
     language->max_reserved_word_set_size = wasm_language.max_reserved_word_set_size;
+
+    // Determine the number of reserved word sets by finding the maximum
+    // reserved_word_set_id across all lex modes.
+    uint16_t max_reserved_word_set_id = 0;
+    for (uint32_t i = 0; i < wasm_language.state_count; i++) {
+      uint16_t id = language->lex_modes[i].reserved_word_set_id;
+      if (id > max_reserved_word_set_id) max_reserved_word_set_id = id;
+    }
+
+    if (max_reserved_word_set_id > 0 && language->max_reserved_word_set_size > 0) {
+      uint32_t reserved_word_count =
+        (max_reserved_word_set_id + 1) * language->max_reserved_word_set_size;
+      language->reserved_words = copy(
+          &wasm_memory,
+          wasm_language.reserved_words,
+          reserved_word_count * sizeof(TSSymbol),
+          &valid_wasm_memory
+      );
+      if (!valid_wasm_memory) goto invalid_language_memory;
+    }
   }
 
   if (language->external_token_count > 0) {
     language->external_scanner.symbol_map = copy(
-      &memory[wasm_language.external_scanner.symbol_map],
-      wasm_language.external_token_count * sizeof(TSSymbol)
+      &wasm_memory,
+      wasm_language.external_scanner.symbol_map,
+      wasm_language.external_token_count * sizeof(TSSymbol),
+      &valid_wasm_memory
     );
+    if (!valid_wasm_memory) goto invalid_language_memory;
     language->external_scanner.states = (void *)(uintptr_t)wasm_language.external_scanner.states;
   }
 
@@ -1476,7 +1639,13 @@ const TSLanguage *ts_wasm_store_load_language(
 
   return language;
 
+invalid_language_memory:
+  wasm_error->kind = TSWasmErrorKindInstantiate;
+  format(&wasm_error->message, "invalid language memory address");
+  goto error;
+
 error:
+  delete_partially_loaded_language(language, &symbol_name_buffer, &field_name_buffer);
   if (module) wasmtime_module_delete(module);
   return NULL;
 }
@@ -1526,7 +1695,13 @@ bool ts_wasm_store_add_language(
 
     LanguageInWasmMemory wasm_language;
     const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
-    memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory));
+    WasmMemory wasm_memory = {
+      .data = memory,
+      .size = wasmtime_memory_data_size(context, &self->memory),
+    };
+    if (!wasm_memory__read(&wasm_memory, language_address, &wasm_language, sizeof(LanguageInWasmMemory))) {
+      return false;
+    }
     array_push(&self->language_instances, ((LanguageWasmInstance) {
       .language_id = language_id_clone(language_module->language_id),
       .instance = instance,
@@ -1546,16 +1721,13 @@ bool ts_wasm_store_add_language(
 
 void ts_wasm_store_reset_heap(TSWasmStore *self) {
   wasmtime_context_t *context = wasmtime_store_context(self->store);
-  wasmtime_func_t func = {
-    self->function_table.store_id,
-    self->builtin_fn_indices.reset_heap
-  };
+  wasmtime_func_t *func = &self->builtin_fn_indices.reset_heap;
   wasm_trap_t *trap = NULL;
   wasmtime_val_t args[1] = {
     {.of.i32 = ts_wasm_store__heap_address(self), .kind = WASMTIME_I32},
   };
 
-  wasmtime_error_t *error = wasmtime_func_call(context, &func, args, 1, NULL, 0, &trap);
+  wasmtime_error_t *error = wasmtime_func_call(context, func, args, 1, NULL, 0, &trap);
   ts_assert(!error);
   ts_assert(!trap);
 }

test/fixtures/error_corpus/javascript_errors.txt

@@ -187,3 +187,23 @@ function main(x) {
             (member_expression (identifier) (property_identifier))
             (arguments (string (string_fragment))))))
       (return_statement (object)))))
+
+===================================================
+Stray tokens around a parenthesized ternary
+===================================================
+
+x
+// one
+( a ? b : c ) :
+// two
+y.
+
+---
+
+(program
+  (ERROR
+    (call_expression
+      (identifier)
+      (comment)
+      (arguments (ternary_expression (identifier) (identifier) (identifier))))
+    (ERROR (comment) (identifier))))

test/fixtures/fixtures.json

@@ -1,17 +1,17 @@
 [
-  ["bash","v0.25.0"],
-  ["c","v0.24.1"],
-  ["cpp","v0.23.4"],
-  ["embedded-template","v0.25.0"],
-  ["go","v0.25.0"],
-  ["html","v0.23.2"],
-  ["java","v0.23.5"],
-  ["javascript","v0.25.0"],
-  ["jsdoc","v0.23.2"],
-  ["json","v0.24.8"],
-  ["php","v0.24.2"],
-  ["python","v0.23.6"],
-  ["ruby","v0.23.1"],
-  ["rust","v0.24.0"],
-  ["typescript","v0.23.2"]
-]
+  ["bash","v0.25.0", null],
+  ["c","v0.24.1", null],
+  ["cpp","v0.23.4", null],
+  ["embedded-template","v0.25.0", null],
+  ["go","v0.25.0", null],
+  ["html","v0.23.2", null],
+  ["java","v0.23.5", null],
+  ["javascript","v0.25.0", null],
+  ["jsdoc","v0.23.2", null],
+  ["json","v0.24.8", null],
+  ["php","v0.24.2", "upstream_test_fixture"],
+  ["python","v0.23.6", null],
+  ["ruby","v0.23.1", null],
+  ["rust","v0.24.0", null],
+  ["typescript","v0.23.2", null]
+]

test/fixtures/test_grammars/associativity_left_with_lower_precedence_shift/corpus.txt

@@ -0,0 +1,37 @@
+================================================================================
+Addition is left associative
+================================================================================
+
+1 + 1 + 1 + 1
+
+--------------------------------------------------------------------------------
+
+(expression
+  (addition
+    (expression
+      (addition
+        (expression
+          (addition
+            (expression
+              (number))
+            (expression
+              (number))))
+        (expression
+          (number))))
+    (expression
+      (number))))
+
+================================================================================
+Superaddition still parses
+================================================================================
+
+1 + + 1
+
+--------------------------------------------------------------------------------
+
+(expression
+  (superaddition
+    (expression
+      (number))
+    (expression
+      (number))))

test/fixtures/test_grammars/associativity_left_with_lower_precedence_shift/grammar.js

@@ -0,0 +1,16 @@
+export default grammar({
+    name: 'associativity_left_with_lower_precedence_shift',
+
+    rules: {
+        expression: $ => choice(
+            $.addition,
+            $.superaddition,
+            $.number,
+        ),
+
+        addition: $ => prec.left(1, seq($.expression, '+', $.expression)),
+        superaddition: $ => prec.right(seq($.expression, '+', '+', $.expression)),
+
+        number: _ => '1',
+    }
+});

test/fixtures/test_grammars/associativity_right_with_lower_precedence_shift/corpus.txt

@@ -0,0 +1,22 @@
+================================================================================
+Addition is right associative
+================================================================================
+
+1 + 1 + 1 + 1
+
+--------------------------------------------------------------------------------
+
+(expression
+  (addition
+    (expression
+      (number))
+    (expression
+      (addition
+        (expression
+          (number))
+        (expression
+          (addition
+            (expression
+              (number))
+            (expression
+              (number))))))))

test/fixtures/test_grammars/associativity_right_with_lower_precedence_shift/grammar.js

@@ -0,0 +1,15 @@
+module.exports = grammar({
+    name: 'associativity_right_with_lower_precedence_shift',
+  
+    rules: {
+      expression: $ => choice(
+        $.addition,
+        $.superaddition,
+        $.number
+      ),
+      addition: $ => prec.right(1, seq($.expression, "+", $.expression)),
+      superaddition: $ => prec.right(seq($.expression, "+", "+", $.expression)),
+      number: _ => "1"
+    }
+  });
+  

test/fixtures/test_grammars/external_lookahead_eof_boundary/corpus.txt

@@ -0,0 +1,13 @@
+=====================
+opened and closed span
+=====================
+``
+---
+(document (span (open_delim) (close_delim)))
+
+==================
+unclosed delimiter
+==================
+`
+---
+(document (unclosed_delim))

test/fixtures/test_grammars/external_lookahead_eof_boundary/grammar.js

@@ -0,0 +1,15 @@
+// External scanner whose token choice depends on `lexer->eof()`: at a
+// '`' it peeks past mark_end for a matching close, emitting open_delim
+// if one is found or unclosed_delim if eof is reached first. Inside an
+// open span the same '`' is emitted as close_delim.
+
+export default grammar({
+    name: 'external_lookahead_eof_boundary',
+
+    externals: $ => [$.open_delim, $.close_delim, $.unclosed_delim],
+
+    rules: {
+        document: $ => repeat(choice($.span, $.unclosed_delim)),
+        span: $ => seq($.open_delim, $.close_delim),
+    }
+});