build(deps): bump rquickjs to v0.13.0

Also use `IoError` more consistently throughout the rest of the project
where its straightforward to do so.

BREAKING CHANGE: Changes public error types for config, generate, and
loader crates.

.dockerignore

@@ -1,2 +0,0 @@
-target
-.git

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -2,6 +2,14 @@ name: Bug Report
 description: Report a problem
 type: Bug
 body:
+  - type: checkboxes
+    attributes:
+      label: AI Policy
+      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
+      options:
+        - label: I have read the AI Policy and this issue complies with it.
+          required: true
+
   - type: textarea
     attributes:
       label: "Problem"

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -2,6 +2,14 @@ name: Feature request
 description: Request an enhancement
 type: Feature
 body:
+  - type: checkboxes
+    attributes:
+      label: AI Policy
+      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
+      options:
+        - label: I have read the AI Policy and this issue complies with it.
+          required: true
+
   - type: markdown
     attributes:
       value: |

.github/actions/cache/action.yml

@@ -10,7 +10,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       id: cache
       with:
         path: |
@@ -21,6 +21,9 @@ runs:
           'lib/src/parser.h',
           'lib/src/array.h',
           'lib/src/alloc.h',
-          'lib/src/wasm/wasm-stdlib.h',
+          'lib/src/wasm-stdlib/external_scanner_stdlib.h',
+          'crates/loader/wasi-sdk-version',
+          'crates/loader/binaryen-version',
           'test/fixtures/grammars/*/**/src/*.c',
+          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}

.github/cliff.toml

@@ -69,4 +69,4 @@ sort_commits = "oldest"
 
 [remote.github]
 owner = "tree-sitter"
-repo  = "tree-sitter"
+repo = "tree-sitter"

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+### AI Policy
+
+- [ ] I have read the [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy) and this PR complies with it.
+- [ ] If AI tools were used: I have disclosed the tool and extent of usage below.
+
+<!-- If you used AI tools, state which tool and how it was used. Delete this section if not applicable. -->

.github/scripts/reviewers_remove.js

@@ -1,16 +0,0 @@
-module.exports = async ({ github, context }) => {
-  const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
-    owner: context.repo.owner,
-    repo: context.repo.repo,
-    pull_number: context.issue.number,
-  });
-
-  const reviewers = requestedReviewers.data.users.map((e) => e.login);
-
-  github.rest.pulls.removeRequestedReviewers({
-    owner: context.repo.owner,
-    repo: context.repo.repo,
-    pull_number: context.issue.number,
-    reviewers: reviewers,
-  });
-};

.github/scripts/wasm_stdlib.js

@@ -5,21 +5,31 @@ module.exports = async ({ github, context, core }) => {
   const owner = context.repo.owner;
   const repo = context.repo.repo;
 
-  const { data: files } = await github.rest.pulls.listFiles({
+  const files = await github.paginate(github.rest.pulls.listFiles, {
     owner,
     repo,
-    pull_number: prNumber
+    pull_number: prNumber,
+    per_page: 100
   });
 
   const changedFiles = files.map(file => file.filename);
 
-  const wasmStdLibSrc = 'crates/language/wasm/';
-  const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc));
+  const wasmStdLibSources = [
+    'lib/src/wasm-stdlib/external_scanner_allocator.c',
+    'lib/src/wasm-stdlib/imports.txt',
+    'lib/src/wasm-stdlib/libc.c',
+    'lib/src/wasm-stdlib/stdio.c'
+  ];
+  const dirChanged = changedFiles.some(file =>
+    wasmStdLibSources.includes(file) ||
+    file.startsWith('lib/src/wasm-stdlib/libc/ctype/') ||
+    file.startsWith('lib/src/wasm-stdlib/libc/string/')
+  );
 
   if (!dirChanged) return;
 
-  const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h';
+  const wasmStdLibHeader = 'lib/src/wasm-stdlib/external_scanner_stdlib.h';
   const requiredChanged = changedFiles.includes(wasmStdLibHeader);
 
-  if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`);
+  if (!requiredChanged) core.setFailed(`Changes detected in the Wasm stdlib sources but ${wasmStdLibHeader} was not modified.`);
 };

.github/workflows/backport.yml

@@ -14,17 +14,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
+        with:
+          persist-credentials: true
+          ref: ${{ github.event.pull_request.base.ref }}
 
       - name: Create app token
-        uses: actions/create-github-app-token@v2
+        uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ vars.BACKPORT_APP }}
           private-key: ${{ secrets.BACKPORT_KEY }}
 
       - name: Create backport PR
-        uses: korthout/backport-action@v4
+        uses: korthout/backport-action@v4.6.0
         with:
           pull_title: "${pull_title}"
           label_pattern: "^ci:backport ([^ ]+)$"

.github/workflows/bindgen.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1

.github/workflows/build.yml

@@ -26,12 +26,14 @@ jobs:
           - windows-x86
           - macos-arm64
           - macos-x64
+          - illumos-x64
           - wasm32
 
         include:
           # When adding a new `target`:
           # 1. Define a new platform alias above
           # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
+          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
           - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
           - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
           - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
@@ -44,6 +46,9 @@ jobs:
           - { platform: macos-x64       , target: x86_64-apple-darwin           , os: macos-15-intel   }
           - { platform: wasm32          , target: wasm32-unknown-unknown        , os: ubuntu-24.04     }
 
+          # illumos is not supported OOTB, it runs in a vm
+          - { platform: illumos-x64     , target: x86_64-unknown-illumos        , os: ubuntu-24.04     , vm: true , no-run: true }
+
           # Extra features
           - { platform: linux-arm64     , features: wasm }
           - { platform: linux-x64       , features: wasm , run-wasm-test: true }
@@ -68,7 +73,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7.0.1
 
     - name: Set up cross-compilation
       if: matrix.cross
@@ -90,7 +95,7 @@ jobs:
 
     - name: Cache Emscripten SDK
       if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      uses: actions/cache@v5
+      uses: actions/cache@v6
       with:
         path: emsdk
         key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
@@ -111,17 +116,22 @@ jobs:
 
     - name: Set up Node.js
       if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      uses: actions/setup-node@v6
+      uses: actions/setup-node@v7.0.0
       with:
         node-version: 24
         cache: npm
         cache-dependency-path: lib/binding_web/package-lock.json
 
     - name: Set up Rust
+      if: ${{ !matrix.vm }}
       uses: actions-rust-lang/setup-rust-toolchain@v1
       with:
         target: ${{ matrix.target }}
 
+    - name: Install Rust Wasm test target
+      if: matrix.run-wasm-test
+      run: rustup toolchain install nightly --profile minimal --component rust-src
+
     - name: Install cross-compilation toolchain
       if: matrix.cross
       run: |
@@ -191,7 +201,7 @@ jobs:
         WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
 
     - name: Build C library (make)
-      if: runner.os != 'Windows'
+      if: runner.os != 'Windows' && !matrix.vm
       run: |
         if [[ $PLATFORM == linux-arm ]]; then
           CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar
@@ -205,10 +215,10 @@ jobs:
         make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
       env:
         PLATFORM: ${{ matrix.platform }}
-        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
     - name: Build C library (CMake)
-      if: "!matrix.cross"
+      if: "!matrix.cross && !matrix.vm"
       run: |
         cmake -S . -B build/static \
           -DBUILD_SHARED_LIBS=OFF \
@@ -227,6 +237,20 @@ jobs:
         CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
         WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
 
+    - name: Build C library and Rust crate (illumos gmake)
+      if: matrix.platform == 'illumos-x64'
+      uses: vmactions/omnios-vm@v1.3.6
+      with:
+        release: r151056-build
+        copyback: false
+        prepare: |
+          pkg install -q build-essential || [ $? -eq 4 ]
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
+        run: |
+          . "$HOME/.cargo/env"
+          gmake -j
+          cargo build -p tree-sitter
+
     - name: Build Wasm library
       if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       shell: bash
@@ -245,7 +269,7 @@ jobs:
       run: cargo check --no-default-features --target='${{ matrix.target }}'
 
     - name: Build target
-      if: "!inputs.run-test"
+      if: "!inputs.run-test && !matrix.vm"
       run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
       env:
         PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
@@ -269,15 +293,19 @@ jobs:
 
     - name: Run main tests
       if: inputs.run-test && !matrix.no-run
-      run: cargo test --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
+      run: cargo test --workspace --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
 
     - name: Run Wasm tests
       if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
       run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
 
+    - name: Run Rust Wasm web test
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-rust-wasm-web
+
     - name: Upload CLI artifact
       if: "!inputs.run-test && !matrix.no-run"
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: tree-sitter.${{ matrix.platform }}
         path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
@@ -286,7 +314,7 @@ jobs:
 
     - name: Upload Wasm artifacts
       if: "!inputs.run-test && matrix.platform == 'linux-x64'"
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
         name: tree-sitter.wasm
         path: |

.github/workflows/ci.yml

@@ -1,21 +1,14 @@
 name: CI
 
 on:
-  pull_request:
-    paths-ignore:
-      - docs/**
-      - "**/README.md"
-      - CONTRIBUTING.md
-      - LICENSE
-      - cli/src/templates
   push:
-    branches: [master]
-    paths-ignore:
-      - docs/**
-      - "**/README.md"
-      - CONTRIBUTING.md
-      - LICENSE
-      - cli/src/templates
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
+  pull_request:
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -26,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -34,10 +27,19 @@ jobs:
           toolchain: stable
           components: clippy, rustfmt
 
-      - name: Lint files
-        run: |
-          make lint
-          make lint-web
+      - name: Lint Rust files
+        run: make lint
+
+      - name: Install Taplo
+        uses: taiki-e/install-action@v2
+        with:
+          tool: taplo@0.10.0
+
+      - name: Lint TOML files
+        run: make lint-toml
+
+      - name: Lint web files
+        run: make lint-web
 
   sanitize:
     uses: ./.github/workflows/sanitize.yml

.github/workflows/crate_versions.yml

@@ -11,7 +11,7 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Set up Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1

.github/workflows/docs.yml

@@ -16,35 +16,29 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
-
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+        uses: actions/checkout@v7.0.1
 
       - name: Install mdbook
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
-          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.4.52 --jq "$jq_expr")
+          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.5.4 --jq "$jq_expr")
           mkdir mdbook
           curl -sSL "$url" | tar -xz -C mdbook
           printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
 
-      - name: Install mdbook-admonish
-        run: cargo install mdbook-admonish
-
       - name: Build Book
         run: mdbook build docs
 
       - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@v6
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v4
+        uses: actions/upload-pages-artifact@v5
         with:
           path: docs/book
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v5

.github/workflows/nvim_ts.yml

@@ -3,6 +3,7 @@ name: nvim-treesitter parser tests
 on:
   pull_request:
     paths:
+      - 'lib/**'
       - 'crates/cli/**'
       - 'crates/config/**'
       - 'crates/generate/**'
@@ -25,34 +26,54 @@ jobs:
     name: ${{ matrix.os }} - ${{ matrix.type }}
     runs-on: ${{ matrix.os }}
     env:
-      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }}
+      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim.exe' || 'nvim' }}
+      NVIM_TAG: stable
+      NVIM_DIR: neovim
       NVIM_TS_DIR: nvim-treesitter
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7.0.1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build --profile optimize
 
-      - uses: actions/checkout@v6
+      - name: Clone Neovim
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: neovim/neovim
+          ref: ${{ env.NVIM_TAG }}
+          path: ${{ env.NVIM_DIR }}
+
+      - if: runner.os != 'Windows'
+        name: Setup environment (Posix)
+        run: |
+          echo ${{ github.workspace }}/target/optimize >> "$GITHUB_PATH"
+          echo ${{ github.workspace }}/neovim/build/bin >> "$GITHUB_PATH"
+          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$GITHUB_ENV"
+
+      - if: runner.os == 'Windows'
+        name: Setup environment (why can't you just be normal?!)
+        run: |
+          ${{ env.NVIM_DIR }}/.github/scripts/env.ps1
+          echo ${{ github.workspace }}/target/optimize >> "$env:GITHUB_PATH"
+          echo ${{ github.workspace }}/neovim/build/bin >> "$env:GITHUB_PATH"
+          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$env:GITHUB_ENV"
+
+      - name: Build Neovim
+        working-directory: ${{ env.NVIM_DIR }}
+        run: |
+          cmake -S cmake.deps -B .deps -G Ninja -D CMAKE_BUILD_TYPE=Release -D TREESITTER_URL=https://github.com/tree-sitter/tree-sitter/archive/${{ github.event.pull_request.head.sha }}.tar.gz -D DEPS_IGNORE_SHA=TRUE
+          cmake --build .deps --config Release
+          cmake -B build -G Ninja -D CMAKE_BUILD_TYPE=Release
+          cmake --build build --config Release
+
+      - name: Clone nvim-treesitter
+        uses: actions/checkout@v7.0.1
         with:
           repository: nvim-treesitter/nvim-treesitter
           path: ${{ env.NVIM_TS_DIR }}
-          ref: main
-
-      - if: runner.os != 'Windows'
-        run: echo ${{ github.workspace }}/target/release >> $GITHUB_PATH
-
-      - if: runner.os == 'Windows'
-        run: echo ${{ github.workspace }}/target/release >> $env:GITHUB_PATH
-
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - run: cargo build --release
-      - uses: ilammy/msvc-dev-cmd@v1
-
-      - name: Install and prepare Neovim
-        run: bash ./scripts/ci-install.sh
-        working-directory: ${{ env.NVIM_TS_DIR }}
 
       - if: matrix.type == 'generate'
         name: Generate and compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=2
+        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=10
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash
 
@@ -63,7 +84,13 @@ jobs:
         shell: bash
 
       - if: "!cancelled()"
-        name: Check query files
+        name: Test parsers
+        run: $NVIM -l ./scripts/check-parsers.lua
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: "!cancelled()"
+        name: Test queries
         run: $NVIM -l ./scripts/check-queries.lua
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash

.github/workflows/release.yml

@@ -1,7 +1,14 @@
 name: Release
 
 on:
+  schedule:
+    - cron: '5 5 * * *'
   workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Tag name for release'
+        required: false
+        default: nightly
   push:
     tags:
       - v[0-9]+.[0-9]+.[0-9]+
@@ -21,11 +28,22 @@ jobs:
       attestations: write
       contents: write
     steps:
+      - if: github.event_name == 'workflow_dispatch'
+        env:
+          TAG_NAME: ${{ github.event.inputs.tag_name }}
+        run: echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_ENV
+
+      - if: github.event_name == 'schedule'
+        run: echo 'TAG_NAME=nightly' >> $GITHUB_ENV
+
+      - if: github.event_name == 'push'
+        run: echo "TAG_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
+
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           path: artifacts
 
@@ -44,28 +62,41 @@ jobs:
 
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
+            chmod +x $exe
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
+            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
 
       - name: Generate attestations
-        uses: actions/attest-build-provenance@v3
+        uses: actions/attest-build-provenance@v4
         with:
           subject-path: |
             target/tree-sitter-*.gz
+            target/tree-sitter-cli-*.zip
             target/web-tree-sitter.tar.gz
 
+      - if: env.TAG_NAME == 'nightly'
+        run: |
+          echo 'PRERELEASE=--prerelease' >> $GITHUB_ENV
+          gh release delete nightly --yes || true
+          git push https://${GITHUB_ACTOR}:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY} :nightly || true
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Create release
         run: |-
-          gh release create $GITHUB_REF_NAME \
+          gh release create ${{ env.TAG_NAME }} $PRERELEASE \
             target/tree-sitter-*.gz \
+            target/tree-sitter-cli-*.zip \
             target/web-tree-sitter.tar.gz
         env:
           GH_TOKEN: ${{ github.token }}
 
   crates_io:
     name: Publish packages to Crates.io
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
     environment: crates
     permissions:
@@ -74,7 +105,7 @@ jobs:
     needs: release
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Set up Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -90,6 +121,7 @@ jobs:
 
   npm:
     name: Publish packages to npmjs.com
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
     environment: npm
     permissions:
@@ -102,10 +134,10 @@ jobs:
         directory: [crates/cli/npm, lib/binding_web]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Set up Node
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7.0.0
         with:
           node-version: 24
           registry-url: https://registry.npmjs.org

.github/workflows/response.yml

@@ -17,13 +17,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout script
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts/close_unresponsive.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/close_unresponsive.js')
@@ -35,13 +35,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout script
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts/remove_response_label.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/remove_response_label.js')

.github/workflows/reviewers_remove.yml

@@ -11,15 +11,21 @@ jobs:
   remove-reviewers:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout script
-        uses: actions/checkout@v6
-        with:
-          sparse-checkout: .github/scripts/reviewers_remove.js
-          sparse-checkout-cone-mode: false
-
-      - name: Run script
-        uses: actions/github-script@v8
+      - name: Remove reviewers
+        uses: actions/github-script@v9
         with:
           script: |
-            const script = require('./.github/scripts/reviewers_remove.js')
-            await script({github, context})
+            const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+
+            const reviewers = requestedReviewers.data.users.map((e) => e.login);
+
+            github.rest.pulls.removeRequestedReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+              reviewers: reviewers,
+            });

.github/workflows/sanitize.yml

@@ -15,7 +15,7 @@ jobs:
       TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Install UBSAN library
         run: sudo apt-get update -y && sudo apt-get install -y libubsan1

.github/workflows/spam.yml

@@ -16,13 +16,13 @@ jobs:
     if: github.event.label.name == 'spam'
     steps:
       - name: Checkout script
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts/close_spam.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/close_spam.js')

.github/workflows/wasm_exports.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -31,7 +31,7 @@ jobs:
       - name: Build C library (make)
         run: make -j CFLAGS="$CFLAGS"
         env:
-          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
       - name: Build Wasm Library
         working-directory: lib/binding_web

.github/workflows/wasm_stdlib.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
 
       - name: Check directory changes
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;

.taplo.toml

@@ -1,20 +1,19 @@
 [formatting]
-align_entries         = true
-column_width          = 100
-compact_arrays        = false
+column_width = 100
+compact_arrays = false
 reorder_inline_tables = true
-reorder_keys          = true
+reorder_keys = true
 
 [[rule]]
 include = [ "**/Cargo.toml" ]
-keys    = [ "package" ]
+keys = [ "package" ]
 
 [rule.formatting]
 reorder_keys = false
 
 [[rule]]
 include = [ "**/Cargo.toml" ]
-keys    = [ "profile" ]
+keys = [ "profile" ]
 
 [rule.formatting]
 reorder_keys = false

.zed/settings.json

@@ -1,11 +0,0 @@
-{
-  "lsp": {
-    "rust-analyzer": {
-      "initialization_options": {
-        "cargo": {
-          "features": "all"
-        }
-      }
-    }
-  }
-}

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.27.0"
+        VERSION "0.28.0"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
@@ -33,7 +33,8 @@ if(MSVC)
 else()
   target_compile_options(tree-sitter PRIVATE
                          -Wall -Wextra -Wshadow -Wpedantic
-                         -Werror=incompatible-pointer-types)
+                         -Werror=incompatible-pointer-types
+                         -Werror=strict-aliasing -Wstrict-aliasing=2)
 endif()
 
 if(TREE_SITTER_FEATURE_WASM)

Cargo.toml

@@ -14,22 +14,22 @@ members = [
 resolver = "2"
 
 [workspace.package]
-authors      = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>" ]
-categories   = [ "command-line-utilities", "parsing" ]
-edition      = "2024"
-homepage     = "https://tree-sitter.github.io/tree-sitter"
-keywords     = [ "incremental", "parsing" ]
-license      = "MIT"
-repository   = "https://github.com/tree-sitter/tree-sitter"
+authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>" ]
+categories = [ "command-line-utilities", "parsing" ]
+edition = "2024"
+homepage = "https://tree-sitter.github.io/tree-sitter"
+keywords = [ "incremental", "parsing" ]
+license = "MIT"
+repository = "https://github.com/tree-sitter/tree-sitter"
 rust-version = "1.90"
-version      = "0.27.0"
+version = "0.28.0"
 
 [workspace.lints.clippy]
-cargo     = { level = "warn", priority = -1 }
+cargo = { level = "warn", priority = -1 }
 dbg_macro = "deny"
-nursery   = { level = "warn", priority = -1 }
-pedantic  = { level = "warn", priority = -1 }
-todo      = "deny"
+nursery = { level = "warn", priority = -1 }
+pedantic = { level = "warn", priority = -1 }
+todo = "deny"
 
 # The lints below are a specific subset of the pedantic+nursery lints
 # that we explicitly allow in the tree-sitter codebase because they either:
@@ -39,49 +39,50 @@ todo      = "deny"
 # 3. Worsen the code
 
 cast_possible_truncation = "allow"
-cast_possible_wrap       = "allow"
-cast_precision_loss      = "allow"
-cast_sign_loss           = "allow"
-if_not_else              = "allow"
-items_after_statements   = "allow"
-missing_errors_doc       = "allow"
-missing_panics_doc       = "allow"
-multiple_crate_versions  = "allow"
-option_if_let_else       = "allow"
-similar_names            = "allow"
-string_lit_as_bytes      = "allow"
-struct_excessive_bools   = "allow"
-too_many_lines           = "allow"
+cast_possible_wrap = "allow"
+cast_precision_loss = "allow"
+cast_sign_loss = "allow"
+if_not_else = "allow"
+items_after_statements = "allow"
+missing_errors_doc = "allow"
+missing_panics_doc = "allow"
+multiple_crate_versions = "allow"
+option_if_let_else = "allow"
+similar_names = "allow"
+string_lit_as_bytes = "allow"
+struct_excessive_bools = "allow"
+too_many_lines = "allow"
+tuple_array_conversions = "allow"
 
 [workspace.lints.rust]
 mismatched_lifetime_syntaxes = "allow"
 
 [profile.optimize]
-inherits      = "release"
-codegen-units = 1         # Maximum size reduction optimizations.
-lto           = true      # Link-time optimization.
-opt-level     = 3         # Optimization level 3.
-strip         = true      # Automatically strip symbols from the binary.
+inherits = "release"
+codegen-units = 1    # Maximum size reduction optimizations.
+lto = true           # Link-time optimization.
+opt-level = 3        # Optimization level 3.
+strip = true         # Automatically strip symbols from the binary.
 
 [profile.size]
-inherits  = "optimize"
-opt-level = "s"        # Optimize for size.
+inherits = "optimize"
+opt-level = "s"       # Optimize for size.
 
 [profile.release-dev]
-inherits         = "release"
-codegen-units    = 256
-debug            = true
+inherits = "release"
+codegen-units = 256
+debug = true
 debug-assertions = true
-incremental      = true
-lto              = false
-overflow-checks  = true
+incremental = true
+lto = false
+overflow-checks = true
 
 [workspace.dependencies]
 ansi_colours = "1.2.3"
-anstyle = "1.0.13"
+anstyle = "1.0.14"
 anyhow = "1.0.102"
-bstr   = "1.12.1"
-cc     = "1.2.56"
+bstr = "1.12.1"
+cc = "1.2.63"
 clap = { features = [
   "cargo",
   "derive",
@@ -90,31 +91,32 @@ clap = { features = [
   "string",
   "unstable-styles",
 ], version = "4.5.58" }
-clap_complete = "4.5.66"
+clap_complete = "4.6.3"
 clap_complete_nushell = "4.5.10"
 crc32fast = "1.5.0"
-ctor      = "0.6.3"
-ctrlc     = { features = [ "termination" ], version = "3.5.2" }
+ctor = "0.6.3"
+ctrlc = { features = [ "termination" ], version = "3.5.2" }
 dialoguer = { features = [ "fuzzy-select" ], version = "0.12.0" }
 etcetera = "0.11.0"
 glob = "0.3.3"
+hashbrown = { default-features = false, version = "0.17.1" }
 heck = "0.5.0"
 html-escape = "0.2.13"
 indexmap = "2.13.0"
 indoc = "2.0.7"
 libloading = "0.9.0"
-log = { features = [ "std" ], version = "0.4.29" }
-memchr = "2.8.0"
+log = { features = [ "std" ], version = "0.4.30" }
+memchr = "2.8.1"
 once_cell = "1.21.3"
 pretty_assertions = "1.4.1"
-rand         = "0.10.0"
-regex        = "1.12.3"
+rand = "0.10.1"
+regex = "1.12.3"
 regex-syntax = "0.8.9"
 rustc-hash = "2.1.1"
-schemars   = "1.2.1"
-semver     = { features = [ "serde" ], version = "1.0.27" }
-serde      = { features = [ "derive" ], version = "1.0.228" }
-serde_json = { features = [ "preserve_order" ], version = "1.0.149" }
+schemars = "1.2.1"
+semver = { features = [ "serde" ], version = "1.0.27" }
+serde = { features = [ "derive" ], version = "1.0.228" }
+serde_json = { features = [ "preserve_order" ], version = "1.0.150" }
 similar = "2.7.0"
 streaming-iterator = "0.1.9"
 tempfile = "3.25.0"
@@ -124,13 +126,13 @@ topological-sort = "0.2.2"
 unindent = "0.2.4"
 walkdir = "2.5.0"
 wasmparser = "0.244.0"
-webbrowser = "1.1.0"
+webbrowser = "1.2.1"
 
-tree-sitter           = { path = "./lib", version = "0.27.0" }
-tree-sitter-config    = { path = "./crates/config", version = "0.27.0" }
-tree-sitter-generate  = { path = "./crates/generate", version = "0.27.0" }
-tree-sitter-highlight = { path = "./crates/highlight", version = "0.27.0" }
-tree-sitter-loader    = { path = "./crates/loader", version = "0.27.0" }
-tree-sitter-tags      = { path = "./crates/tags", version = "0.27.0" }
+tree-sitter = { path = "./lib", version = "0.28.0" }
+tree-sitter-config = { path = "./crates/config", version = "0.28.0" }
+tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.28.0" }
+tree-sitter-highlight = { path = "./crates/highlight", version = "0.28.0" }
+tree-sitter-loader = { path = "./crates/loader", version = "0.28.0" }
+tree-sitter-tags = { path = "./crates/tags", version = "0.28.0" }
 
-tree-sitter-language = { path = "./crates/language", version = "0.1" }
+tree-sitter-language = { path = "./crates/language", version = "0.1.8" }

Dockerfile

@@ -1,10 +0,0 @@
-FROM rust:1.76-buster
-
-WORKDIR /app
-
-RUN apt-get update
-RUN apt-get install -y nodejs
-
-COPY . .
-
-CMD cargo test --all-features

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.27.0
+VERSION := 0.28.0
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -22,7 +22,7 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
 override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
@@ -129,8 +129,13 @@ lint-web:
 	npm --prefix lib/binding_web ci
 	npm --prefix lib/binding_web run lint
 
+lint-toml:
+	taplo check
+	taplo format --check --diff
+
 format:
 	cargo fmt --all
+	taplo format
 
 changelog:
 	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)

Package.swift

@@ -1,35 +0,0 @@
-// swift-tools-version: 5.8
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
-import PackageDescription
-
-let package = Package(
-    name: "TreeSitter",
-    products: [
-        // Products define the executables and libraries a package produces, and make them visible to other packages.
-        .library(
-            name: "TreeSitter",
-            targets: ["TreeSitter"]),
-    ],
-    targets: [
-        .target(name: "TreeSitter",
-                path: "lib",
-                exclude: [
-                        "src/unicode/ICU_SHA",
-                        "src/unicode/README.md",
-                        "src/unicode/LICENSE",
-                        "src/wasm/stdlib-symbols.txt",
-                        "src/lib.c",
-                ],
-                sources: ["src"],
-                publicHeadersPath: "include",
-                cSettings: [
-                        .headerSearchPath("src"),
-                        .define("_POSIX_C_SOURCE", to: "200112L"),
-                        .define("_DEFAULT_SOURCE"),
-                        .define("_BSD_SOURCE"),
-                        .define("_DARWIN_C_SOURCE"),
-                ]),
-    ],
-    cLanguageStandard: .c11
-)

build.zig

@@ -4,51 +4,56 @@ pub fn build(b: *std.Build) !void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
+    var threaded: std.Io.Threaded = .init(b.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
     const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
     const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
     const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
 
+    var tree_sitter = b.createModule(.{
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .pic = if (shared) true else null,
+    });
     const lib: *std.Build.Step.Compile = b.addLibrary(.{
         .name = "tree-sitter",
         .linkage = if (shared) .dynamic else .static,
-        .root_module = b.createModule(.{
-            .target = target,
-            .optimize = optimize,
-            .link_libc = true,
-            .pic = if (shared) true else null,
-        }),
+        .root_module = tree_sitter,
     });
 
     if (amalgamated) {
-        lib.addCSourceFile(.{
+        tree_sitter.addCSourceFile(.{
             .file = b.path("lib/src/lib.c"),
             .flags = &.{"-std=c11"},
         });
     } else {
-        const files = try findSourceFiles(b);
+        const files = try findSourceFiles(b, io);
         defer b.allocator.free(files);
-        lib.addCSourceFiles(.{
+        tree_sitter.addCSourceFiles(.{
             .root = b.path("lib/src"),
             .files = files,
             .flags = &.{"-std=c11"},
         });
     }
 
-    lib.addIncludePath(b.path("lib/include"));
-    lib.addIncludePath(b.path("lib/src"));
-    lib.addIncludePath(b.path("lib/src/wasm"));
+    tree_sitter.addIncludePath(b.path("lib/include"));
+    tree_sitter.addIncludePath(b.path("lib/src"));
+    tree_sitter.addIncludePath(b.path("lib/src/wasm"));
 
-    lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
-    lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
-    lib.root_module.addCMacro("_BSD_SOURCE", "");
-    lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
+    tree_sitter.addCMacro("_POSIX_C_SOURCE", "200112L");
+    tree_sitter.addCMacro("_DEFAULT_SOURCE", "");
+    tree_sitter.addCMacro("_BSD_SOURCE", "");
+    tree_sitter.addCMacro("_DARWIN_C_SOURCE", "");
 
     if (wasm) {
         if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
-            lib.root_module.addCMacro("TREE_SITTER_FEATURE_WASM", "");
-            lib.addSystemIncludePath(wasmtime.path("include"));
-            lib.addLibraryPath(wasmtime.path("lib"));
-            if (shared) lib.linkSystemLibrary("wasmtime");
+            tree_sitter.addCMacro("TREE_SITTER_FEATURE_WASM", "");
+            tree_sitter.addSystemIncludePath(wasmtime.path("include"));
+            tree_sitter.addLibraryPath(wasmtime.path("lib"));
+            if (shared) tree_sitter.linkSystemLibrary("wasmtime", .{});
         }
     }
 
@@ -122,14 +127,14 @@ pub fn wasmtimeDep(target: std.Target) []const u8 {
     );
 }
 
-fn findSourceFiles(b: *std.Build) ![]const []const u8 {
+fn findSourceFiles(b: *std.Build, io: std.Io) ![]const []const u8 {
     var sources: std.ArrayListUnmanaged([]const u8) = .empty;
 
-    var dir = try b.build_root.handle.openDir("lib/src", .{ .iterate = true });
+    var dir = try b.build_root.handle.openDir(io, "lib/src", .{ .iterate = true });
     var iter = dir.iterate();
-    defer dir.close();
+    defer dir.close(io);
 
-    while (try iter.next()) |entry| {
+    while (try iter.next(io)) |entry| {
         if (entry.kind != .file) continue;
         const file = entry.name;
         const ext = std.fs.path.extension(file);

build.zig.zon

@@ -1,8 +1,8 @@
 .{
     .name = .tree_sitter,
     .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.27.0",
-    .minimum_zig_version = "0.14.1",
+    .version = "0.28.0",
+    .minimum_zig_version = "0.16.0",
     .paths = .{
         "build.zig",
         "build.zig.zon",
@@ -13,83 +13,83 @@
     },
     .dependencies = .{
         .wasmtime_c_api_aarch64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIfPIgdw2YnV3QyiFQ2NHdrxrXzzCdjYJyxJDOta",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAIp_mQVzQOITXcYcWxYLJkvB1W1SvLlrdiU2G7fj",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAIt97QZi7Pf7nNJ2mVY6uxA80Klyuvvtop3pLMRK",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAMztsgU5Aj4oI3MRHXJVe5rW72op-kT_78I3kZVM",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAAO48QQf91w9RmmUDHTja8DrXZA1n6Bmc8waW3qe",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AANZxOwT27sdrKxDDGGKsiwtcZlHy204xAWNIgDBH",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAI196wa9pwADoA2RbCDp5F7bKQg1iOPq6gIh8-FH",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAJL1zQW9yxC98uc60lSuVUHhH77QHTto0zwIQnBj",
             .lazy = true,
         },
         .wasmtime_c_api_aarch64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-windows-c-api.zip",
-            .hash = "N-V-__8AAC9u4wXfqd1Q6XyQaC8_DbQZClXux60Vu5743N05",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-windows-c-api.zip",
+            .hash = "N-V-__8AAHRCtQU93hJcRFOgVcof3IQpRV9stT2Pp54wpJc2",
             .lazy = true,
         },
         .wasmtime_c_api_armv7_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-armv7-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAHXe8gWs3s83Cc5G6SIq0_jWxj8fGTT5xG4vb6-x",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-armv7-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAJaW6gT8QdULOU0jxX4a_DOCA5YD6cxWBC8IqhQF",
             .lazy = true,
         },
         .wasmtime_c_api_i686_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAN2pzgUUfulRCYnipSfis9IIYHoTHVlieLRmKuct",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-linux-c-api.tar.xz",
+            .hash = "N-V-__8AANguMgVX4XMhdOVkdj4yfFKXrG8RTgZDs3nQB8J8",
             .lazy = true,
         },
         .wasmtime_c_api_i686_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-windows-c-api.zip",
-            .hash = "N-V-__8AAJu0YAUUTFBLxFIOi-MSQVezA6MMkpoFtuaf2Quf",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-windows-c-api.zip",
+            .hash = "N-V-__8AANY9ggXg4rK2_1o3EIlrCq124l5RfykPv-DPforq",
             .lazy = true,
         },
         .wasmtime_c_api_riscv64gc_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-riscv64gc-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAG8m-gc3E3AIImtTZ3l1c7HC6HUWazQ9OH5KACX4",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-riscv64gc-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAPDtCAdQ0dD9Rs-qWl-kPr2c7L3PVsUVjwy12Iz1",
             .lazy = true,
         },
         .wasmtime_c_api_s390x_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-s390x-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAH314gd-gE4IBp2uvAL3gHeuW1uUZjMiLLeUdXL_",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-s390x-linux-c-api.tar.xz",
+            .hash = "N-V-__8AANA3BwY1ZOoGCWCR_tTY9G1vfIX128RGVxufN3ov",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIPNRwfNkznebrcGb0IKUe7f35bkuZEYOjcx6q3f",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAF4AIgY0ltjevj1ybGfvMU1ErPRnNve5X1TmvCru",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAI8EDwcyTtk_Afhk47SEaqfpoRqGkJeZpGs69ChF",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAIR0cAbjf3DkrTbu81Oq_zociz-0lCpb5DR0lIC9",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAGtGNgVaOpHSxC22IjrampbRIy6lLwscdcAE8nG1",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAFJ4lgRgCBnYdz8-Yfc4hLve45Hv-0RICAEuCp4s",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_mingw = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-mingw-c-api.zip",
-            .hash = "N-V-__8AAPS2PAbVix50L6lnddlgazCPTz3whLUFk1qnRtnZ",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-mingw-c-api.zip",
+            .hash = "N-V-__8AAMxZxQZUpp1cU8J5zgLiMNq4e4dy0hcchlQFy03J",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAF-WEQe0nzvi09PgusM5i46FIuCKJmIDWUleWgQ3",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAN5pWgZrZBt8VYWkN82WjyFe3DkGcE7uLn3jpt38",
             .lazy = true,
         },
         .wasmtime_c_api_x86_64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-windows-c-api.zip",
-            .hash = "N-V-__8AAKGNXwbpJQsn0_6kwSIVDDWifSg8cBzf7T2RzsC9",
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-windows-c-api.zip",
+            .hash = "N-V-__8AAEIJkgaVHFETgakbognNUpFELuV17vpjw6NjsWhQ",
             .lazy = true,
         },
     },

crates/cli/Cargo.toml

@@ -1,18 +1,17 @@
 [package]
-name                   = "tree-sitter-cli"
-version.workspace      = true
-description            = "CLI tool for developing, testing, and using Tree-sitter parsers"
-authors.workspace      = true
-edition.workspace      = true
+name = "tree-sitter-cli"
+version.workspace = true
+description = "CLI tool for developing, testing, and using Tree-sitter parsers"
+authors.workspace = true
+edition.workspace = true
 rust-version.workspace = true
-readme                 = "README.md"
-homepage.workspace     = true
-repository.workspace   = true
-documentation          = "https://docs.rs/tree-sitter-cli"
-license.workspace      = true
-keywords.workspace     = true
-categories.workspace   = true
-include                = [ "build.rs", "README.md", "LICENSE", "benches/*", "src/**" ]
+homepage.workspace = true
+repository.workspace = true
+documentation = "https://docs.rs/tree-sitter-cli"
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+include = [ "build.rs", "README.md", "LICENSE", "benches/*", "src/**" ]
 
 [lints]
 workspace = true
@@ -21,63 +20,96 @@ workspace = true
 path = "src/tree_sitter_cli.rs"
 
 [[bin]]
-doc  = false
+doc = false
 name = "tree-sitter"
 path = "src/main.rs"
 
 [[bench]]
 harness = false
-name    = "benchmark"
+name = "benchmark"
 
 [features]
 default = [ "qjs-rt" ]
-qjs-rt  = [ "tree-sitter-generate/qjs-rt" ]
-wasm    = [ "tree-sitter/wasm", "tree-sitter-loader/wasm" ]
+qjs-rt = [ "tree-sitter-generate/qjs-rt" ]
+wasm = [ "tree-sitter/wasm", "tree-sitter-loader/wasm" ]
 
 [dependencies]
-ansi_colours.workspace          = true
-anstyle.workspace               = true
-anyhow.workspace                = true
-bstr.workspace                  = true
-clap.workspace                  = true
-clap_complete.workspace         = true
+ansi_colours.workspace = true
+anstyle.workspace = true
+anyhow.workspace = true
+bstr.workspace = true
+clap.workspace = true
+clap_complete.workspace = true
 clap_complete_nushell.workspace = true
-crc32fast.workspace             = true
-ctor.workspace                  = true
-ctrlc.workspace                 = true
-dialoguer.workspace             = true
-glob.workspace                  = true
-heck.workspace                  = true
-html-escape.workspace           = true
-indoc.workspace                 = true
-log.workspace                   = true
-memchr.workspace                = true
-rand.workspace                  = true
-regex.workspace                 = true
-schemars.workspace              = true
-semver.workspace                = true
-serde.workspace                 = true
-serde_json.workspace            = true
-similar.workspace               = true
-streaming-iterator.workspace    = true
-thiserror.workspace             = true
-tiny_http.workspace             = true
-walkdir.workspace               = true
-wasmparser.workspace            = true
-webbrowser.workspace            = true
+crc32fast.workspace = true
+ctor.workspace = true
+ctrlc.workspace = true
+dialoguer.workspace = true
+glob.workspace = true
+heck.workspace = true
+html-escape.workspace = true
+indoc.workspace = true
+log.workspace = true
+memchr.workspace = true
+rand.workspace = true
+regex.workspace = true
+schemars.workspace = true
+semver.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+similar.workspace = true
+streaming-iterator.workspace = true
+thiserror.workspace = true
+tiny_http.workspace = true
+walkdir.workspace = true
+wasmparser.workspace = true
+webbrowser.workspace = true
 
-tree-sitter.workspace           = true
-tree-sitter-config.workspace    = true
-tree-sitter-generate.workspace  = true
+tree-sitter.workspace = true
+tree-sitter-config.workspace = true
+tree-sitter-generate = { features = [ "load" ], workspace = true }
 tree-sitter-highlight.workspace = true
-tree-sitter-loader.workspace    = true
-tree-sitter-tags.workspace      = true
+tree-sitter-loader.workspace = true
+tree-sitter-tags.workspace = true
 
 [dev-dependencies]
-encoding_rs            = "0.8.35"
+encoding_rs = "0.8.35"
 tree_sitter_proc_macro = { package = "tree-sitter-tests-proc-macro", path = "src/tests/proc_macro" }
-widestring             = "1.2.1"
+widestring = "1.2.1"
 
 pretty_assertions.workspace = true
-tempfile.workspace          = true
-unindent.workspace          = true
+tempfile.workspace = true
+unindent.workspace = true
+
+[package.metadata.binstall]
+pkg-fmt = "zip"
+
+[package.metadata.binstall.overrides.aarch64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.armv7-unknown-linux-gnueabihf]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.powerpc64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-powerpc64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-x64{ archive-suffix }"

crates/cli/README.md

@@ -12,16 +12,15 @@ The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars fr
 
 ### Installation
 
-You can install the `tree-sitter-cli` with `cargo`:
+You can install the `tree-sitter-cli` with [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall):
 
 ```sh
-cargo install --locked tree-sitter-cli
+cargo binstall tree-sitter-cli
 ```
 
-or with `npm`:
-
+or you can build it from source:
 ```sh
-npm install tree-sitter-cli
+cargo install --locked tree-sitter-cli
 ```
 
 You can also download a pre-built binary for your platform from [the releases page].

crates/cli/benches/benchmark.rs

@@ -2,7 +2,6 @@ use std::{
     collections::BTreeMap,
     env, fs,
     path::{Path, PathBuf},
-    str,
     sync::LazyLock,
     time::Instant,
 };
@@ -10,6 +9,8 @@ use std::{
 use anyhow::Context;
 use log::info;
 use tree_sitter::{Language, Parser, Query};
+#[cfg(feature = "wasm")]
+use tree_sitter::{WasmStore, wasmtime};
 use tree_sitter_loader::{CompileConfig, Loader};
 
 include!("../src/tests/helpers/dirs.rs");
@@ -21,8 +22,11 @@ static EXAMPLE_FILTER: LazyLock<Option<String>> =
 static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
     env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
 });
+static WASM: LazyLock<bool> = LazyLock::new(|| env::var_os("TREE_SITTER_BENCHMARK_WASM").is_some());
 static TEST_LOADER: LazyLock<Loader> =
     LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
+#[cfg(feature = "wasm")]
+static WASM_ENGINE: LazyLock<wasmtime::Engine> = LazyLock::new(Default::default);
 
 #[expect(
     clippy::type_complexity,
@@ -95,7 +99,11 @@ fn main() {
         }
 
         info!("\nLanguage: {language_name}");
-        let language = get_language(language_path);
+        let language = if *WASM {
+            get_wasm_language(language_name, &mut parser)
+        } else {
+            get_language(language_path)
+        };
         parser.set_language(&language).unwrap();
 
         info!("  Constructing Queries");
@@ -215,3 +223,32 @@ fn get_language(path: &Path) -> Language {
         .with_context(|| format!("Failed to load language at path {}", src_path.display()))
         .unwrap()
 }
+
+#[cfg(feature = "wasm")]
+fn get_wasm_language(language_name: &str, parser: &mut Parser) -> Language {
+    let wasm_language_name = language_name.replace('-', "_");
+    let wasm_path = ROOT_DIR
+        .join("target")
+        .join("release")
+        .join(format!("tree-sitter-{language_name}.wasm"));
+    let wasm = fs::read(&wasm_path)
+        .with_context(|| {
+            format!(
+                "Failed to read {}. Generate Wasm fixtures with `cargo xtask generate-fixtures --wasm`",
+                wasm_path.display()
+            )
+        })
+        .unwrap();
+    let mut store = WasmStore::new(&WASM_ENGINE).expect("Failed to create Wasm store");
+    let language = store
+        .load_language(&wasm_language_name, &wasm)
+        .with_context(|| format!("Failed to load Wasm language at {}", wasm_path.display()))
+        .unwrap();
+    parser.set_wasm_store(store).unwrap();
+    language
+}
+
+#[cfg(not(feature = "wasm"))]
+fn get_wasm_language(_language_name: &str, _parser: &mut Parser) -> Language {
+    panic!("Wasm benchmarking requires the `wasm` feature");
+}

crates/cli/eslint/package-lock.json

@@ -135,29 +135,43 @@
       }
     },
     "node_modules/@humanfs/core": {
-      "version": "0.19.1",
-      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
-      "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==",
+      "version": "0.19.2",
+      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz",
+      "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==",
       "license": "Apache-2.0",
       "peer": true,
+      "dependencies": {
+        "@humanfs/types": "^0.15.0"
+      },
       "engines": {
         "node": ">=18.18.0"
       }
     },
     "node_modules/@humanfs/node": {
-      "version": "0.16.7",
-      "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz",
-      "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==",
+      "version": "0.16.8",
+      "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz",
+      "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==",
       "license": "Apache-2.0",
       "peer": true,
       "dependencies": {
-        "@humanfs/core": "^0.19.1",
+        "@humanfs/core": "^0.19.2",
+        "@humanfs/types": "^0.15.0",
         "@humanwhocodes/retry": "^0.4.0"
       },
       "engines": {
         "node": ">=18.18.0"
       }
     },
+    "node_modules/@humanfs/types": {
+      "version": "0.15.0",
+      "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz",
+      "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==",
+      "license": "Apache-2.0",
+      "peer": true,
+      "engines": {
+        "node": ">=18.18.0"
+      }
+    },
     "node_modules/@humanwhocodes/module-importer": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
@@ -289,9 +303,9 @@
       }
     },
     "node_modules/brace-expansion": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz",
-      "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "license": "MIT",
       "peer": true,
       "dependencies": {
@@ -629,9 +643,9 @@
       }
     },
     "node_modules/flatted": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
-      "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==",
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
+      "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
       "license": "ISC",
       "peer": true
     },

crates/cli/npm/dsl.d.ts

@@ -3,18 +3,22 @@ type BlankRule = { type: 'BLANK' };
 type ChoiceRule = { type: 'CHOICE'; members: Rule[] };
 type FieldRule = { type: 'FIELD'; name: string; content: Rule };
 type ImmediateTokenRule = { type: 'IMMEDIATE_TOKEN'; content: Rule };
-type PatternRule = { type: 'PATTERN'; value: string };
+type PatternRule = { type: 'PATTERN'; value: string; flags?: string };
+type PrecedenceValue = string | number;
 type PrecDynamicRule = { type: 'PREC_DYNAMIC'; content: Rule; value: number };
-type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: number };
-type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number };
-type PrecRule = { type: 'PREC'; content: Rule; value: number };
+type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: PrecedenceValue };
+type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: PrecedenceValue };
+type PrecRule = { type: 'PREC'; content: Rule; value: PrecedenceValue };
 type Repeat1Rule = { type: 'REPEAT1'; content: Rule };
 type RepeatRule = { type: 'REPEAT'; content: Rule };
 type ReservedRule = { type: 'RESERVED'; content: Rule; context_name: string };
 type SeqRule = { type: 'SEQ'; members: Rule[] };
 type StringRule = { type: 'STRING'; value: string };
 type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
+type PrecedenceEntry = StringRule | SymbolRule<string>;
 type TokenRule = { type: 'TOKEN'; content: Rule };
+type EOFRule = { type: 'EOF' };
+
 
 type Rule =
   | AliasRule
@@ -33,7 +37,8 @@ type Rule =
   | SeqRule
   | StringRule
   | SymbolRule<string>
-  | TokenRule;
+  | TokenRule
+  | EOFRule;
 
 declare class RustRegex {
   value: string;
@@ -86,8 +91,8 @@ interface Grammar<
    */
   precedences?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[][],
-  ) => RuleOrLiteral[][],
+    previous: PrecedenceEntry[][],
+  ) => (string | PrecedenceEntry)[][],
 
   /**
    * An array of arrays of rule names. Each inner array represents a set of
@@ -101,8 +106,8 @@ interface Grammar<
    */
   conflicts?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[][],
-  ) => RuleOrLiteral[][];
+    previous: SymbolRule<string>[][],
+  ) => SymbolRule<string>[][];
 
   /**
    * An array of token names which can be returned by an _external scanner_.
@@ -127,9 +132,11 @@ interface Grammar<
    * specify extras: `$ => []` in your grammar.
    *
    *  @param $ grammar rules
+   *  @param previous array of extras from the base grammar
    */
   extras?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
+    previous: Rule[],
   ) => RuleOrLiteral[];
 
   /**
@@ -142,8 +149,8 @@ interface Grammar<
    */
   inline?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[],
-  ) => RuleOrLiteral[];
+    previous: SymbolRule<string>[],
+  ) => SymbolRule<string>[];
 
   /**
    * A list of hidden rule names that should be considered supertypes in the
@@ -155,8 +162,8 @@ interface Grammar<
    */
   supertypes?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[],
-  ) => RuleOrLiteral[];
+    previous: SymbolRule<string>[],
+  ) => SymbolRule<string>[];
 
   /**
    * The name of a token that will match keywords for the purpose of the
@@ -166,24 +173,47 @@ interface Grammar<
    *
    * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar#keyword-extraction
    */
-  word?: ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral;
+  word?: (
+    $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
+  ) => SymbolRule<string>;
 
 
   /**
    * Mapping of names to reserved word sets. The first reserved word set is the
    * global word set, meaning it applies to every rule in every parse state.
-   * The other word sets can be used with the `reserved` function.
+   * The other word sets can be used with the `reserved` function. Each callback
+   * receives the base grammar's reserved word set of the same name as its second
+   * argument, or `undefined` if no matching set exists.
    */
   reserved?: Record<
     string,
-    ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral[]
+    (
+      $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
+      previous: Rule[] | undefined,
+    ) => RuleOrLiteral[]
   >;
 }
 
+/**
+ * Return type of grammar(). The runtime evaluates and normalizes the grammar
+ * beneath a "grammar" key. Optional input fields become required output fields
+ * with default values when not provided.
+ */
 type GrammarSchema<RuleName extends string> = {
-  [K in keyof Grammar<RuleName>]: K extends 'rules'
-  ? Record<RuleName, Rule>
-  : Grammar<RuleName>[K];
+  grammar: {
+    name: string;
+    /** Base grammar name when extending; undefined for root grammars. */
+    inherits: string | undefined;
+    rules: Record<RuleName, Rule>;
+    precedences: PrecedenceEntry[][];
+    conflicts: string[][];
+    externals: Rule[];
+    extras: Rule[];
+    inline: string[];
+    supertypes: string[];
+    word: string | undefined;
+    reserved: Record<string, Rule[]>;
+  };
 };
 
 /**
@@ -311,7 +341,7 @@ declare const prec: {
    *
    * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
    */
-  dynamic(value: string | number, rule: RuleOrLiteral): PrecDynamicRule;
+  dynamic(value: number, rule: RuleOrLiteral): PrecDynamicRule;
 };
 
 /**
@@ -382,6 +412,19 @@ declare const token: {
   immediate(rule: RuleOrLiteral): ImmediateTokenRule;
 };
 
+/**
+ * Matches the end of input. May only appear as the final symbol of a
+ * (possibly nested) sequence; a production ending in `eof()` reduces only
+ * when the lookahead is end-of-input, rather than shifting a token.
+ *
+ * Choice branches that continue past `eof()` are dropped as unreachable,
+ * and `eof()` is not allowed inside `token()`.
+ *
+ * Useful when a rule should match either an explicit terminator (e.g. a
+ * newline) or the end of the file.
+ */
+declare function eof(): EOFRule;
+
 /**
  * Creates a new language grammar with the provided schema.
  *

crates/cli/npm/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "tree-sitter-cli",
-      "version": "0.27.0",
+      "version": "0.28.0",
       "hasInstallScript": true,
       "license": "MIT",
       "bin": {

crates/cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

crates/cli/package.nix

@@ -13,7 +13,7 @@
   installShellFiles,
 }:
 let
-  isCross = stdenv.targetPlatform == stdenv.buildPlatform;
+  canRunHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 in
 rustPlatform.buildRustPackage {
   pname = "tree-sitter-cli";
@@ -28,7 +28,7 @@ rustPlatform.buildRustPackage {
     pkg-config
     nodejs_22
   ]
-  ++ lib.optionals (!isCross) [ installShellFiles ];
+  ++ lib.optionals canRunHost [ installShellFiles ];
 
   cargoLock.lockFile = ../../Cargo.lock;
 
@@ -42,9 +42,9 @@ rustPlatform.buildRustPackage {
   '';
 
   preCheck = "export HOME=$TMPDIR";
-  doCheck = !isCross;
+  doCheck = canRunHost;
 
-  postInstall = lib.optionalString (!isCross) ''
+  postInstall = lib.optionalString canRunHost ''
     installShellCompletion --cmd tree-sitter \
       --bash <($out/bin/tree-sitter complete --shell bash) \
       --zsh  <($out/bin/tree-sitter complete --shell zsh) \

crates/cli/src/fuzz.rs

@@ -25,7 +25,7 @@ use crate::{
         random::Rand,
     },
     parse::perform_edit,
-    test::{DiffKey, TestDiff, TestEntry, parse_tests, strip_sexp_fields},
+    test::{DiffKey, TestDiff, TestEntry, TestExpectation, parse_tests, render_test_output},
 };
 
 pub static LOG_ENABLED: LazyLock<bool> = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok());
@@ -65,7 +65,7 @@ pub fn new_seed() -> usize {
     int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
         let mut rng = rand::rng();
         let seed = rng.random_range(0..=usize::MAX);
-        info!("Seed: {seed}");
+        eprintln!("fuzz seed: {seed}");
         seed
     })
 }
@@ -144,7 +144,7 @@ pub fn fuzz_language_corpus(
         .take()
         .unwrap_or_default()
         .into_iter()
-        .chain(tests.iter().filter(|x| x.skip).map(get_test_name))
+        .chain(tests.iter().filter(|t| t.skip()).map(get_test_name))
         .map(|x| (x, 0))
         .collect::<HashMap<String, usize>>();
 
@@ -169,31 +169,8 @@ pub fn fuzz_language_corpus(
         println!("  {test_index}. {test_name}");
 
         let passed = allocations::record_checked(|| {
-            let mut log_session = None;
-            let mut parser = get_parser(&mut log_session, "log.html");
-            parser.set_language(language).unwrap();
-            set_included_ranges(&mut parser, &test.input, test.template_delimiters);
-
-            let tree = parser.parse(&test.input, None).unwrap();
-
-            if test.error {
-                return true;
-            }
-
-            let mut actual_output = tree.root_node().to_sexp();
-            if !test.has_fields {
-                actual_output = strip_sexp_fields(&actual_output);
-            }
-
-            if actual_output != test.output {
-                println!("Incorrect initial parse for {test_name}");
-                DiffKey::print();
-                println!("{}", TestDiff::new(&actual_output, &test.output));
-                println!();
-                return false;
-            }
-
-            true
+            let check_output = !test.error();
+            test.check_initial_parse(language, &test_name, check_output)
         })
         .unwrap_or_else(|e| {
             error!("{e}");
@@ -257,7 +234,7 @@ pub fn fuzz_language_corpus(
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
+                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 
@@ -273,12 +250,9 @@ pub fn fuzz_language_corpus(
                 let tree3 = parser.parse(&input, Some(&tree2)).unwrap();
 
                 // Verify that the final tree matches the expectation from the corpus.
-                let mut actual_output = tree3.root_node().to_sexp();
-                if !test.has_fields {
-                    actual_output = strip_sexp_fields(&actual_output);
-                }
+                let actual_output = render_test_output(&input, &tree3, test.cst, test.has_fields).unwrap();
 
-                if actual_output != test.output && !test.error {
+                if actual_output != test.output && !test.error() {
                     println!("Incorrect parse for {test_name} - seed {seed}");
                     DiffKey::print();
                     println!("{}", TestDiff::new(&actual_output, &test.output));
@@ -326,12 +300,56 @@ pub struct FlattenedTest {
     pub input: Vec<u8>,
     pub output: String,
     pub languages: Vec<Box<str>>,
-    pub error: bool,
-    pub skip: bool,
+    pub expectation: TestExpectation,
     pub has_fields: bool,
+    pub cst: bool,
     pub template_delimiters: Option<(&'static str, &'static str)>,
 }
 
+impl FlattenedTest {
+    #[must_use]
+    fn skip(&self) -> bool {
+        self.expectation == TestExpectation::Skip
+    }
+
+    #[must_use]
+    fn error(&self) -> bool {
+        self.expectation == TestExpectation::Error
+    }
+
+    #[must_use]
+    pub(crate) fn check_initial_parse(
+        &self,
+        language: &Language,
+        display_name: &str,
+        check_output: bool,
+    ) -> bool {
+        let mut log_session = None;
+        let mut parser = get_parser(&mut log_session, "log.html");
+        parser.set_language(language).unwrap();
+        set_included_ranges(&mut parser, &self.input, self.template_delimiters);
+
+        let tree = parser.parse(&self.input, None).unwrap();
+
+        if !check_output {
+            return true;
+        }
+
+        let actual_output =
+            render_test_output(&self.input, &tree, self.cst, self.has_fields).unwrap();
+
+        if actual_output == self.output {
+            true
+        } else {
+            println!("Incorrect initial parse for {display_name}");
+            DiffKey::print();
+            println!("{}", TestDiff::new(&actual_output, &self.output));
+            println!();
+            false
+        }
+    }
+}
+
 #[must_use]
 pub fn flatten_tests(
     test: TestEntry,
@@ -374,10 +392,10 @@ pub fn flatten_tests(
                     name,
                     input,
                     output,
-                    has_fields,
                     languages: attributes.languages,
-                    error: attributes.error,
-                    skip: attributes.skip,
+                    expectation: attributes.expectation,
+                    has_fields,
+                    cst: attributes.cst,
                     template_delimiters: None,
                 });
             }

crates/cli/src/fuzz/allocations.rs

@@ -10,12 +10,12 @@ use std::{
 #[ctor::ctor]
 unsafe fn initialize_allocation_recording() {
     unsafe {
-        tree_sitter::set_allocator(
-            Some(ts_record_malloc),
-            Some(ts_record_calloc),
-            Some(ts_record_realloc),
-            Some(ts_record_free),
-        );
+        tree_sitter::set_allocator(Some(tree_sitter::Allocator {
+            malloc: ts_record_malloc,
+            calloc: ts_record_calloc,
+            realloc: ts_record_realloc,
+            free: ts_record_free,
+        }));
     }
 }
 

crates/cli/src/fuzz/corpus_test.rs

@@ -3,8 +3,20 @@ use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
 use super::{LOG_ENABLED, LOG_GRAPH_ENABLED, scope_sequence::ScopeSequence};
 use crate::util;
 
-pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
-    fn check(node: Node, line_offsets: &[usize]) {
+struct SizeCheckFrame<'a> {
+    node: Node<'a>,
+    end_byte: usize,
+    end_point: Point,
+    child_count: u32,
+    child_index: u32,
+    last_child_end_byte: usize,
+    last_child_end_point: Point,
+    some_child_has_changes: bool,
+    actual_named_child_count: usize,
+}
+
+impl SizeCheckFrame<'_> {
+    fn new<'a>(node: Node<'a>, line_offsets: &[usize]) -> SizeCheckFrame<'a> {
         let start_byte = node.start_byte();
         let end_byte = node.end_byte();
         let start_point = node.start_position();
@@ -18,37 +30,21 @@ pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
         );
         assert_eq!(end_byte, line_offsets[end_point.row] + end_point.column);
 
-        let mut last_child_end_byte = start_byte;
-        let mut last_child_end_point = start_point;
-        let mut some_child_has_changes = false;
-        let mut actual_named_child_count = 0;
-        for i in 0..node.child_count() {
-            let child = node.child(i).unwrap();
-            assert!(child.start_byte() >= last_child_end_byte);
-            assert!(child.start_position() >= last_child_end_point);
-            check(child, line_offsets);
-            if child.has_changes() {
-                some_child_has_changes = true;
-            }
-            if child.is_named() {
-                actual_named_child_count += 1;
-            }
-            last_child_end_byte = child.end_byte();
-            last_child_end_point = child.end_position();
-        }
-
-        assert_eq!(actual_named_child_count, node.named_child_count());
-
-        if node.child_count() > 0 {
-            assert!(end_byte >= last_child_end_byte);
-            assert!(end_point >= last_child_end_point);
-        }
-
-        if some_child_has_changes {
-            assert!(node.has_changes());
+        SizeCheckFrame {
+            node,
+            end_byte,
+            end_point,
+            child_count: node.child_count(),
+            child_index: 0,
+            last_child_end_byte: start_byte,
+            last_child_end_point: start_point,
+            some_child_has_changes: false,
+            actual_named_child_count: 0,
         }
     }
+}
 
+pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
     let mut line_offsets = vec![0];
     for (i, c) in input.iter().enumerate() {
         if *c == b'\n' {
@@ -56,7 +52,41 @@ pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
         }
     }
 
-    check(tree.root_node(), &line_offsets);
+    let mut stack: Vec<SizeCheckFrame> = vec![SizeCheckFrame::new(tree.root_node(), &line_offsets)];
+    while let Some(top) = stack.last_mut() {
+        if top.child_index < top.child_count {
+            let i = top.child_index;
+            let child = top.node.child(i).unwrap();
+
+            assert!(child.start_byte() >= top.last_child_end_byte);
+            assert!(child.start_position() >= top.last_child_end_point);
+            if child.has_changes() {
+                top.some_child_has_changes = true;
+            }
+            if child.is_named() {
+                top.actual_named_child_count += 1;
+            }
+            top.last_child_end_byte = child.end_byte();
+            top.last_child_end_point = child.end_position();
+            top.child_index += 1;
+
+            stack.push(SizeCheckFrame::new(child, &line_offsets));
+            continue;
+        }
+
+        let frame = stack.pop().unwrap();
+        assert_eq!(
+            frame.actual_named_child_count,
+            frame.node.named_child_count()
+        );
+        if frame.child_count > 0 {
+            assert!(frame.end_byte >= frame.last_child_end_byte);
+            assert!(frame.end_point >= frame.last_child_end_point);
+        }
+        if frame.some_child_has_changes {
+            assert!(frame.node.has_changes());
+        }
+    }
 }
 
 pub fn check_changed_ranges(old_tree: &Tree, new_tree: &Tree, input: &[u8]) -> Result<(), String> {

crates/cli/src/fuzz/scope_sequence.rs

@@ -1,13 +1,13 @@
 use tree_sitter::{Point, Range, Tree};
 
 #[derive(Debug)]
-pub struct ScopeSequence(Vec<ScopeStack>);
+pub struct ScopeSequence<'a>(Vec<ScopeStack<'a>>);
 
-type ScopeStack = Vec<&'static str>;
+type ScopeStack<'a> = Vec<&'a str>;
 
-impl ScopeSequence {
+impl<'a> ScopeSequence<'a> {
     #[must_use]
-    pub fn new(tree: &Tree) -> Self {
+    pub fn new(tree: &'a Tree) -> Self {
         let mut result = Self(Vec::new());
         let mut scope_stack = Vec::new();
 
@@ -49,7 +49,7 @@ impl ScopeSequence {
         for i in 0..(self.0.len().max(other.0.len())) {
             let stack = &self.0.get(i);
             let other_stack = &other.0.get(i);
-            if *stack != *other_stack && ![b'\r', b'\n'].contains(&text[i]) {
+            if *stack != *other_stack && !b"\r\n".contains(&text[i]) {
                 let containing_range = known_changed_ranges
                     .iter()
                     .find(|range| range.start_point <= position && position < range.end_point);

crates/cli/src/highlight.rs

@@ -4,7 +4,6 @@ use std::{
     fs,
     io::{self, Write as _},
     path::{self, Path, PathBuf},
-    str,
     sync::{Arc, atomic::AtomicUsize},
     time::Instant,
 };
@@ -12,6 +11,7 @@ use std::{
 use ansi_colours::{ansi256_from_rgb, rgb_from_ansi256};
 use anstyle::{Ansi256Color, AnsiColor, Color, Effects, RgbColor};
 use anyhow::Result;
+use clap::ValueEnum;
 use log::{info, warn};
 use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeMap};
 use serde_json::{Value, json};
@@ -26,8 +26,9 @@ pub const HTML_HEAD_HEADER: &str = "
   <style>
     body {
       font-family: monospace
-    }
-    .line-number {
+    }";
+
+pub const HTML_LINE_NUMBER_STYLE: &str = "    .line-number {
       user-select: none;
       text-align: right;
       color: rgba(27,31,35,.3);
@@ -35,8 +36,7 @@ pub const HTML_HEAD_HEADER: &str = "
     }
     .line {
       white-space: pre;
-    }
-  </style>";
+    }";
 
 pub const HTML_BODY_HEADER: &str = "
 </head>
@@ -191,20 +191,14 @@ fn parse_style(style: &mut Style, json: Value) {
     if let Value::Object(entries) = json {
         for (property_name, value) in entries {
             match property_name.as_str() {
-                "bold" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.bold();
-                    }
+                "bold" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.bold();
                 }
-                "italic" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.italic();
-                    }
+                "italic" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.italic();
                 }
-                "underline" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.underline();
-                    }
+                "underline" if value == Value::Bool(true) => {
+                    style.ansi = style.ansi.underline();
                 }
                 "color" => {
                     if let Some(color) = parse_color(value) {
@@ -314,12 +308,36 @@ fn terminal_supports_truecolor() -> bool {
         .is_ok_and(|truecolor| truecolor == "truecolor" || truecolor == "24bit")
 }
 
+/// The kind of HTML emitted when highlighting to HTML.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+pub enum HtmlOutput {
+    /// A complete, self-contained document wrapping a plain
+    /// `<div class="highlight"><pre><code>` block.
+    Document,
+    /// A complete document with a line-number column (a `<table>` layout).
+    #[value(name = "line-numbers")]
+    NumberedDocument,
+    /// Only the code markup, without the surrounding document.
+    Fragment,
+}
+
+/// How token colors are applied in HTML output.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+pub enum HtmlStyling {
+    /// `class="..."` spans plus a generated `<style>` carrying the theme's colors.
+    Classes,
+    /// `style="..."` spans with the colors inlined.
+    Inline,
+    /// `class="..."` spans with no colors emitted (supply your own stylesheet).
+    Minimal,
+}
+
 pub struct HighlightOptions {
     pub theme: Theme,
     pub check: bool,
     pub captures_path: Option<PathBuf>,
-    pub inline_styles: bool,
-    pub html: bool,
+    /// `None` for regular output, `Some((layout, style))` when emitting HTML.
+    pub html: Option<(HtmlOutput, HtmlStyling)>,
     pub quiet: bool,
     pub print_time: bool,
     pub cancellation_flag: Arc<AtomicUsize>,
@@ -401,19 +419,25 @@ pub fn highlight(
     )?;
     let theme = &opts.theme;
 
-    if !opts.quiet && print_name {
+    // A fragment is pure code markup, so it must not be prefixed with the filename.
+    let html_fragment = opts
+        .html
+        .is_some_and(|(layout, _)| layout == HtmlOutput::Fragment);
+    if !opts.quiet && print_name && !html_fragment {
         writeln!(&mut stdout, "{name}")?;
     }
 
-    if opts.html {
-        if !opts.quiet {
+    if let Some((layout, style)) = opts.html {
+        if !opts.quiet && layout != HtmlOutput::Fragment {
             writeln!(&mut stdout, "{HTML_HEAD_HEADER}")?;
-            writeln!(&mut stdout, "  <style>")?;
-            let names = theme.highlight_names.iter();
-            let styles = theme.styles.iter();
-            for (name, style) in names.zip(styles) {
-                if let Some(css) = &style.css {
-                    writeln!(&mut stdout, "    .{name} {{ {css}; }}")?;
+            if layout == HtmlOutput::NumberedDocument {
+                writeln!(&mut stdout, "{HTML_LINE_NUMBER_STYLE}")?;
+            }
+            if style == HtmlStyling::Classes {
+                for (name, style) in theme.highlight_names.iter().zip(&theme.styles) {
+                    if let Some(css) = &style.css {
+                        writeln!(&mut stdout, "    .{name} {{ {css}; }}")?;
+                    }
                 }
             }
             writeln!(&mut stdout, "  </style>")?;
@@ -422,7 +446,7 @@ pub fn highlight(
 
         let mut renderer = HtmlRenderer::new();
         renderer.render(events, &source, &move |highlight, output| {
-            if opts.inline_styles {
+            if style == HtmlStyling::Inline {
                 output.extend(b"style='");
                 output.extend(
                     theme.styles[highlight.0]
@@ -444,16 +468,29 @@ pub fn highlight(
         })?;
 
         if !opts.quiet {
-            writeln!(&mut stdout, "<table>")?;
-            for (i, line) in renderer.lines().enumerate() {
+            if layout == HtmlOutput::NumberedDocument {
+                writeln!(&mut stdout, "<table>")?;
+                for (i, line) in renderer.lines().enumerate() {
+                    writeln!(
+                        &mut stdout,
+                        "<tr><td class=line-number>{}</td><td class=line>{line}</td></tr>",
+                        i + 1,
+                    )?;
+                }
+                writeln!(&mut stdout, "</table>")?;
+            } else {
+                let mut body = renderer.lines().collect::<String>();
+                if body.ends_with('\n') {
+                    body.pop();
+                }
                 writeln!(
                     &mut stdout,
-                    "<tr><td class=line-number>{}</td><td class=line>{line}</td></tr>",
-                    i + 1,
+                    "<div class=\"highlight\">\n<pre><code>{body}</code></pre>\n</div>",
                 )?;
             }
-            writeln!(&mut stdout, "</table>")?;
-            writeln!(&mut stdout, "{HTML_FOOTER}")?;
+            if layout != HtmlOutput::Fragment {
+                writeln!(&mut stdout, "{HTML_FOOTER}")?;
+            }
         }
     } else {
         let mut style_stack = vec![theme.default_style().ansi];

crates/cli/src/init.rs

@@ -1,7 +1,7 @@
 use std::{
     fs,
     path::{Path, PathBuf},
-    str::{self, FromStr},
+    str::FromStr as _,
 };
 
 use anyhow::{Context, Result, anyhow};
@@ -836,6 +836,8 @@ fn generate_java_bindings(
     Ok(())
 }
 
+// TODO: remove old migrations
+
 fn update_package_json(path: &Path) -> Result<()> {
     let mut contents = fs::read_to_string(path)?
         .replace(
@@ -867,7 +869,7 @@ fn update_package_json(path: &Path) -> Result<()> {
 fn update_grammar_js(path: &Path) -> Result<()> {
     let mut contents = fs::read_to_string(path)?;
     if contents.contains("module.exports") {
-        info!("Migrating grammars.js to ESM");
+        info!("Migrating grammar.js to ESM");
         contents = contents.replace("module.exports =", "export default");
         write_file(path, contents)?;
     }
@@ -974,18 +976,8 @@ fn update_rust_build_rs(path: &Path, language_name: &str, opts: &GenerateOpts) -
                 let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
                     panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
                 };
-                let Ok(wasm_src) =
-                    std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
-                else {
-                    panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
-                };
 
                 c_config.include(&wasm_headers);
-                c_config.files([
-                    wasm_src.join("stdio.c"),
-                    wasm_src.join("stdlib.c"),
-                    wasm_src.join("string.c"),
-                ]);
             }
         "#}
             .lines()
@@ -1060,6 +1052,19 @@ fn update_c_makefile(path: &Path, language_name: &str, opts: &GenerateOpts) -> R
                     "},
             );
         }
+        if !contents.contains("\nDESCRIPTION :=")
+            && let Some(version_line) = contents.lines().find(|l| l.starts_with("VERSION := "))
+        {
+            info!("Adding DESCRIPTION to Makefile");
+            let description = opts.description.map_or_else(
+                || format!("{} grammar for tree-sitter", opts.camel_parser_name),
+                str::to_string,
+            );
+            contents = contents.replace(
+                version_line,
+                &format!("{version_line}\nDESCRIPTION := {description}"),
+            );
+        }
         write_file(path, contents)?;
     }
     Ok(())
@@ -1200,14 +1205,23 @@ fn update_python_setup_py(path: &Path, language_name: &str, opts: &GenerateOpts)
     if !contents.contains("build_ext") {
         info!("Replacing setup.py");
         generate_file(path, SETUP_PY_TEMPLATE, language_name, opts)?;
-    }
-    if !contents.contains(" and not get_config_var") {
-        info!("Updating Python free-threading support in setup.py");
-        contents = contents.replace(
-            r#"startswith("cp"):"#,
-            r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#,
-        );
-        write_file(path, contents)?;
+    } else {
+        if !contents.contains(" and not get_config_var") {
+            info!("Updating Python free-threading support in setup.py");
+            contents = contents.replace(
+                r#"startswith("cp"):"#,
+                r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#,
+            );
+            write_file(path, &contents)?;
+        }
+        if !contents.contains("include(\"src/*.c\")") {
+            info!("Updating sdist file list in setup.py");
+            let contents = contents.replace(
+                "include(\"src/tree_sitter/*.h\")",
+                "include(\"src/tree_sitter/*.h\")\n        self.filelist.include(\"src/*.c\")",
+            );
+            write_file(path, &contents)?;
+        }
     }
     Ok(())
 }
@@ -1228,8 +1242,13 @@ fn update_swift_package(path: &Path) -> Result<()> {
             "atPath: \"src/scanner.c\"",
             "atPath: \"\\(dir)/src/scanner.c\"",
         )
-        .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
-        .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
+        .replace("version: \"0.8.0\")", "version: \"0.10.0\")")
+        .replace("version: \"0.9.0\")", "version: \"0.10.0\")")
+        .replace("(name: \"SwiftTreeSitter\", url:", "(url:")
+        .replace(
+            "    \"SwiftTreeSitter\"",
+            "    .product(name: \"SwiftTreeSitter\", package: \"swift-tree-sitter\")",
+        );
     if !replaced_contents.eq(&contents) {
         info!("Updating Package.swift");
         write_file(path, replaced_contents)?;
@@ -1496,24 +1515,18 @@ fn generate_file(
         replacement = replacement
             .replace(
                 PARSER_URL_STRIPPED_PLACEHOLDER,
-                &repository.replace("https://", "").to_lowercase(),
+                &repository.replace("https://", ""),
             )
-            .replace(PARSER_URL_PLACEHOLDER, &repository.to_lowercase());
+            .replace(PARSER_URL_PLACEHOLDER, repository);
     } else {
         replacement = replacement
             .replace(
                 PARSER_URL_STRIPPED_PLACEHOLDER,
-                &format!(
-                    "github.com/tree-sitter/tree-sitter-{}",
-                    language_name.to_lowercase()
-                ),
+                &format!("github.com/tree-sitter/tree-sitter-{language_name}"),
             )
             .replace(
                 PARSER_URL_PLACEHOLDER,
-                &format!(
-                    "https://github.com/tree-sitter/tree-sitter-{}",
-                    language_name.to_lowercase()
-                ),
+                &format!("https://github.com/tree-sitter/tree-sitter-{language_name}"),
             );
     }
 

crates/cli/src/logger.rs

@@ -1,12 +1,8 @@
 use std::io::Write;
 
-use anstyle::{AnsiColor, Color, Style};
 use log::{Level, LevelFilter, Log, Metadata, Record};
 
-pub fn paint(color: Option<impl Into<Color>>, text: &str) -> String {
-    let style = Style::new().fg_color(color.map(Into::into));
-    format!("{style}{text}{style:#}")
-}
+use crate::paint::{Paint, RED, YELLOW};
 
 struct Logger;
 
@@ -17,16 +13,8 @@ impl Log for Logger {
 
     fn log(&self, record: &Record) {
         match record.level() {
-            Level::Error => eprintln!(
-                "{} {}",
-                paint(Some(AnsiColor::Red), "Error:"),
-                record.args()
-            ),
-            Level::Warn => eprintln!(
-                "{} {}",
-                paint(Some(AnsiColor::Yellow), "Warning:"),
-                record.args()
-            ),
+            Level::Error => eprintln!("{} {}", Paint(RED, "Error:"), record.args()),
+            Level::Warn => eprintln!("{} {}", Paint(YELLOW, "Warning:"), record.args()),
             Level::Info | Level::Debug => eprintln!("{}", record.args()),
             Level::Trace => eprintln!(
                 "[{}] {}",

crates/cli/src/main.rs

@@ -6,7 +6,7 @@ use std::{
 
 use anstyle::{AnsiColor, Color, Style};
 use anyhow::{Context, Result, anyhow};
-use clap::{Args, Command, FromArgMatches as _, Subcommand, ValueEnum, crate_authors};
+use clap::{ArgGroup, Args, Command, FromArgMatches as _, Subcommand, ValueEnum, crate_authors};
 use clap_complete::generate;
 use dialoguer::{Confirm, FuzzySelect, Input, MultiSelect, theme::ColorfulTheme};
 use heck::ToUpperCamelCase;
@@ -19,10 +19,10 @@ use tree_sitter_cli::{
         DEFAULT_EDIT_COUNT, DEFAULT_ITERATION_COUNT, EDIT_COUNT, FuzzOptions, ITERATION_COUNT,
         LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED, fuzz_language_corpus,
     },
-    highlight::{self, HighlightOptions},
+    highlight::{self, HighlightOptions, HtmlOutput, HtmlStyling},
     init::{JsonConfigOpts, TREE_SITTER_JSON_SCHEMA, generate_grammar_files},
     input::{CliInput, get_input, get_tmp_source_file},
-    logger,
+    logger, paint,
     parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
     playground,
     query::{self, QueryFileOptions},
@@ -33,7 +33,7 @@ use tree_sitter_cli::{
     wasm,
 };
 use tree_sitter_config::Config;
-use tree_sitter_generate::OptLevel;
+use tree_sitter_generate::{Diagnostic, GenerateError, OptLevel};
 use tree_sitter_highlight::Highlighter;
 use tree_sitter_loader::{self as loader, Bindings, TreeSitterJSON};
 use tree_sitter_tags::TagsContext;
@@ -195,6 +195,7 @@ struct Build {
 
 #[derive(Args)]
 #[command(alias = "p")]
+#[command(group(ArgGroup::new("graph_output").multiple(true)))]
 struct Parse {
     /// The path to a file with paths to source file(s)
     #[arg(long = "paths")]
@@ -210,7 +211,7 @@ struct Parse {
     pub lib_path: Option<PathBuf>,
     /// If `--lib-path` is used, the name of the language used to extract the
     /// library's language function
-    #[arg(long)]
+    #[arg(long, requires = "lib_path")]
     pub lang_name: Option<String>,
     /// Select a language by the scope instead of a file extension
     #[arg(long)]
@@ -226,13 +227,13 @@ struct Parse {
     #[arg(long, short = '0')]
     pub debug_build: bool,
     /// Produce the log.html file with debug graphs
-    #[arg(long, short = 'D')]
+    #[arg(long, short = 'D', group = "graph_output")]
     pub debug_graph: bool,
     /// Compile parsers to Wasm instead of native dynamic libraries
     #[arg(long, hide = cfg!(not(feature = "wasm")))]
     pub wasm: bool,
     /// Output the parse data with graphviz dot
-    #[arg(long = "dot")]
+    #[arg(long = "dot", group = "graph_output")]
     pub output_dot: bool,
     /// Output the parse data in XML format
     #[arg(long = "xml", short = 'x')]
@@ -240,7 +241,7 @@ struct Parse {
     /// Output the parse data in a pretty-printed CST format
     #[arg(long = "cst", short = 'c')]
     pub output_cst: bool,
-    /// Show parsing statistic
+    /// Show parsing statistics
     #[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")]
     pub stat: bool,
     /// Interrupt the parsing process by timeout (µs)
@@ -266,8 +267,8 @@ struct Parse {
     /// The encoding of the input files
     #[arg(long)]
     pub encoding: Option<Encoding>,
-    /// Open `log.html` in the default browser, if `--debug-graph` is supplied
-    #[arg(long)]
+    /// Open `log.html` in the default browser, if `--debug-graph` or `--dot` is supplied
+    #[arg(long, requires = "graph_output")]
     pub open_log: bool,
     /// Deprecated: use --json-summary
     #[arg(long, conflicts_with = "json_summary", conflicts_with = "stat")]
@@ -317,7 +318,7 @@ struct Test {
     pub lib_path: Option<PathBuf>,
     /// If `--lib-path` is used, the name of the language used to extract the
     /// library's language function
-    #[arg(long)]
+    #[arg(long, requires = "lib_path")]
     pub lang_name: Option<String>,
     /// Update all syntax trees in corpus files with current parser output
     #[arg(long, short)]
@@ -335,7 +336,7 @@ struct Test {
     #[arg(long, hide = cfg!(not(feature = "wasm")))]
     pub wasm: bool,
     /// Open `log.html` in the default browser, if `--debug-graph` is supplied
-    #[arg(long)]
+    #[arg(long, requires = "debug_graph")]
     pub open_log: bool,
     /// The path to an alternative config.json file
     #[arg(long)]
@@ -343,6 +344,9 @@ struct Test {
     /// Force showing fields in test diffs
     #[arg(long)]
     pub show_fields: bool,
+    /// Force showing '+' and '-' in test diffs
+    #[arg(long)]
+    pub show_diff_markers: bool,
     /// Show parsing statistics
     #[arg(long)]
     pub stat: Option<TestStats>,
@@ -402,7 +406,7 @@ struct Fuzz {
     pub lib_path: Option<PathBuf>,
     /// If `--lib-path` is used, the name of the language used to extract the
     /// library's language function
-    #[arg(long)]
+    #[arg(long, requires = "lib_path")]
     pub lang_name: Option<String>,
     #[arg(
         long,
@@ -449,7 +453,7 @@ struct Query {
     pub lib_path: Option<PathBuf>,
     /// If `--lib-path` is used, the name of the language used to extract the
     /// library's language function
-    #[arg(long)]
+    #[arg(long, requires = "lib_path")]
     pub lang_name: Option<String>,
     /// Measure execution time
     #[arg(long, short)]
@@ -504,14 +508,20 @@ struct Highlight {
     /// Generate highlighting as an HTML document
     #[arg(long, short = 'H')]
     pub html: bool,
-    /// When generating HTML, use css classes rather than inline styles
-    #[arg(long)]
+    /// Deprecated: use `--style classes`
+    #[arg(long, requires = "html", conflicts_with = "style")]
     pub css_classes: bool,
+    /// When generating HTML, the document structure to emit
+    #[arg(long, requires = "html", value_enum, default_value = "document")]
+    pub layout: HtmlOutput,
+    /// When generating HTML, how token colors are applied
+    #[arg(long, requires = "html", value_enum, default_value = "classes")]
+    pub style: HtmlStyling,
     /// Check that highlighting captures conform strictly to standards
     #[arg(long)]
     pub check: bool,
     /// The path to a file with captures
-    #[arg(long)]
+    #[arg(long, requires = "check")]
     pub captures_path: Option<PathBuf>,
     /// The paths to files with queries
     #[arg(long, num_args = 1..)]
@@ -901,7 +911,7 @@ impl Init {
 
             let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
             // Write the re-serialized config back, as newly added optional boolean fields
-            // will be included with explicit `false`s rather than implict `null`s
+            // will be included with explicit `false`s rather than implicit `null`s
             if self.update && !old_config.trim().eq(new_config.trim()) {
                 info!("Updating tree-sitter.json");
                 fs::write(
@@ -948,7 +958,8 @@ impl Generate {
             self.json_summary
         };
 
-        if let Err(err) = tree_sitter_generate::generate_parser_in_directory(
+        let mut diagnostics = Vec::new();
+        let result = tree_sitter_generate::generate_parser_in_directory(
             current_dir,
             self.output.as_deref(),
             self.grammar_path.as_deref(),
@@ -961,16 +972,33 @@ impl Generate {
             } else {
                 OptLevel::default()
             },
-        ) {
-            if json_summary {
-                eprintln!("{}", serde_json::to_string_pretty(&err)?);
+            &mut diagnostics,
+        );
+        if json_summary {
+            #[derive(serde::Serialize)]
+            struct Envelope<'a> {
+                diagnostics: &'a [Diagnostic],
+                error: Option<&'a GenerateError>,
+            }
+            let envelope = Envelope {
+                diagnostics: &diagnostics,
+                error: result.as_ref().err(),
+            };
+            eprintln!("{}", serde_json::to_string_pretty(&envelope)?);
+            if result.is_err() {
                 // Exit early to prevent errors from being printed a second time in the caller
                 std::process::exit(1);
-            } else {
+            }
+        } else {
+            for d in &diagnostics {
+                warn!("{d}");
+            }
+            if let Err(err) = result {
                 // Removes extra context associated with the error
                 Err(anyhow!(err.to_string())).with_context(|| "Error when generating parser")?;
             }
         }
+
         if self.build {
             warn!("--build is deprecated, use the `build` command");
             if let Some(path) = self.libdir {
@@ -1006,7 +1034,7 @@ impl Build {
                     .context("Output path must have a parent")?;
                 let name = full_path
                     .file_name()
-                    .context("Ouput path must have a filename")?;
+                    .context("Output path must have a filename")?;
                 fs::create_dir_all(parent_path).context("Failed to create output path")?;
                 let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
                 canon_path.push(name);
@@ -1044,7 +1072,6 @@ impl Build {
 impl Parse {
     fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> {
         let config = Config::load(self.config_path)?;
-        let color = env::var("NO_COLOR").map_or(true, |v| v != "1");
         let json_summary = if self.json {
             warn!("--json is deprecated, use --json-summary instead");
             true
@@ -1063,7 +1090,7 @@ impl Parse {
             ParseOutput::Normal
         };
 
-        let parse_theme = if color {
+        let parse_theme = if paint::color_enabled() {
             config
                 .get::<parse::Config>()
                 .with_context(|| "Failed to parse CST theme")?
@@ -1101,9 +1128,6 @@ impl Parse {
         let timeout = self.timeout.unwrap_or_default();
 
         let mut has_error = false;
-        let loader_config = config.get()?;
-        loader.find_all_languages(&loader_config)?;
-
         let should_track_stats = self.stat;
         let mut stats = parse::ParseStats::default();
         let debug: ParseDebugType = match self.debug {
@@ -1146,10 +1170,11 @@ impl Parse {
             has_error |= !parse_result.successful;
         };
 
-        if self.lib_path.is_none() && self.lang_name.is_some() {
-            warn!("--lang-name` specified without --lib-path. This argument will be ignored.");
-        }
         let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir);
+        if lib_info.is_none() {
+            let loader_config = config.get()?;
+            loader.find_all_languages(&loader_config)?;
+        }
 
         let input = get_input(
             self.paths_file.as_deref(),
@@ -1300,7 +1325,6 @@ fn check_test(
 impl Test {
     fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> {
         let config = Config::load(self.config_path)?;
-        let color = env::var("NO_COLOR").map_or(true, |v| v != "1");
         let stat = self.stat.unwrap_or_default();
 
         loader.debug_build(self.debug_build);
@@ -1318,9 +1342,6 @@ impl Test {
             });
         }
 
-        if self.lib_path.is_none() && self.lang_name.is_some() {
-            warn!("--lang-name` specified without --lib-path. This argument will be ignored.");
-        }
         let languages = loader.languages_at_path(current_dir)?;
         let language = if let Some(ref lib_path) = self.lib_path {
             let lib_info =
@@ -1342,13 +1363,9 @@ impl Test {
         parser.set_language(language)?;
 
         let test_dir = current_dir.join("test");
-        let mut test_summary = TestSummary::new(
-            color,
-            stat,
-            self.update,
-            self.overview_only,
-            self.json_summary,
-        );
+        let mut test_summary =
+            TestSummary::new(stat, self.update, self.overview_only, self.json_summary);
+        test_summary.use_markers = self.show_diff_markers;
 
         // Run the corpus tests. Look for them in `test/corpus`.
         let test_corpus_dir = test_dir.join("corpus");
@@ -1363,7 +1380,6 @@ impl Test {
                 update: self.update,
                 open_log: self.open_log,
                 languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(),
-                color,
                 show_fields: self.show_fields,
                 overview_only: self.overview_only,
             };
@@ -1374,16 +1390,20 @@ impl Test {
                 self.json_summary,
             )?;
             test_summary.test_num = 1;
+        } else {
+            warn!("Test corpus not found at {}", test_corpus_dir.display());
         }
 
         // Check that all of the queries are valid.
         let query_dir = current_dir.join("queries");
-        check_test(
-            test::check_queries_at_path(language, &query_dir),
-            &test_summary,
-            self.json_summary,
-        )?;
-        test_summary.test_num = 1;
+        if query_dir.is_dir() {
+            check_test(
+                test::check_queries_at_path(language, &query_dir),
+                &test_summary,
+                self.json_summary,
+            )?;
+            test_summary.test_num = 1;
+        }
 
         // Run the syntax highlighting tests.
         let test_highlight_dir = test_dir.join("highlight");
@@ -1497,9 +1517,6 @@ impl Fuzz {
         loader.sanitize_build(true);
         loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
 
-        if self.lib_path.is_none() && self.lang_name.is_some() {
-            warn!("--lang-name` specified without --lib-path. This argument will be ignored.");
-        }
         let languages = loader.languages_at_path(current_dir)?;
         let (language, language_name) = if let Some(ref lib_path) = self.lib_path {
             let lib_info = get_lib_info(Some(lib_path), self.lang_name.as_ref(), current_dir)
@@ -1547,9 +1564,12 @@ impl Fuzz {
 impl Query {
     fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> {
         let config = Config::load(self.config_path)?;
-        let loader_config = config.get()?;
+        let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir);
+        if lib_info.is_none() {
+            let loader_config = config.get()?;
+            loader.find_all_languages(&loader_config)?;
+        }
         loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
-        loader.find_all_languages(&loader_config)?;
         let query_path = Path::new(&self.query_path);
 
         let byte_range = parse_range(self.byte_range.as_deref(), |x| x)?;
@@ -1561,11 +1581,6 @@ impl Query {
 
         let cancellation_flag = util::cancel_on_signal();
 
-        if self.lib_path.is_none() && self.lang_name.is_some() {
-            warn!("--lang-name specified without --lib-path. This argument will be ignored.");
-        }
-        let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir);
-
         let input = get_input(
             self.paths_file.as_deref(),
             self.paths,
@@ -1695,12 +1710,19 @@ impl Highlight {
             Encoding::Utf16BE => ffi::TSInputEncodingUTF16BE,
         });
 
+        let style = if self.css_classes {
+            // TODO: Remove during the 0.28 release cycle
+            warn!("--css-classes is deprecated, use --style classes instead");
+            HtmlStyling::Classes
+        } else {
+            self.style
+        };
+
         let options = HighlightOptions {
             theme: theme_config.theme,
             check: self.check,
             captures_path: self.captures_path,
-            inline_styles: !self.css_classes,
-            html: self.html,
+            html: self.html.then_some((self.layout, style)),
             quiet: self.quiet,
             print_time: self.time,
             cancellation_flag: cancellation_flag.clone(),
@@ -1985,7 +2007,7 @@ impl DumpLanguages {
                 concat!(
                     "name: {}\n",
                     "scope: {}\n",
-                    "parser: {:?}\n",
+                    "parser: {}\n",
                     "highlights: {:?}\n",
                     "file_types: {:?}\n",
                     "content_regex: {:?}\n",
@@ -1993,7 +2015,7 @@ impl DumpLanguages {
                 ),
                 configuration.language_name,
                 configuration.scope.as_ref().unwrap_or(&String::new()),
-                language_path,
+                language_path.display(),
                 configuration.highlights_filenames,
                 configuration.file_types,
                 configuration.content_regex,

crates/cli/src/paint.rs

@@ -0,0 +1,27 @@
+use anstyle::{AnsiColor, Color, Style};
+
+pub const RED: Style = Style::new().fg_color(Some(Color::Ansi(AnsiColor::Red)));
+pub const YELLOW: Style = Style::new().fg_color(Some(Color::Ansi(AnsiColor::Yellow)));
+
+/// Wraps a `Display` value with a style; emits ANSI codes only when
+/// [`color_enabled`] is true.
+pub struct Paint<T>(pub Style, pub T);
+
+pub fn color_enabled() -> bool {
+    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ENABLED.get_or_init(|| std::env::var_os("NO_COLOR").is_none_or(|v| v.is_empty()))
+}
+
+pub fn paint<T>(color: Option<impl Into<Color>>, text: T) -> Paint<T> {
+    Paint(Style::new().fg_color(color.map(Into::into)), text)
+}
+
+impl<T: std::fmt::Display> std::fmt::Display for Paint<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if color_enabled() {
+            write!(f, "{}{}{:#}", self.0, self.1, self.0)
+        } else {
+            self.1.fmt(f)
+        }
+    }
+}

crates/cli/src/parse.rs

@@ -18,7 +18,7 @@ use tree_sitter::{
     ffi,
 };
 
-use crate::{fuzz::edits::Edit, logger::paint, util};
+use crate::{fuzz::edits::Edit, paint::paint, util};
 
 #[derive(Debug, Default, Serialize, JsonSchema)]
 pub struct Stats {
@@ -301,7 +301,6 @@ pub fn parse_file_at_path(
     // Log to stderr if `--debug` was passed
     else if opts.debug != ParseDebugType::Quiet {
         let mut curr_version: usize = 0;
-        let use_color = std::env::var("NO_COLOR").map_or(true, |v| v != "1");
         let debug = opts.debug;
         parser.set_logger(Some(Box::new(move |log_type, message| {
             if debug == ParseDebugType::Normal {
@@ -310,13 +309,13 @@ pub fn parse_file_at_path(
                 }
                 writeln!(&mut io::stderr(), "{message}").unwrap();
             } else {
+                #[rustfmt::skip]
                 let colors = &[
-                    AnsiColor::White,
-                    AnsiColor::Red,
-                    AnsiColor::Blue,
-                    AnsiColor::Green,
-                    AnsiColor::Cyan,
-                    AnsiColor::Yellow,
+                    AnsiColor::White, AnsiColor::Red, AnsiColor::Blue, AnsiColor::Green,
+                    AnsiColor::Cyan, AnsiColor::Yellow, AnsiColor::Magenta,
+                    AnsiColor::BrightWhite, AnsiColor::BrightRed, AnsiColor::BrightBlue,
+                    AnsiColor::BrightGreen, AnsiColor::BrightCyan, AnsiColor::BrightYellow,
+                    AnsiColor::BrightMagenta,
                 ];
                 if message.starts_with("process version:") {
                     let comma_idx = message.find(',').unwrap();
@@ -324,18 +323,9 @@ pub fn parse_file_at_path(
                         .parse()
                         .unwrap();
                 }
-                let color = if use_color {
-                    Some(colors[curr_version])
-                } else {
-                    None
-                };
-                let mut out = if log_type == LogType::Lex {
-                    "  ".to_string()
-                } else {
-                    String::new()
-                };
-                out += &paint(color, message);
-                writeln!(&mut io::stderr(), "{out}").unwrap();
+                let color = Some(colors[curr_version % colors.len()]);
+                let prefix = if log_type == LogType::Lex { "  " } else { "" };
+                writeln!(&mut io::stderr(), "{prefix}{}", paint(color, message)).unwrap();
             }
         })));
     }
@@ -390,8 +380,10 @@ pub fn parse_file_at_path(
     let tree = match encoding {
         Some(encoding) if encoding == ffi::TSInputEncodingUTF16LE => {
             let source_code_utf16 = source_code
-                .chunks_exact(2)
-                .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
+                .as_chunks::<2>()
+                .0
+                .iter()
+                .map(|&chunk| u16::from_le_bytes(chunk))
                 .collect::<Vec<_>>();
             parser.parse_utf16_le_with_options(
                 &mut |i, _| {
@@ -407,8 +399,10 @@ pub fn parse_file_at_path(
         }
         Some(encoding) if encoding == ffi::TSInputEncodingUTF16BE => {
             let source_code_utf16 = source_code
-                .chunks_exact(2)
-                .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
+                .as_chunks::<2>()
+                .0
+                .iter()
+                .map(|&chunk| u16::from_be_bytes(chunk))
                 .collect::<Vec<_>>();
             parser.parse_utf16_be_with_options(
                 &mut |i, _| {
@@ -437,7 +431,7 @@ pub fn parse_file_at_path(
     let parse_duration = parse_time.elapsed();
 
     let stdout = io::stdout();
-    let mut stdout = stdout.lock();
+    let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
 
     if let Some(mut tree) = tree {
         if opts.debug_graph && !opts.edits.is_empty() {
@@ -514,7 +508,7 @@ pub fn parse_file_at_path(
                 }
             }
             cursor.reset(tree.root_node());
-            println!();
+            writeln!(&mut stdout)?;
         }
 
         if opts.output == ParseOutput::Cst {
@@ -585,11 +579,11 @@ pub fn parse_file_at_path(
                         }
                         let start = node.start_position();
                         let end = node.end_position();
-                        write!(&mut stdout, " srow=\"{}\"", start.row)?;
-                        write!(&mut stdout, " scol=\"{}\"", start.column)?;
-                        write!(&mut stdout, " erow=\"{}\"", end.row)?;
-                        write!(&mut stdout, " ecol=\"{}\"", end.column)?;
-                        write!(&mut stdout, ">")?;
+                        write!(
+                            &mut stdout,
+                            " srow=\"{}\" scol=\"{}\" erow=\"{}\" ecol=\"{}\">",
+                            start.row, start.column, end.row, end.column
+                        )?;
                         tags.push(node.kind());
                         needs_newline = true;
                     }
@@ -780,12 +774,16 @@ pub fn render_cst<'a, 'b: 'a>(
     cursor: &mut TreeCursor<'a>,
     opts: &ParseFileOptions,
     out: &mut impl Write,
-) -> Result<()> {
+) -> io::Result<()> {
     let lossy_source_code = String::from_utf8_lossy(source_code);
     let total_width = lossy_source_code
         .lines()
         .enumerate()
-        .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
+        .map(|(row, col)| {
+            row.checked_ilog10().unwrap_or(0) as usize
+                + col.len().checked_ilog10().unwrap_or(0) as usize
+                + 1
+        })
         .max()
         .unwrap_or(1);
     let mut indent_level = usize::from(!opts.no_ranges);
@@ -829,19 +827,19 @@ pub fn render_cst<'a, 'b: 'a>(
     Ok(())
 }
 
-fn render_node_text(source: &str) -> String {
-    source
-        .chars()
-        .fold(String::with_capacity(source.len()), |mut acc, c| {
-            if let Some(esc) = escape_invisible(c) {
-                acc.push_str(esc);
-            } else if let Some(esc) = escape_delimiter(c) {
-                acc.push_str(esc);
-            } else {
-                acc.push(c);
+struct CstNodeText<'a>(&'a str);
+
+impl std::fmt::Display for CstNodeText<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        use std::fmt::Write as _;
+        for c in self.0.chars() {
+            match escape_invisible(c).or_else(|| escape_delimiter(c)) {
+                Some(esc) => f.write_str(esc)?,
+                None => f.write_char(c)?,
             }
-            acc
-        })
+        }
+        Ok(())
+    }
 }
 
 fn write_node_text(
@@ -852,7 +850,7 @@ fn write_node_text(
     source: &str,
     color: Option<impl Into<Color> + Copy>,
     text_info: (usize, usize),
-) -> Result<()> {
+) -> io::Result<()> {
     let (total_width, indent_level) = text_info;
     let (quote, quote_color) = if is_named {
         ('`', opts.parse_theme.backtick)
@@ -864,9 +862,9 @@ fn write_node_text(
         write!(
             out,
             "{}{}{}",
-            paint(quote_color, &String::from(quote)),
-            paint(color, &render_node_text(source)),
-            paint(quote_color, &String::from(quote)),
+            paint(quote_color, quote),
+            paint(color, CstNodeText(source)),
+            paint(quote_color, quote),
         )?;
     } else {
         let multiline = source.contains('\n');
@@ -885,24 +883,34 @@ fn write_node_text(
                 } else {
                     0
                 };
-            let formatted_line = render_line_feed(line, opts);
+            if multiline {
+                writeln!(out)?;
+                if !opts.no_ranges {
+                    write!(
+                        out,
+                        "{}",
+                        CstNodeRange {
+                            opts,
+                            has_field_name: cursor.field_name().is_some(),
+                            is_named,
+                            is_multiline: true,
+                            total_width,
+                            range: node_range,
+                        }
+                    )?;
+                }
+                for _ in 0..=indent_level {
+                    write!(out, "  ")?;
+                }
+            } else {
+                write!(out, " ")?;
+            }
             write!(
                 out,
-                "{}{}{}{}{}{}",
-                if multiline { "\n" } else { " " },
-                if multiline && !opts.no_ranges {
-                    render_node_range(opts, cursor, is_named, true, total_width, node_range)
-                } else {
-                    String::new()
-                },
-                if multiline {
-                    "  ".repeat(indent_level + 1)
-                } else {
-                    String::new()
-                },
-                paint(quote_color, &String::from(quote)),
-                paint(color, &render_node_text(&formatted_line)),
-                paint(quote_color, &String::from(quote)),
+                "{}{}{}",
+                paint(quote_color, quote),
+                paint(color, CstLineFeed { source: line, opts }),
+                paint(quote_color, quote),
             )?;
         }
     }
@@ -910,49 +918,68 @@ fn write_node_text(
     Ok(())
 }
 
-fn render_line_feed(source: &str, opts: &ParseFileOptions) -> String {
-    if cfg!(windows) {
-        source.replace("\r\n", &paint(opts.parse_theme.line_feed, "\r\n"))
-    } else {
-        source.replace('\n', &paint(opts.parse_theme.line_feed, "\n"))
+struct CstLineFeed<'src, 'opt> {
+    source: &'src str,
+    opts: &'src ParseFileOptions<'opt>,
+}
+
+impl std::fmt::Display for CstLineFeed<'_, '_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        #[cfg(windows)]
+        let lf = "\r\n";
+        #[cfg(not(windows))]
+        let lf = "\n";
+        let painted = paint(self.opts.parse_theme.line_feed, CstNodeText(lf));
+        let mut parts = self.source.split(lf);
+        if let Some(first) = parts.next() {
+            write!(f, "{}", CstNodeText(first))?;
+        }
+        for part in parts {
+            write!(f, "{painted}{}", CstNodeText(part))?;
+        }
+        Ok(())
     }
 }
 
-fn render_node_range(
-    opts: &ParseFileOptions,
-    cursor: &TreeCursor,
+struct CstNodeRange<'src, 'opt> {
+    opts: &'src ParseFileOptions<'opt>,
+    has_field_name: bool,
     is_named: bool,
     is_multiline: bool,
     total_width: usize,
     range: Range,
-) -> String {
-    let has_field_name = cursor.field_name().is_some();
-    let range_color = if is_named && !is_multiline && !has_field_name {
-        opts.parse_theme.row_color_named
-    } else {
-        opts.parse_theme.row_color
-    };
+}
 
-    let remaining_width_start = (total_width
-        - (range.start_point.row as f64).log10() as usize
-        - (range.start_point.column as f64).log10() as usize)
-        .max(1);
-    let remaining_width_end = (total_width
-        - (range.end_point.row as f64).log10() as usize
-        - (range.end_point.column as f64).log10() as usize)
-        .max(1);
-    paint(
-        range_color,
-        &format!(
-            "{}:{}{:remaining_width_start$}- {}:{}{:remaining_width_end$}",
-            range.start_point.row,
-            range.start_point.column,
-            ' ',
-            range.end_point.row,
-            range.end_point.column,
-            ' ',
-        ),
-    )
+impl std::fmt::Display for CstNodeRange<'_, '_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let start = self.range.start_point;
+        let end = self.range.end_point;
+        let range_color = if self.is_named && !self.is_multiline && !self.has_field_name {
+            self.opts.parse_theme.row_color_named
+        } else {
+            self.opts.parse_theme.row_color
+        };
+        let remaining_width = |row: usize, col: usize| {
+            (self
+                .total_width
+                .saturating_sub(row.checked_ilog10().unwrap_or(0) as usize)
+                .saturating_sub(col.checked_ilog10().unwrap_or(0) as usize))
+            .max(1)
+        };
+        let remaining_width_start = remaining_width(start.row, start.column);
+        let remaining_width_end = remaining_width(end.row, end.column);
+        write!(
+            f,
+            "{}",
+            paint(
+                range_color,
+                format_args!(
+                    "{}:{}{:remaining_width_start$}- {}:{}{:remaining_width_end$}",
+                    start.row, start.column, ' ', end.row, end.column, ' ',
+                ),
+            )
+        )
+    }
 }
 
 fn cst_render_node(
@@ -963,14 +990,21 @@ fn cst_render_node(
     total_width: usize,
     indent_level: usize,
     in_error: bool,
-) -> Result<()> {
+) -> io::Result<()> {
     let node = cursor.node();
     let is_named = node.is_named();
     if !opts.no_ranges {
         write!(
             out,
             "{}",
-            render_node_range(opts, cursor, is_named, false, total_width, node.range())
+            CstNodeRange {
+                opts,
+                has_field_name: cursor.field_name().is_some(),
+                is_named,
+                is_multiline: false,
+                total_width,
+                range: node.range(),
+            }
         )?;
     }
     write!(
@@ -988,7 +1022,7 @@ fn cst_render_node(
             write!(
                 out,
                 "{}",
-                paint(opts.parse_theme.field, &format!("{field_name}: "))
+                paint(opts.parse_theme.field, format_args!("{field_name}: "))
             )?;
         }
 
@@ -1103,30 +1137,25 @@ fn parse_edit_flag(source_code: &[u8], flag: &str) -> Result<Edit> {
 
 pub fn offset_for_position(input: &[u8], position: Point) -> Result<usize> {
     let mut row = 0;
-    let mut offset = 0;
-    let mut iter = memchr::memchr_iter(b'\n', input);
-    loop {
-        if let Some(pos) = iter.next()
-            && row < position.row
-        {
-            row += 1;
-            offset = pos;
-            continue;
+    let mut line_start = 0;
+    for line_end in memchr::memchr_iter(b'\n', input) {
+        if row == position.row {
+            if position.column > line_end - line_start {
+                return Err(anyhow!("Failed to address a column: {}", position.column));
+            }
+            return Ok(line_start + position.column);
         }
-        offset += 1;
-        break;
+        row += 1;
+        line_start = line_end + 1;
     }
-    if position.row - row > 0 {
+
+    if row != position.row {
         return Err(anyhow!("Failed to address a row: {}", position.row));
     }
-    if let Some(pos) = iter.next() {
-        if (pos - offset < position.column) || (input[offset] == b'\n' && position.column > 0) {
-            return Err(anyhow!("Failed to address a column: {}", position.column));
-        }
-    } else if input.len() - offset < position.column {
+    if position.column > input.len() - line_start {
         return Err(anyhow!("Failed to address a column over the end"));
     }
-    Ok(offset + position.column)
+    Ok(line_start + position.column)
 }
 
 pub fn position_for_offset(input: &[u8], offset: usize) -> Result<Point> {
@@ -1146,3 +1175,45 @@ pub fn position_for_offset(input: &[u8], offset: usize) -> Result<Point> {
     };
     Ok(result)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{offset_for_position, parse_edit_flag};
+    use tree_sitter::Point;
+
+    #[test]
+    fn offset_for_position_uses_zero_based_line_and_column_coordinates() {
+        let input = b"abc\n";
+        assert_eq!(
+            offset_for_position(input, Point { row: 0, column: 0 }).unwrap(),
+            0
+        );
+        assert_eq!(
+            offset_for_position(input, Point { row: 0, column: 1 }).unwrap(),
+            1
+        );
+        assert_eq!(
+            offset_for_position(input, Point { row: 0, column: 3 }).unwrap(),
+            3
+        );
+        assert_eq!(
+            offset_for_position(input, Point { row: 1, column: 0 }).unwrap(),
+            4
+        );
+    }
+
+    #[test]
+    fn offset_for_position_rejects_out_of_bounds_coordinates() {
+        let input = b"abc\ndef";
+        assert!(offset_for_position(input, Point { row: 0, column: 4 }).is_err());
+        assert!(offset_for_position(input, Point { row: 2, column: 0 }).is_err());
+    }
+
+    #[test]
+    fn parse_edit_flag_resolves_first_line_positions() {
+        let edit = parse_edit_flag(b"abc\n", "0,0 0 X").unwrap();
+        assert_eq!(edit.position, 0);
+        assert_eq!(edit.deleted_length, 0);
+        assert_eq!(edit.inserted_text, b"X");
+    }
+}

crates/cli/src/playground.html

@@ -470,7 +470,7 @@
     <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.js"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.19.0/clusterize.min.js"></script>
 
-    <script>LANGUAGE_BASE_URL = "";</script>
+    <script>LANGUAGE_BASE_URL = ".";</script>
     <script type="module" src="playground.js"></script>
     <script type="module">
       import * as TreeSitter from './web-tree-sitter.js';

crates/cli/src/playground.rs

@@ -3,7 +3,7 @@ use std::{
     env, fs,
     net::TcpListener,
     path::{Path, PathBuf},
-    str::{self, FromStr as _},
+    str::FromStr as _,
 };
 
 use anyhow::{Context, Result, anyhow};

crates/cli/src/query.rs

@@ -37,7 +37,7 @@ pub fn query_file_at_path(
     test_summary: Option<&mut TestSummary>,
 ) -> Result<()> {
     let stdout = io::stdout();
-    let mut stdout = stdout.lock();
+    let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
 
     let query_source = fs::read_to_string(query_path)
         .with_context(|| format!("Error reading query file {}", query_path.display()))?;
@@ -75,7 +75,7 @@ pub fn query_file_at_path(
     if opts.ordered_captures {
         let mut captures = query_cursor.captures(&query, tree.root_node(), source_code.as_slice());
         while let Some((mat, capture_index)) = captures.next() {
-            let capture = mat.captures[*capture_index];
+            let capture = mat.captures()[*capture_index];
             let capture_name = &query.capture_names()[capture.index as usize];
             if !opts.quiet && !should_test {
                 writeln!(
@@ -102,7 +102,7 @@ pub fn query_file_at_path(
             if !opts.quiet && !should_test {
                 writeln!(&mut stdout, "  pattern: {}", m.pattern_index)?;
             }
-            for capture in m.captures {
+            for capture in m.captures() {
                 let start = capture.node.start_position();
                 let end = capture.node.end_position();
                 let capture_name = &query.capture_names()[capture.index as usize];
@@ -142,7 +142,9 @@ pub fn query_file_at_path(
         };
         // Invariant: `test_summary` will always be `Some` when `should_test` is true
         let test_summary = test_summary.unwrap();
-        match query_testing::assert_expected_captures(&results, path, &mut parser, language) {
+        let assertions =
+            query_testing::parse_position_comments(&mut parser, language, source_code.as_slice())?;
+        match query_testing::assert_expected_captures(&results, &assertions) {
             Ok(assertion_count) => {
                 test_summary.query_results.add_case(TestResult {
                     name: path_name.to_string(),

crates/cli/src/query_testing.rs

@@ -1,4 +1,4 @@
-use std::{fs, path::Path, sync::LazyLock};
+use std::sync::LazyLock;
 
 use anyhow::{Result, anyhow};
 use bstr::{BStr, ByteSlice};
@@ -219,19 +219,14 @@ pub fn parse_position_comments(
     Ok(result)
 }
 
-pub fn assert_expected_captures(
-    infos: &[CaptureInfo],
-    path: &Path,
-    parser: &mut Parser,
-    language: &Language,
-) -> Result<usize> {
-    let contents = fs::read_to_string(path)?;
-    let pairs = parse_position_comments(parser, language, contents.as_bytes())?;
-    for assertion in &pairs {
+pub fn assert_expected_captures(infos: &[CaptureInfo], assertions: &[Assertion]) -> Result<usize> {
+    for assertion in assertions {
         if let Some(found) = &infos.iter().find(|p| {
-            assertion.position >= p.start
-                && (assertion.position.row < p.end.row
-                    || assertion.position.column + assertion.length - 1 < p.end.column)
+            let assertion_end = Utf8Point::new(
+                assertion.position.row,
+                assertion.position.column + assertion.length - 1,
+            );
+            assertion.position >= p.start && assertion_end < p.end
         }) {
             if assertion.expected_capture_name != found.name && found.name != "name" {
                 return Err(anyhow!(
@@ -250,5 +245,24 @@ pub fn assert_expected_captures(
             ));
         }
     }
-    Ok(pairs.len())
+    Ok(assertions.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Assertion, CaptureInfo, Utf8Point, assert_expected_captures};
+
+    #[test]
+    fn test_assertion_after_multiline_capture_does_not_match() {
+        let captures = [CaptureInfo {
+            name: "foo".to_string(),
+            start: Utf8Point::new(0, 0),
+            end: Utf8Point::new(1, 1),
+        }];
+        let assertions = [Assertion::new(2, 0, 1, false, "foo".to_string())];
+
+        let result = assert_expected_captures(&captures, &assertions);
+
+        assert!(result.is_err());
+    }
 }

crates/cli/src/tags.rs

@@ -2,7 +2,6 @@ use std::{
     fs,
     io::{self, Write},
     path::Path,
-    str,
     sync::{Arc, atomic::AtomicUsize},
     time::Instant,
 };
@@ -49,7 +48,7 @@ pub fn generate_tags(
                 &mut stdout,
                 "{indent_str}{:<10}\t | {:<8}\t{} {} - {} `{}`",
                 str::from_utf8(&source[tag.name_range]).unwrap_or(""),
-                &config.syntax_type_name(tag.syntax_type_id),
+                config.syntax_type_name(tag.syntax_type_id),
                 if tag.is_definition { "def" } else { "ref" },
                 tag.span.start,
                 tag.span.end,
@@ -59,7 +58,7 @@ pub fn generate_tags(
                 if docs.len() > 120 {
                     write!(&mut stdout, "\t{:?}...", docs.get(0..120).unwrap_or(""))?;
                 } else {
-                    write!(&mut stdout, "\t{:?}", &docs)?;
+                    write!(&mut stdout, "\t{docs:?}")?;
                 }
             }
             writeln!(&mut stdout)?;

crates/cli/src/templates/__init__.py

@@ -1,13 +1,15 @@
 """PARSER_DESCRIPTION"""
 
-from importlib.resources import files as _files
-
 from ._binding import language
 
 
 def _get_query(name, file):
+    files = globals().get("_files")
+    if files is None:
+        from importlib.resources import files
+        globals()["_files"] = files
     try:
-        query = _files(f"{__package__}") / file
+        query = files(f"{__package__}") / file
         globals()[name] = query.read_text()
     except FileNotFoundError:
         globals()[name] = None

crates/cli/src/templates/_cargo.toml

@@ -1,17 +1,17 @@
 [package]
-authors      = [ "PARSER_AUTHOR_NAME PARSER_AUTHOR_EMAIL" ]
+authors = [ "PARSER_AUTHOR_NAME PARSER_AUTHOR_EMAIL" ]
 autoexamples = false
-categories   = [ "parser-implementations", "parsing", "text-editors" ]
-description  = "PARSER_DESCRIPTION"
-edition      = "2024"
-keywords     = [ "incremental", "parsing", "tree-sitter", "PARSER_NAME" ]
-license      = "PARSER_LICENSE"
-name         = "tree-sitter-PARSER_NAME"
-readme       = "README.md"
-repository   = "PARSER_URL"
-version      = "PARSER_VERSION"
+categories = [ "parser-implementations", "parsing", "text-editors" ]
+description = "PARSER_DESCRIPTION"
+edition = "2024"
+keywords = [ "incremental", "parsing", "tree-sitter", "PARSER_NAME" ]
+license = "PARSER_LICENSE"
+name = "tree-sitter-PARSER_NAME"
+readme = "README.md"
+repository = "PARSER_URL"
+version = "PARSER_VERSION"
 
-build   = "bindings/rust/build.rs"
+build = "bindings/rust/build.rs"
 include = [ "bindings/rust/*", "grammar.js", "queries/*", "src/*", "tree-sitter.json", "/LICENSE" ]
 
 [lib]

crates/cli/src/templates/build.rs

@@ -11,18 +11,8 @@ fn main() {
         let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
             panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
         };
-        let Ok(wasm_src) =
-            std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
-        else {
-            panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
-        };
 
         c_config.include(&wasm_headers);
-        c_config.files([
-            wasm_src.join("stdio.c"),
-            wasm_src.join("stdlib.c"),
-            wasm_src.join("string.c"),
-        ]);
     }
 
     let parser_path = src_dir.join("parser.c");

crates/cli/src/templates/build.zig

@@ -4,46 +4,51 @@ pub fn build(b: *std.Build) !void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
+    var threaded: std.Io.Threaded = .init(b.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
     const shared = b.option(bool, "build-shared", "Build a shared library") orelse true;
     const reuse_alloc = b.option(bool, "reuse-allocator", "Reuse the library allocator") orelse false;
 
     const library_name = "tree-sitter-PARSER_NAME";
 
+    var grammar = b.createModule(.{
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .pic = if (shared) true else null,
+    });
     const lib: *std.Build.Step.Compile = b.addLibrary(.{
         .name = library_name,
         .linkage = if (shared) .dynamic else .static,
-        .root_module = b.createModule(.{
-            .target = target,
-            .optimize = optimize,
-            .link_libc = true,
-            .pic = if (shared) true else null,
-        }),
+        .root_module = grammar,
     });
 
-    lib.addCSourceFile(.{
+    grammar.addCSourceFile(.{
         .file = b.path("src/parser.c"),
         .flags = &.{"-std=c11"},
     });
-    if (fileExists(b, "src/scanner.c")) {
-        lib.addCSourceFile(.{
+    if (fileExists(b, io, "src/scanner.c")) {
+        grammar.addCSourceFile(.{
             .file = b.path("src/scanner.c"),
             .flags = &.{"-std=c11"},
         });
     }
 
     if (reuse_alloc) {
-        lib.root_module.addCMacro("TREE_SITTER_REUSE_ALLOCATOR", "");
+        grammar.addCMacro("TREE_SITTER_REUSE_ALLOCATOR", "");
     }
     if (optimize == .Debug) {
-        lib.root_module.addCMacro("TREE_SITTER_DEBUG", "");
+        grammar.addCMacro("TREE_SITTER_DEBUG", "");
     }
 
-    lib.addIncludePath(b.path("src"));
+    grammar.addIncludePath(b.path("src"));
 
     b.installArtifact(lib);
     b.installFile("src/node-types.json", "node-types.json");
 
-    if (fileExists(b, "queries")) {
+    if (fileExists(b, io, "queries")) {
         b.installDirectory(.{
             .source_dir = b.path("queries"),
             .install_dir = .prefix,
@@ -69,16 +74,10 @@ pub fn build(b: *std.Build) !void {
     tests.root_module.addImport(library_name, module);
 
     // HACK: fetch tree-sitter dependency only when testing this module
-    if (b.pkg_hash.len == 0) {
-        var args = try std.process.argsWithAllocator(b.allocator);
-        defer args.deinit();
-        while (args.next()) |a| {
-            if (std.mem.eql(u8, a, "test")) {
-                const ts_dep = b.lazyDependency("tree_sitter", .{}) orelse continue;
-                tests.root_module.addImport("tree-sitter", ts_dep.module("tree-sitter"));
-                break;
-            }
-        }
+    if (b.option(bool, "test", "Fetch test dependencies") orelse false) {
+        const ts_dep = b.lazyDependency("tree_sitter", .{});
+        if (ts_dep) |dep|
+            tests.root_module.addImport("tree-sitter", dep.module("tree_sitter"));
     }
 
     const run_tests = b.addRunArtifact(tests);
@@ -86,8 +85,8 @@ pub fn build(b: *std.Build) !void {
     test_step.dependOn(&run_tests.step);
 }
 
-inline fn fileExists(b: *std.Build, filename: []const u8) bool {
+inline fn fileExists(b: *std.Build, io: std.Io, filename: []const u8) bool {
     const dir = b.build_root.handle;
-    dir.access(filename, .{}) catch return false;
+    dir.access(io, filename, .{}) catch return false;
     return true;
 }

crates/cli/src/templates/build.zig.zon

@@ -1,11 +1,12 @@
 .{
     .name = .tree_sitter_PARSER_NAME,
     .fingerprint = PARSER_FINGERPRINT,
+    .minimum_zig_version = "0.16.0",
     .version = "PARSER_VERSION",
     .dependencies = .{
         .tree_sitter = .{
-            .url = "git+https://github.com/tree-sitter/zig-tree-sitter#b4b72c903e69998fc88e27e154a5e3cc9166551b",
-            .hash = "tree_sitter-0.25.0-8heIf51vAQConvVIgvm-9mVIbqh7yabZYqPXfOpS3YoG",
+            .url = "git+https://github.com/tree-sitter/zig-tree-sitter#0cf58172e61f6fdd16f681cde42b4acb531a23db",
+            .hash = "tree_sitter-0.26.0-8heIf3CaAQDeVTQc0DMSBhbQAEx5aF-dTen4_LPxMgrv",
             .lazy = true,
         },
     },

crates/cli/src/templates/index.d.ts

@@ -21,7 +21,7 @@ type NodeInfo =
 /**
  * The tree-sitter language object for this grammar.
  *
- * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language}
+ * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Language.html Parser.Language}
  *
  * @example
  * import Parser from "tree-sitter";

crates/cli/src/templates/makefile

@@ -1,6 +1,7 @@
 LANGUAGE_NAME := tree-sitter-KEBAB_PARSER_NAME
 HOMEPAGE_URL := PARSER_URL
 VERSION := PARSER_VERSION
+DESCRIPTION := PARSER_DESCRIPTION
 
 # repository
 SRC_DIR := src

crates/cli/src/templates/package.swift

@@ -15,7 +15,7 @@ let package = Package(
         .library(name: "PARSER_CLASS_NAME", targets: ["PARSER_CLASS_NAME"]),
     ],
     dependencies: [
-        .package(name: "SwiftTreeSitter", url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.9.0"),
+        .package(url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.10.0"),
     ],
     targets: [
         .target(
@@ -32,7 +32,7 @@ let package = Package(
         .testTarget(
             name: "PARSER_CLASS_NAMETests",
             dependencies: [
-                "SwiftTreeSitter",
+                .product(name: "SwiftTreeSitter", package: "swift-tree-sitter"),
                 "PARSER_CLASS_NAME",
             ],
             path: "bindings/swift/PARSER_CLASS_NAMETests"

crates/cli/src/templates/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 build-backend = "setuptools.build_meta"
-requires      = [ "setuptools>=62.4.0", "wheel" ]
+requires = [ "setuptools>=62.4.0", "wheel" ]
 
 [project]
 authors = [ { email = "PARSER_AUTHOR_EMAIL", name = "PARSER_AUTHOR_NAME" } ]
@@ -19,12 +19,12 @@ requires-python = ">=3.10"
 version = "PARSER_VERSION"
 
 [project.urls]
-Funding  = "FUNDING_URL"
+Funding = "FUNDING_URL"
 Homepage = "PARSER_URL"
 
 [project.optional-dependencies]
 core = [ "tree-sitter~=0.24" ]
 
 [tool.cibuildwheel]
-build          = "cp310-*"
+build = "cp310-*"
 build-frontend = "build"

crates/cli/src/templates/setup.py

@@ -42,6 +42,7 @@ class EggInfo(egg_info):
         super().find_sources()
         self.filelist.recursive_include("queries", "*.scm")
         self.filelist.include("src/tree_sitter/*.h")
+        self.filelist.include("src/*.c")
 
 
 setup(

crates/cli/src/test_highlight.rs

@@ -13,10 +13,10 @@ use crate::{
 
 #[derive(Debug)]
 pub struct Failure {
-    row: usize,
-    column: usize,
-    expected_highlight: String,
-    actual_highlights: Vec<String>,
+    pub(crate) row: usize,
+    pub(crate) column: usize,
+    pub(crate) expected_highlight: String,
+    pub(crate) actual_highlights: Vec<String>,
 }
 
 impl std::error::Error for Failure {}
@@ -122,6 +122,7 @@ pub fn test_highlights(
 
     if failed { Err(anyhow!("")) } else { Ok(()) }
 }
+
 pub fn iterate_assertions(
     assertions: &[Assertion],
     highlights: &[(Utf8Point, Utf8Point, Highlight)],
@@ -138,49 +139,48 @@ pub fn iterate_assertions(
         expected_capture_name: expected_highlight,
     } in assertions
     {
-        let mut passed = false;
-        let mut end_column = position.column + length - 1;
+        // Iterate through all of the highlights that start at or before this assertion's
+        // position, looking for one that matches the assertion.
         actual_highlights.clear();
-
-        // The assertions are ordered by position, so skip past all of the highlights that
-        // end at or before this assertion's position.
-        'highlight_loop: while let Some(highlight) = highlights.get(i) {
+        let mut passed = false;
+        let end_column = position.column + length - 1;
+        for highlight in &highlights[i..] {
+            // The assertions are ordered by position, so skip past all of the highlights that
+            // end at or before this assertion's position.
             if highlight.1 <= *position {
                 i += 1;
                 continue;
             }
+            if (highlight.0.row > position.row)
+                || (highlight.0.row == position.row && highlight.0.column > end_column)
+            {
+                break;
+            }
 
-            // Iterate through all of the highlights that start at or before this assertion's
-            // position, looking for one that matches the assertion.
-            let mut j = i;
-            while let (false, Some(highlight)) = (passed, highlights.get(j)) {
-                end_column = position.column + length - 1;
-                if highlight.0.row >= position.row && highlight.0.column > end_column {
-                    break 'highlight_loop;
-                }
-
-                // If the highlight matches the assertion, or if the highlight doesn't
-                // match the assertion but it's negative, this test passes. Otherwise,
-                // add this highlight to the list of actual highlights that span the
-                // assertion's position, in order to generate an error message in the event
-                // of a failure.
-                let highlight_name = &highlight_names[(highlight.2).0];
-                if (*highlight_name == *expected_highlight) == *negative {
-                    actual_highlights.push(highlight_name);
-                } else {
-                    passed = true;
-                    break 'highlight_loop;
-                }
-
-                j += 1;
+            // If the highlight matches the assertion, or if the highlight doesn't
+            // match the assertion but it's negative, this test passes. Otherwise,
+            // add this highlight to the list of actual highlights that span the
+            // assertion's position, in order to generate an error message in the event
+            // of a failure.
+            let highlight_name = &highlight_names[(highlight.2).0];
+            if (*highlight_name == *expected_highlight) == *negative {
+                actual_highlights.push(highlight_name);
+            } else {
+                passed = true;
+                break;
             }
         }
 
         if !passed {
+            let mut expected = String::with_capacity(expected_highlight.len() + 1);
+            if *negative {
+                expected.push('!');
+            }
+            expected.push_str(expected_highlight);
             return Err(Failure {
                 row: position.row,
                 column: end_column,
-                expected_highlight: expected_highlight.clone(),
+                expected_highlight: expected,
                 actual_highlights: actual_highlights.into_iter().cloned().collect(),
             }
             .into());

crates/cli/src/tests.rs

@@ -17,7 +17,7 @@ mod tree_test;
 #[cfg(feature = "wasm")]
 mod wasm_language_test;
 
-use tree_sitter_generate::GenerateResult;
+use tree_sitter_generate::{GenerateResult, OptLevel};
 
 pub use crate::fuzz::{
     ITERATION_COUNT, allocations,
@@ -30,5 +30,10 @@ pub use helpers::fixtures::get_language;
 /// This is a simple wrapper around [`tree_sitter_generate::generate_parser_for_grammar`], because
 /// our tests do not need to pass in a version number, only the grammar JSON.
 fn generate_parser(grammar_json: &str) -> GenerateResult<(String, String)> {
-    tree_sitter_generate::generate_parser_for_grammar(grammar_json, Some((0, 0, 0)))
+    tree_sitter_generate::generate_parser_for_grammar(
+        grammar_json,
+        Some((0, 0, 0)),
+        OptLevel::default(),
+        &mut Vec::new(),
+    )
 }

crates/cli/src/tests/corpus_test.rs

@@ -2,6 +2,7 @@ use std::{collections::HashMap, env, fs};
 
 use anyhow::Context;
 use tree_sitter::Parser;
+use tree_sitter_generate::OptLevel;
 use tree_sitter_proc_macro::test_with_seed;
 
 use crate::{
@@ -16,7 +17,7 @@ use crate::{
         random::Rand,
     },
     parse::perform_edit,
-    test::{DiffKey, TestDiff, parse_tests, strip_sexp_fields},
+    test::{DiffKey, TestDiff, parse_tests, render_test_output},
     tests::{
         allocations,
         helpers::fixtures::{SCRATCH_BASE_DIR, fixtures_dir, get_language, get_test_language},
@@ -195,28 +196,7 @@ pub fn test_language_corpus(
 
         println!("  {test_index}. {test_name}");
 
-        let passed = allocations::record(|| {
-            let mut log_session = None;
-            let mut parser = get_parser(&mut log_session, "log.html");
-            parser.set_language(&language).unwrap();
-            set_included_ranges(&mut parser, &test.input, test.template_delimiters);
-
-            let tree = parser.parse(&test.input, None).unwrap();
-            let mut actual_output = tree.root_node().to_sexp();
-            if !test.has_fields {
-                actual_output = strip_sexp_fields(&actual_output);
-            }
-
-            if actual_output != test.output {
-                println!("Incorrect initial parse for {test_name}");
-                DiffKey::print();
-                println!("{}", TestDiff::new(&actual_output, &test.output));
-                println!();
-                return false;
-            }
-
-            true
-        });
+        let passed = allocations::record(|| test.check_initial_parse(&language, &test_name, true));
 
         if !passed {
             failure_count += 1;
@@ -292,10 +272,8 @@ pub fn test_language_corpus(
                 let tree3 = parser.parse(&input, Some(&tree2)).unwrap();
 
                 // Verify that the final tree matches the expectation from the corpus.
-                let mut actual_output = tree3.root_node().to_sexp();
-                if !test.has_fields {
-                    actual_output = strip_sexp_fields(&actual_output);
-                }
+                let actual_output =
+                    render_test_output(&input, &tree3, test.cst, test.has_fields).unwrap();
 
                 if actual_output != test.output {
                     println!("Incorrect parse for {test_name} - seed {seed}");
@@ -375,8 +353,12 @@ fn test_feature_corpus_files() {
                 )
             })
             .unwrap();
-        let generate_result =
-            tree_sitter_generate::generate_parser_for_grammar(&grammar_json, Some((0, 0, 0)));
+        let generate_result = tree_sitter_generate::generate_parser_for_grammar(
+            &grammar_json,
+            Some((0, 0, 0)),
+            OptLevel::default(),
+            &mut Vec::new(),
+        );
 
         if error_message_path.exists() {
             if EXAMPLE_INCLUDE.is_some() || EXAMPLE_EXCLUDE.is_some() {
@@ -397,12 +379,12 @@ fn test_feature_corpus_files() {
                     failure_count += 1;
                 }
             } else {
-                eprintln!("Expected error message but got none for test grammar '{language_name}'",);
+                eprintln!("Expected error message but got none for test grammar '{language_name}'");
                 failure_count += 1;
             }
         } else {
             if let Err(e) = &generate_result {
-                eprintln!("Unexpected error for test grammar '{language_name}':\n{e}",);
+                eprintln!("Unexpected error for test grammar '{language_name}':\n{e}");
                 failure_count += 1;
                 continue;
             }
@@ -420,24 +402,8 @@ fn test_feature_corpus_files() {
             for test in tests {
                 eprintln!("  example: {:?}", test.name);
 
-                let passed = allocations::record(|| {
-                    let mut log_session = None;
-                    let mut parser = get_parser(&mut log_session, "log.html");
-                    parser.set_language(&language).unwrap();
-                    let tree = parser.parse(&test.input, None).unwrap();
-                    let mut actual_output = tree.root_node().to_sexp();
-                    if !test.has_fields {
-                        actual_output = strip_sexp_fields(&actual_output);
-                    }
-                    if actual_output == test.output {
-                        true
-                    } else {
-                        DiffKey::print();
-                        print!("{}", TestDiff::new(&actual_output, &test.output));
-                        println!();
-                        false
-                    }
-                });
+                let passed =
+                    allocations::record(|| test.check_initial_parse(&language, &test.name, true));
 
                 if !passed {
                     failure_count += 1;

crates/cli/src/tests/detect_language.rs

@@ -127,6 +127,52 @@ fn detect_language_by_double_barrel_file_extension() {
     );
 }
 
+#[test]
+fn detect_language_with_dots_in_filename() {
+    let blade_dir = tree_sitter_dir(
+        r#"{
+  "grammars": [
+    {
+      "name": "blade_dots",
+      "path": ".",
+      "scope": "source.blade",
+      "file-types": [
+        "blade.php"
+      ]
+    },
+    {
+      "name": "php_dots",
+      "path": ".",
+      "scope": "source.php",
+      "file-types": [
+        "php"
+      ]
+    }
+  ],
+  "metadata": {
+    "version": "0.0.1"
+  }
+}
+"#,
+        "blade_dots",
+    );
+
+    let mut loader = Loader::with_parser_lib_path(scratch_dir().to_path_buf());
+    let config = loader
+        .find_language_configurations_at_path(blade_dir.path(), false)
+        .unwrap();
+
+    // this is just to validate that we can read the tree-sitter.json correctly
+    assert_eq!(config[0].scope.as_ref().unwrap(), "source.blade");
+
+    let file_name = blade_dir.path().join("foo.bar.baz.blade.php");
+    fs::write(&file_name, "").unwrap();
+    assert_eq!(
+        get_lang_scope(&loader, &file_name),
+        Some("source.blade".into())
+    );
+}
+
 #[test]
 fn detect_language_without_filename() {
     let gitignore_dir = tree_sitter_dir(

crates/cli/src/tests/helpers/edits.rs

@@ -1,4 +1,4 @@
-use std::{ops::Range, str};
+use std::ops::Range;
 
 #[derive(Debug)]
 pub struct ReadRecorder<'a> {

crates/cli/src/tests/helpers/fixtures.rs

@@ -1,7 +1,8 @@
 use std::{
+    collections::HashSet,
     env, fs,
     path::{Path, PathBuf},
-    sync::LazyLock,
+    sync::{LazyLock, Mutex},
 };
 
 use anyhow::Context;
@@ -23,6 +24,10 @@ static TEST_LOADER: LazyLock<Loader> = LazyLock::new(|| {
     loader
 });
 
+// Prevents parallel tests from racing on the same per-grammar
+// `src_dir/tree_sitter/` and observing a half-rewritten header.
+static WRITTEN_HEADER_DIRS: LazyLock<Mutex<HashSet<PathBuf>>> = LazyLock::new(Default::default);
+
 #[cfg(feature = "wasm")]
 pub static ENGINE: LazyLock<tree_sitter::wasmtime::Engine> = LazyLock::new(Default::default);
 
@@ -134,17 +139,22 @@ fn get_test_language_internal(
     };
 
     let header_path = src_dir.join("tree_sitter");
-    fs::create_dir_all(&header_path).unwrap();
-
-    for (file, content) in [
-        ("alloc.h", ALLOC_HEADER),
-        ("array.h", ARRAY_HEADER),
-        ("parser.h", tree_sitter::PARSER_HEADER),
-    ] {
-        let file = header_path.join(file);
-        fs::write(&file, content)
-            .with_context(|| format!("Failed to write {:?}", file.file_name().unwrap()))
-            .unwrap();
+    if WRITTEN_HEADER_DIRS
+        .lock()
+        .unwrap()
+        .insert(header_path.clone())
+    {
+        fs::create_dir_all(&header_path).unwrap();
+        for (file, content) in [
+            ("alloc.h", ALLOC_HEADER),
+            ("array.h", ARRAY_HEADER),
+            ("parser.h", tree_sitter::PARSER_HEADER),
+        ] {
+            let path = header_path.join(file);
+            fs::write(&path, content)
+                .with_context(|| format!("Failed to write {}", path.display()))
+                .unwrap();
+        }
     }
 
     let paths_to_check = if let Some(scanner_path) = &scanner_path {

crates/cli/src/tests/helpers/query_helpers.rs

@@ -7,10 +7,10 @@ use tree_sitter::{
 };
 
 #[derive(Debug)]
-pub struct Pattern {
-    kind: Option<&'static str>,
+pub struct Pattern<'a> {
+    kind: Option<&'a str>,
     named: bool,
-    field: Option<&'static str>,
+    field: Option<&'a str>,
     capture: Option<String>,
     children: Vec<Self>,
 }
@@ -25,8 +25,8 @@ const CAPTURE_NAMES: &[&str] = &[
     "one", "two", "three", "four", "five", "six", "seven", "eight",
 ];
 
-impl Pattern {
-    pub fn random_pattern_in_tree(tree: &Tree, rng: &mut impl Rng) -> (Self, Range<Point>) {
+impl<'a> Pattern<'a> {
+    pub fn random_pattern_in_tree(tree: &'a Tree, rng: &mut impl Rng) -> (Self, Range<Point>) {
         let mut cursor = tree.walk();
 
         // Descend to the node at a random byte offset and depth.
@@ -75,7 +75,7 @@ impl Pattern {
         (pattern, pattern_start..pattern_end)
     }
 
-    fn random_pattern_for_node(cursor: &mut TreeCursor, rng: &mut impl Rng) -> Self {
+    fn random_pattern_for_node(cursor: &mut TreeCursor<'a>, rng: &mut impl Rng) -> Self {
         let node = cursor.node();
 
         // Sometimes specify the node's type, sometimes use a wildcard.
@@ -268,7 +268,7 @@ impl Pattern {
     }
 }
 
-impl std::fmt::Display for Pattern {
+impl std::fmt::Display for Pattern<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let mut result = String::new();
         self.write_to_string(&mut result, 0);
@@ -336,7 +336,7 @@ pub fn collect_matches<'a>(
     while let Some(m) = matches.next() {
         result.push((
             m.pattern_index,
-            format_captures(m.captures.iter().into_streaming_iter_ref(), query, source),
+            format_captures(m.captures().iter().into_streaming_iter_ref(), query, source),
         ));
     }
     result
@@ -347,7 +347,7 @@ pub fn collect_captures<'a>(
     query: &'a Query,
     source: &'a str,
 ) -> Vec<(&'a str, &'a str)> {
-    format_captures(captures.map(|(m, i)| m.captures[*i]), query, source)
+    format_captures(captures.map(|(m, i)| m.captures()[*i]), query, source)
 }
 
 fn format_captures<'a>(

crates/cli/src/tests/highlight_test.rs

@@ -2,7 +2,7 @@ use std::{
     ffi::CString,
     fs,
     os::raw::c_char,
-    ptr, slice, str,
+    ptr, slice,
     sync::{
         LazyLock,
         atomic::{AtomicUsize, Ordering},
@@ -496,7 +496,7 @@ fn test_highlighting_cancellation() {
     let found_cancellation_error = events.any(|event| match event {
         Ok(_) => false,
         Err(Error::Cancelled) => true,
-        Err(Error::InvalidLanguage | Error::Unknown) => {
+        Err(Error::InvalidLanguage(_) | Error::Unknown) => {
             unreachable!("Unexpected error type while iterating events")
         }
     });

crates/cli/src/tests/language_test.rs

@@ -31,11 +31,12 @@ fn test_lookahead_iterator() {
     let mut lookahead = language.lookahead_iterator(next_state).unwrap();
     assert_eq!(*lookahead.language(), language);
     assert!(lookahead.iter_names().eq(expected_symbols));
+    assert_eq!(lookahead.iter_names().count(), 0);
 
-    lookahead.reset_state(next_state);
+    assert!(lookahead.reset_state(next_state));
     assert!(lookahead.iter_names().eq(expected_symbols));
 
-    lookahead.reset(&language, next_state);
+    assert!(lookahead.reset(&language, next_state));
     assert!(
         lookahead
             .map(|s| language.node_kind_for_id(s).unwrap())
@@ -66,6 +67,33 @@ fn test_lookahead_iterator_modifiable_only_by_mut() {
     let _ = names.next();
 }
 
+#[test]
+fn test_lookahead_iterator_exhaustion() {
+    let language = get_language("json");
+
+    for state in 0..language.parse_state_count() {
+        let state = u16::try_from(state).unwrap();
+        let mut lookahead = language.lookahead_iterator(state).unwrap();
+
+        // A fresh iterator is not positioned on a symbol.
+        assert_eq!(lookahead.current_symbol(), None);
+        assert_eq!(lookahead.current_symbol_name(), None);
+
+        let count = lookahead.by_ref().count();
+
+        // An exhausted iterator is not positioned on a symbol, and stays exhausted.
+        assert_eq!(lookahead.current_symbol(), None);
+        assert_eq!(lookahead.current_symbol_name(), None);
+        assert_eq!(lookahead.by_ref().count(), 0);
+        assert_eq!(lookahead.iter_names().count(), 0);
+
+        // Resetting restores it exactly.
+        assert!(lookahead.reset_state(state));
+        assert_eq!(lookahead.current_symbol(), None);
+        assert_eq!(lookahead.by_ref().count(), count);
+    }
+}
+
 #[test]
 fn test_symbol_metadata_checks() {
     let language = get_language("rust");

crates/cli/src/tests/node_test.rs

@@ -454,7 +454,7 @@ fn test_node_child_by_field_name_with_extra_hidden_children() {
     // In the Python grammar, some fields are applied to `suite` nodes,
     // which consist of an invisible `indent` token followed by a block.
     // Check that when searching for a child with a field name, we don't
-    //
+    // return a hidden child node.
     let tree = parser.parse("while a:\n  pass", None).unwrap();
     let while_node = tree.root_node().child(0).unwrap();
     assert_eq!(while_node.kind(), "while_statement");
@@ -950,6 +950,13 @@ fn test_node_sexp() {
 
 #[test]
 fn test_node_field_names() {
+    // - "x":
+    //      This isn't used in the test, but prevents `_hidden_rule1` from being eliminated as a
+    //      unit reduction.
+    // - "_hidden_rule1":
+    //      Fields pointing to hidden nodes with a single child resolve to the child.
+    // - "_hidden_rule2":
+    //      Fields within hidden nodes can be referenced through the parent node.
     let (parser_name, parser_code) = generate_parser(
         r#"
         {
@@ -972,8 +979,6 @@ fn test_node_field_names() {
                                 {"type": "STRING", "value": "child-1"},
                                 {"type": "BLANK"},
 
-                                // This isn't used in the test, but prevents `_hidden_rule1`
-                                // from being eliminated as a unit reduction.
                                 {
                                     "type": "ALIAS",
                                     "value": "x",
@@ -994,7 +999,6 @@ fn test_node_field_names() {
                     ]
                 },
 
-                // Fields pointing to hidden nodes with a single child resolve to the child.
                 "_hidden_rule1": {
                     "type": "CHOICE",
                     "members": [
@@ -1003,7 +1007,6 @@ fn test_node_field_names() {
                     ]
                 },
 
-                // Fields within hidden nodes can be referenced through the parent node.
                 "_hidden_rule2": {
                     "type": "SEQ",
                     "members": [

crates/cli/src/tests/parser_test.rs

@@ -66,9 +66,13 @@ fn test_parsing_with_logging() {
     parser.set_language(&get_language("rust")).unwrap();
 
     let mut messages = Vec::new();
-    parser.set_logger(Some(Box::new(|log_type, message| {
-        messages.push((log_type, message.to_string()));
-    })));
+    // SAFETY: the logger borrows `messages` and is only invoked during the
+    // `parse` call below while `messages` is in scope.
+    unsafe {
+        parser.set_logger_unchecked(Some(Box::new(|log_type, message| {
+            messages.push((log_type, message.to_string()));
+        })));
+    }
 
     parser
         .parse(
@@ -249,6 +253,79 @@ fn test_parsing_with_custom_utf16_be_input() {
     assert_eq!(root.child(0).unwrap().kind(), "function_item");
 }
 
+#[test]
+fn test_utf16_decodes_surrogate_pairs() {
+    let mut parser = Parser::new();
+    let language = get_test_fixture_language("utf16_surrogate_oob");
+    parser.set_language(&language).unwrap();
+
+    let le = [0xD83D_u16.to_le(), 0xDE00_u16.to_le()];
+    let tree = parser.parse_utf16_le(le, None).unwrap();
+    assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))");
+
+    let be = [0xD83D_u16.to_be(), 0xDE00_u16.to_be()];
+    let tree = parser.parse_utf16_be(be, None).unwrap();
+    assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))");
+}
+
+#[test]
+fn test_utf16_decode_does_not_read_oob() {
+    // Test for a buffer over-read in ts_decode_utf16_le/be when a lead surrogate
+    // is the last code unit in a chunk. The test grammar's external scanner
+    // distinguishes surrogate code points from supplementary-plane characters,
+    // making the over-read directly observable in the parse tree.
+    //
+    // Buffer layout:
+    //   buf[0] = 0xD83E  (lead surrogate)
+    //   buf[1] = 0xDD8B  (POISON: fake trail surrogate, adjacent in memory)
+    //
+    // The callback returns only buf[0..1] (one code unit = 2 bytes).
+    //
+    // When functioning correctly, this test passes a length of 2 bytes, which is
+    // interpreted as 2/2 = 1 code unit, and thus doesn't over-read into the "poison"
+    // fake trail surrogate. If an over-read does occur, the scanner sees a
+    // supplementary token.
+    let mut parser = Parser::new();
+    let language = get_test_fixture_language("utf16_surrogate_oob");
+    parser.set_language(&language).unwrap();
+
+    let buf = vec![
+        0xD83E, // lead surrogate (the only "visible" code unit)
+        0xDD8B, // POISON: adjacent in Vec memory, past the chunk
+    ];
+    assert_eq!("🦋", String::from_utf16(&buf).unwrap());
+
+    let mut callback = |offset: usize, _position: Point| -> &[u16] {
+        // only expose buf[0], never buf[1]
+        if offset >= 1 {
+            return [].as_slice();
+        }
+        &buf[0..1]
+    };
+
+    // Use the parse function matching the host endianness, since the
+    // buffer contains native u16 values.
+    #[cfg(target_endian = "little")]
+    let tree = parser
+        .parse_utf16_le_with_options(&mut callback, None, None)
+        .unwrap();
+    #[cfg(target_endian = "big")]
+    let tree = parser
+        .parse_utf16_be_with_options(&mut callback, None, None)
+        .unwrap();
+
+    let root = tree.root_node();
+
+    // Correct: scanner sees raw surrogate (0xD83E) -> `surrogate` node
+    // Incorrect: scanner sees supplementary (U+1F98B, aka 🦋) -> `supplementary` node
+    assert_eq!(
+        root.to_sexp(),
+        "(program (surrogate))",
+        "buffer over-read: decoder read past chunk boundary and formed a \
+         supplementary character from OOB adjacent memory"
+    );
+}
+
 #[test]
 fn test_parsing_with_callback_returning_owned_strings() {
     let mut parser = Parser::new();
@@ -1059,7 +1136,7 @@ fn test_parsing_with_timeout_during_balancing() {
                 Some(ParseOptions::new().progress_callback(&mut |state| {
                     // Because we've already finished parsing, we should only be resuming the
                     // balancing phase.
-                    assert!(state.current_byte_offset() == current_byte_offset);
+                    assert_eq!(state.current_byte_offset(), current_byte_offset);
                     ControlFlow::Continue(())
                 })),
             )
@@ -1727,11 +1804,15 @@ fn test_parsing_with_scanner_logging() {
         .unwrap();
 
     let mut found = false;
-    parser.set_logger(Some(Box::new(|log_type, message| {
-        if log_type == LogType::Lex && message == "Found a percent string" {
-            found = true;
-        }
-    })));
+    // SAFETY: the logger borrows `found` and is only invoked during the `parse`
+    // call below, while `found` is in scope.
+    unsafe {
+        parser.set_logger_unchecked(Some(Box::new(|log_type, message| {
+            if log_type == LogType::Lex && message == "Found a percent string" {
+                found = true;
+            }
+        })));
+    }
 
     let source_code = "x + %(sup (external) scanner?)";
 

crates/cli/src/tests/proc_macro/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
-name                   = "tree-sitter-tests-proc-macro"
-version                = "0.0.0"
-edition.workspace      = true
+name = "tree-sitter-tests-proc-macro"
+version = "0.0.0"
+edition.workspace = true
 rust-version.workspace = true
-publish                = false
+publish = false
 
 [lints]
 workspace = true
@@ -13,5 +13,5 @@ proc-macro = true
 
 [dependencies]
 proc-macro2 = "1.0.93"
-quote       = "1.0.38"
-syn         = { features = [ "full" ], version = "2.0.96" }
+quote = "1.0.38"
+syn = { features = [ "full" ], version = "2.0.96" }

crates/cli/src/tests/query_test.rs

@@ -5,8 +5,8 @@ use rand::{SeedableRng, prelude::StdRng};
 use streaming_iterator::StreamingIterator;
 use tree_sitter::{
     CaptureQuantifier, InputEdit, Language, Node, Parser, Point, Query, QueryCursor,
-    QueryCursorOptions, QueryError, QueryErrorKind, QueryPredicate, QueryPredicateArg,
-    QueryProperty, Range,
+    QueryCursorOptions, QueryCursorState, QueryError, QueryErrorKind, QueryPredicate,
+    QueryPredicateArg, QueryProperty, Range,
 };
 use tree_sitter_generate::load_grammar_file;
 use unindent::Unindent;
@@ -256,6 +256,51 @@ fn test_query_errors_on_invalid_syntax() {
     });
 }
 
+#[test]
+fn test_query_errors_on_anchor_at_group_edge() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+
+        // Anchors between siblings, or at the first/last position of a *node*
+        // pattern, are valid.
+        assert!(Query::new(&language, "((_) . (_))").is_ok());
+        assert!(Query::new(&language, "(program (_) (_) .)").is_ok());
+        assert!(Query::new(&language, "(program (_)* @x . (_))").is_ok());
+
+        // A `.` at the edge of a *group* is rejected. A group is not a node, so it
+        // has no last child to anchor, and there is no sibling within the group to
+        // anchor to.
+        assert_eq!(
+            Query::new(&language, "((_) .)").unwrap_err(),
+            QueryError {
+                row: 0,
+                offset: 5,
+                column: 5,
+                kind: QueryErrorKind::Syntax,
+                message: [
+                    "((_) .)", //
+                    "     ^"
+                ]
+                .join("\n")
+            }
+        );
+        assert_eq!(
+            Query::new(&language, "(program ((_)+ .)? (_))").unwrap_err(),
+            QueryError {
+                row: 0,
+                offset: 15,
+                column: 15,
+                kind: QueryErrorKind::Syntax,
+                message: [
+                    "(program ((_)+ .)? (_))", //
+                    "               ^"
+                ]
+                .join("\n")
+            }
+        );
+    });
+}
+
 #[test]
 fn test_query_errors_on_invalid_symbols() {
     allocations::record(|| {
@@ -371,6 +416,16 @@ fn test_query_errors_on_invalid_symbols() {
                 message: "\"fakefield\"".to_string()
             }
         );
+        assert_eq!(
+            Query::new(&language, "(MISS)").unwrap_err(),
+            QueryError {
+                row: 0,
+                offset: 1,
+                column: 1,
+                kind: QueryErrorKind::NodeType,
+                message: "\"MISS\"".to_string(),
+            }
+        );
     });
 }
 
@@ -1108,8 +1163,8 @@ fn test_query_matches_with_immediate_siblings() {
         //    siblings before that child node.
         // 2. After the last child node in a pattern, it means that there cannot be any named
         //    sibling after that child node.
-        // 2. Between two child nodes in a pattern, it specifies that there cannot be any named
-        //    siblings between those two child snodes.
+        // 3. Between two child nodes in a pattern, it specifies that there cannot be any named
+        //    siblings between those two child nodes.
         let query = Query::new(
             &language,
             "
@@ -1177,6 +1232,209 @@ fn test_query_matches_with_immediate_siblings() {
     });
 }
 
+#[test]
+fn test_query_matches_with_anchor_after_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            "(program (comment)* @doc . (function_declaration name: (identifier) @name))",
+        )
+        .unwrap();
+
+        // No comments and the function is not the first child. An anchor after a
+        // zero-matched quantifier is vacuous, so the function still matches.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+class X {}
+function foo() {}
+",
+            &[(0, vec![("name", "foo")])],
+        );
+
+        // With at least one comment the anchor applies, so the comments must
+        // immediately precede the function.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+// c
+function foo() {}
+",
+            &[(0, vec![("doc", "// c"), ("name", "foo")])],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_anchor_after_nested_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            r#"
+            (_
+              (field_definition
+                property: (_) @name
+                value: (_)? @value
+              ) @field
+              .
+              ";" @semicolon
+            )
+            "#,
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "class Foo { bar; baz = 0; }",
+            &[
+                (
+                    0,
+                    vec![("field", "bar"), ("name", "bar"), ("semicolon", ";")],
+                ),
+                (
+                    0,
+                    vec![
+                        ("field", "baz = 0"),
+                        ("name", "baz"),
+                        ("value", "0"),
+                        ("semicolon", ";"),
+                    ],
+                ),
+            ],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_last_child_anchor_after_optional() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(preproc_if (preproc_def)+ @def . (preproc_else)? @else .)",
+        )
+        .unwrap();
+
+        // The optional `(preproc_else)?` is absent, so the trailing anchor's
+        // last-child requirement transfers to the last `preproc_def`. A trailing
+        // comment means the def is not the last child, so nothing matches.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+#if X
+#define A
+// c
+#endif
+",
+            &[],
+        );
+
+        // With the def as the last child, the (else-less) match is allowed.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+#if X
+#define A
+#endif
+",
+            &[(0, vec![("def", "#define A\n")])],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_anchors_on_both_sides_of_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            "(program (lexical_declaration) @a . (comment)* . (function_declaration) @b)",
+        )
+        .unwrap();
+
+        // Anchors on both sides of a zero-matched quantifier collapse into a single
+        // adjacency constraint: with no comments, the declaration must be immediately
+        // followed by the function.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+const a = 1;
+const b = 2;
+function foo() {}
+",
+            &[(0, vec![("a", "const b = 2;"), ("b", "function foo() {}")])],
+        );
+
+        // With a comment present the quantifier is non-zero, so the anchors apply
+        // normally: the comment must sit immediately between the declaration and the
+        // function.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+const b = 2;
+// c
+function foo() {}
+",
+            &[(0, vec![("a", "const b = 2;"), ("b", "function foo() {}")])],
+        );
+    });
+}
+
+#[test]
+fn test_query_matches_with_leading_anchor_before_zero_quantifier() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(translation_unit . (comment)* (function_definition) @f)",
+        )
+        .unwrap();
+
+        // The leading `.` anchors the comment run to the parent's first child. When the
+        // run matches zero comments, that first-child requirement transfers to the
+        // function, so it matches only when it is itself the first child.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+int main() {}
+",
+            &[(0, vec![("f", "int main() {}")])],
+        );
+
+        // The function is the second child, so with no leading comments it must not match.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+int a;
+int main() {}
+",
+            &[],
+        );
+
+        // With a leading comment the run starts at the first child and the function follows.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+// c
+int main() {}
+",
+            &[(0, vec![("f", "int main() {}")])],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_last_named_child() {
     allocations::record(|| {
@@ -1349,6 +1607,35 @@ fn test_query_matches_with_repeated_leaf_nodes() {
     });
 }
 
+#[test]
+fn test_query_matches_optional_capture_before_uncaptured_required_sibling() {
+    allocations::record(|| {
+        let language = get_language("rust");
+        let query = Query::new(&language, "(block (line_comment)? @doc (line_comment))").unwrap();
+
+        // The optional `(line_comment)? @doc` before an *uncaptured* required
+        // `(line_comment)` yields two candidate completions at the block:
+        //     - one where the optional captured `// a` (the required node is then `// b`),
+        //     - one where the optional matched zero (the required node absorbs `// a`,
+        //     leaving `@doc` unbound).
+        // The zero-match completion's captures are a strict subset of the other's, so the
+        // longest-match rule must drop it (there is exactly one match). This regressed when
+        // the dedup pass gained an early-break. Without the accompanying capture-position
+        // sort, the break skips the subset "loser" and it leaks as an extra empty match.
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            fn f() {
+                // a
+                // b
+            }
+            ",
+            &[(0, vec![("doc", "// a")])],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_optional_nodes_inside_of_repetitions() {
     allocations::record(|| {
@@ -1593,6 +1880,148 @@ fn test_query_matches_with_leading_zero_or_more_repeated_leaf_nodes() {
     });
 }
 
+#[test]
+fn test_matches_with_anchor_sibling_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[(0, vec![("name", "a")]), (0, vec![("name", "b")])],
+        );
+    });
+}
+
+#[test]
+fn test_matches_with_anchor_sibling_with_quantifier_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)+
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[(0, vec![("name", "a")]), (0, vec![("name", "b")])],
+        );
+    });
+}
+
+#[test]
+fn test_matches_with_anchor_sibling_with_quantifier_captured_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("rust");
+
+        let query = Query::new(
+            &language,
+            "
+            (source_file
+                (line_comment)+ @doc
+                .
+                (function_item
+                    name: (identifier) @name)
+            )",
+        )
+        .unwrap();
+
+        assert_query_matches(
+            &language,
+            &query,
+            "
+            // A
+            fn a() {}
+
+            // B
+            fn b() {}
+            ",
+            &[
+                (0, vec![("doc", "// A"), ("name", "a")]),
+                (0, vec![("doc", "// B"), ("name", "b")]),
+            ],
+        );
+    });
+}
+
+#[test]
+fn test_matches_anchored_quantified_sibling_inside_parent() {
+    allocations::record(|| {
+        let language = get_language("c");
+        let query = Query::new(
+            &language,
+            "(translation_unit (comment)* @comment . (declaration) @decl)",
+        )
+        .unwrap();
+        assert_query_matches(
+            &language,
+            &query,
+            "
+void foo() {}
+
+// this one has
+// two comments
+extern int baz;
+
+// this one has a comment
+extern int bar;
+",
+            &[
+                (
+                    0,
+                    vec![
+                        ("comment", "// this one has"),
+                        ("comment", "// two comments"),
+                        ("decl", "extern int baz;"),
+                    ],
+                ),
+                (
+                    0,
+                    vec![
+                        ("comment", "// this one has a comment"),
+                        ("decl", "extern int bar;"),
+                    ],
+                ),
+            ],
+        );
+    });
+}
+
 #[test]
 fn test_query_matches_with_trailing_optional_nodes() {
     allocations::record(|| {
@@ -1941,7 +2370,7 @@ fn test_query_matches_with_too_many_permutations_to_track() {
         let matches = cursor.matches(&query, tree.root_node(), source.as_bytes());
 
         // For this pathological query, some match permutations will be dropped.
-        // Just check that a subset of the results are returned, and crash or
+        // Just check that a subset of the results are returned, and no crash or
         // leak occurs.
         assert_eq!(
             collect_matches(matches, &query, source.as_str())[0],
@@ -2489,7 +2918,7 @@ fn test_query_matches_with_wildcard_at_root_intersecting_byte_range() {
         );
 
         while let Some(mat) = match_iter.next() {
-            if let Some(capture) = mat.captures.first() {
+            if let Some(capture) = mat.captures().first() {
                 matches.push(capture.node.kind());
             }
         }
@@ -2505,7 +2934,7 @@ fn test_query_matches_with_wildcard_at_root_intersecting_byte_range() {
         );
 
         while let Some(mat) = match_iter.next() {
-            if let Some(capture) = mat.captures.first() {
+            if let Some(capture) = mat.captures().first() {
                 matches.push(capture.node.kind());
             }
         }
@@ -2521,7 +2950,7 @@ fn test_query_matches_with_wildcard_at_root_intersecting_byte_range() {
         );
 
         while let Some(mat) = match_iter.next() {
-            if let Some(capture) = mat.captures.first() {
+            if let Some(capture) = mat.captures().first() {
                 matches.push(capture.node.kind());
             }
         }
@@ -2579,7 +3008,7 @@ fn test_query_captures_within_byte_range_assigned_after_iterating() {
         let mut results = Vec::new();
         let mut first_five = captures.by_ref().take(5);
         while let Some((mat, capture_ix)) = first_five.next() {
-            let capture = mat.captures[*capture_ix];
+            let capture = mat.captures()[*capture_ix];
             results.push((
                 query.capture_names()[capture.index as usize],
                 &source[capture.node.byte_range()],
@@ -2602,7 +3031,7 @@ fn test_query_captures_within_byte_range_assigned_after_iterating() {
         results.clear();
         captures.set_byte_range(source.find("Ok").unwrap()..source.len());
         while let Some((mat, capture_ix)) = captures.next() {
-            let capture = mat.captures[*capture_ix];
+            let capture = mat.captures()[*capture_ix];
             results.push((
                 query.capture_names()[capture.index as usize],
                 &source[capture.node.byte_range()],
@@ -2901,7 +3330,7 @@ fn test_query_matches_with_captured_wildcard_at_root() {
 
         while let Some(m) = match_iter.next() {
             let captures = m
-                .captures
+                .captures()
                 .iter()
                 .map(|c| {
                     (
@@ -3111,6 +3540,39 @@ fn test_query_alternation_with_inner_quantifier() {
     assert_query_matches(&language, &query, source_code, matches);
 }
 
+#[test]
+fn test_query_alternation_with_outer_quantifier() {
+    let language = get_language("c");
+    let source_code = "#include <foo>
+#include <bar>
+#include <baz>
+
+// comment";
+    let matches = &[(
+        0,
+        vec![
+            ("capture", "#include <foo>\n"),
+            ("capture", "#include <bar>\n"),
+            ("capture", "#include <baz>\n"),
+            ("capture", "// comment"),
+        ],
+    )];
+
+    let query = "[
+        (preproc_include)
+        (comment)
+    ]+ @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+
+    let query = "([
+        (preproc_include)
+        (comment)
+    ] (_)?)+ @capture";
+    let query = Query::new(&language, query).unwrap();
+    assert_query_matches(&language, &query, source_code, matches);
+}
+
 #[test]
 fn test_query_matches_with_alternations_and_predicates() {
     allocations::record(|| {
@@ -3849,7 +4311,7 @@ fn test_query_captures_with_matches_removed() {
 
         let mut captures = cursor.captures(&query, tree.root_node(), source.as_bytes());
         while let Some((m, i)) = captures.next() {
-            let capture = m.captures[*i];
+            let capture = m.captures()[*i];
             let text = capture.node.utf8_text(source.as_bytes()).unwrap();
             if text == "a" {
                 m.remove();
@@ -3894,7 +4356,7 @@ fn test_query_captures_with_matches_removed_before_they_finish() {
         let mut captured_strings = Vec::new();
         let mut captures = cursor.captures(&query, tree.root_node(), source.as_bytes());
         while let Some((m, i)) = captures.next() {
-            let capture = m.captures[*i];
+            let capture = m.captures()[*i];
             let text = capture.node.utf8_text(source.as_bytes()).unwrap();
             if text == "as" {
                 m.remove();
@@ -3935,18 +4397,18 @@ fn test_query_captures_and_matches_iterators_are_fused() {
         let mut cursor = QueryCursor::new();
         let mut captures = cursor.captures(&query, tree.root_node(), source.as_bytes());
 
-        assert_eq!(captures.next().unwrap().0.captures[0].index, 0);
-        assert_eq!(captures.next().unwrap().0.captures[0].index, 0);
-        assert_eq!(captures.next().unwrap().0.captures[0].index, 0);
+        assert_eq!(captures.next().unwrap().0.captures()[0].index, 0);
+        assert_eq!(captures.next().unwrap().0.captures()[0].index, 0);
+        assert_eq!(captures.next().unwrap().0.captures()[0].index, 0);
         assert!(captures.next().is_none());
         assert!(captures.next().is_none());
         assert!(captures.next().is_none());
         drop(captures);
 
         let mut matches = cursor.matches(&query, tree.root_node(), source.as_bytes());
-        assert_eq!(matches.next().unwrap().captures[0].index, 0);
-        assert_eq!(matches.next().unwrap().captures[0].index, 0);
-        assert_eq!(matches.next().unwrap().captures[0].index, 0);
+        assert_eq!(matches.next().unwrap().captures()[0].index, 0);
+        assert_eq!(matches.next().unwrap().captures()[0].index, 0);
+        assert_eq!(matches.next().unwrap().captures()[0].index, 0);
         assert!(matches.next().is_none());
         assert!(matches.next().is_none());
         assert!(matches.next().is_none());
@@ -4126,7 +4588,7 @@ fn test_query_lifetime_is_separate_from_nodes_lifetime() {
                 .matches(&query, node, source.as_bytes())
                 .next()
                 .unwrap()
-                .captures[0]
+                .captures()[0]
                 .node
         }
 
@@ -4147,7 +4609,7 @@ fn test_query_lifetime_is_separate_from_nodes_lifetime() {
                 .next()
                 .unwrap()
                 .0
-                .captures[0]
+                .captures()[0]
                 .node
         }
 
@@ -4161,7 +4623,7 @@ fn test_query_with_no_patterns() {
     allocations::record(|| {
         let language = get_language("javascript");
         let query = Query::new(&language, "").unwrap();
-        assert!(query.capture_names().is_empty());
+        assert_eq!(query.capture_names(), [] as [&str; 0]);
         assert_eq!(query.pattern_count(), 0);
     });
 }
@@ -4234,6 +4696,54 @@ fn test_query_disable_pattern() {
     });
 }
 
+#[test]
+fn test_query_deep_clone() {
+    allocations::record(|| {
+        let language = get_language("javascript");
+        let query = Query::new(
+            &language,
+            "
+                (function_declaration
+                    name: (identifier) @name)
+                (function_declaration
+                    body: (statement_block) @body)
+            ",
+        )
+        .unwrap();
+
+        let mut clone = query.deep_clone();
+        clone.disable_pattern(1);
+
+        let source = "function foo() { return 1; }";
+        let mut parser = Parser::new();
+        parser.set_language(&language).unwrap();
+        let tree = parser.parse(source, None).unwrap();
+        let mut cursor = QueryCursor::new();
+
+        // The clone with pattern 1 disabled only produces the @name match.
+        let clone_matches = collect_matches(
+            cursor.matches(&clone, tree.root_node(), source.as_bytes()),
+            &clone,
+            source,
+        );
+        assert_eq!(clone_matches, &[(0, vec![("name", "foo")])]);
+
+        // The original is unaffected and still produces both @name and @body.
+        let original_matches = collect_matches(
+            cursor.matches(&query, tree.root_node(), source.as_bytes()),
+            &query,
+            source,
+        );
+        assert_eq!(
+            original_matches,
+            &[
+                (0, vec![("name", "foo")]),
+                (1, vec![("body", "{ return 1; }")]),
+            ]
+        );
+    });
+}
+
 #[test]
 fn test_query_alternative_predicate_prefix() {
     allocations::record(|| {
@@ -4309,7 +4819,7 @@ fn test_query_random() {
                 let transformed_match = Match {
                     last_node: None,
                     captures: mat
-                        .captures
+                        .captures()
                         .iter()
                         .map(|c| (query.capture_names()[c.index as usize], c.node))
                         .collect::<Vec<_>>(),
@@ -5323,10 +5833,10 @@ fn test_consecutive_zero_or_modifiers() {
         let mut len_1 = false;
 
         while let Some(m) = matches.next() {
-            if m.captures.len() == 3 {
+            if m.captures().len() == 3 {
                 len_3 = true;
             }
-            if m.captures.len() == 1 {
+            if m.captures().len() == 1 {
                 len_1 = true;
             }
         }
@@ -5388,7 +5898,7 @@ fn test_query_max_start_depth_more() {
         let query = Query::new(&language, "(compound_statement) @capture").unwrap();
 
         let mut matches = cursor.matches(&query, tree.root_node(), source.as_bytes());
-        let node = matches.next().unwrap().captures[0].node;
+        let node = matches.next().unwrap().captures()[0].node;
         assert_eq!(node.kind(), "compound_statement");
 
         for row in rows {
@@ -5564,21 +6074,21 @@ fn test_query_execution_with_timeout() {
     let tree = parser.parse(&source_code, None).unwrap();
 
     let query = Query::new(&language, "(function_declaration) @function").unwrap();
-    let mut cursor = QueryCursor::new();
-
     let start_time = std::time::Instant::now();
+    let mut progress_callback = |_: &QueryCursorState| {
+        if start_time.elapsed().as_micros() > 1000 {
+            ControlFlow::Break(())
+        } else {
+            ControlFlow::Continue(())
+        }
+    };
+    let mut cursor = QueryCursor::new();
     let matches = cursor
         .matches_with_options(
             &query,
             tree.root_node(),
             source_code.as_bytes(),
-            QueryCursorOptions::new().progress_callback(&mut |_| {
-                if start_time.elapsed().as_micros() > 1000 {
-                    ControlFlow::Break(())
-                } else {
-                    ControlFlow::Continue(())
-                }
-            }),
+            QueryCursorOptions::new().progress_callback(&mut progress_callback),
         )
         .count();
     assert!(matches < 1000);
@@ -5589,6 +6099,33 @@ fn test_query_execution_with_timeout() {
     assert_eq!(matches, 1000);
 }
 
+#[test]
+fn test_query_progress_callback_lives_as_long_as_matches() {
+    let language = get_language("javascript");
+    let mut parser = Parser::new();
+    parser.set_language(&language).unwrap();
+
+    let source_code = "function foo() {}\n".repeat(1000);
+    let tree = parser.parse(&source_code, None).unwrap();
+    let query = Query::new(&language, "(function_declaration) @function").unwrap();
+
+    let mut cursor = QueryCursor::new();
+    let mut callback_was_called = false;
+    let mut progress_callback = |_: &QueryCursorState| {
+        callback_was_called = true;
+        ControlFlow::Continue(())
+    };
+    let matches = cursor.matches_with_options(
+        &query,
+        tree.root_node(),
+        source_code.as_bytes(),
+        QueryCursorOptions::new().progress_callback(&mut progress_callback),
+    );
+
+    assert_eq!(matches.count(), 1000);
+    assert!(callback_was_called);
+}
+
 #[test]
 fn test_query_execution_with_points_causing_underflow() {
     let language = get_language("rust");
@@ -6000,3 +6537,64 @@ export default grammar({
 
     assert_query_matches(&language, &query, source, &[(0, vec![("tuple", "()")])]);
 }
+
+#[test]
+fn test_last_child_anchor_looks_past_hidden_repeat() {
+    let language = get_test_fixture_language("last_child_anchor_past_hidden_repeat");
+
+    let source = "T a.b.c\nL a.b.c\nN a!.b!.c\n";
+
+    let query = Query::new(
+        &language,
+        "
+        (trailing_sep (name) @last .)
+        (leading_sep (name) @last .)
+        (trailing_named (name) @last .)
+        ",
+    )
+    .unwrap();
+
+    assert_query_matches(
+        &language,
+        &query,
+        source,
+        &[
+            (0, vec![("last", "c")]),
+            (1, vec![("last", "c")]),
+            (2, vec![("last", "c")]),
+        ],
+    );
+
+    let query = Query::new(
+        &language,
+        "
+        (trailing_sep . (name) @first)
+        (trailing_sep (name) @a . (name) @b)
+        ",
+    )
+    .unwrap();
+
+    assert_query_matches(
+        &language,
+        &query,
+        source,
+        &[
+            (0, vec![("first", "a")]),
+            (1, vec![("a", "a"), ("b", "b")]),
+            (1, vec![("a", "b"), ("b", "c")]),
+        ],
+    );
+}
+
+#[test]
+fn test_last_child_anchor_looks_past_hidden_node() {
+    allocations::record(|| {
+        let language = get_language("c");
+
+        let query = Query::new(&language, "(translation_unit (_) @last .)").unwrap();
+
+        let source = "enum E { A };\nint x;\nint y;\n";
+
+        assert_query_matches(&language, &query, source, &[(0, vec![("last", "int y;")])]);
+    });
+}

crates/cli/src/tests/tags_test.rs

@@ -1,6 +1,6 @@
 use std::{
     ffi::{CStr, CString},
-    fs, ptr, slice, str,
+    fs, ptr, slice,
     sync::atomic::{AtomicUsize, Ordering},
 };
 

crates/cli/src/tests/test_highlight_test.rs

@@ -4,7 +4,7 @@ use tree_sitter_highlight::{Highlight, Highlighter};
 use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
 use crate::{
     query_testing::{Assertion, Utf8Point, parse_position_comments},
-    test_highlight::get_highlight_positions,
+    test_highlight::{Failure, get_highlight_positions, iterate_assertions},
 };
 
 #[test]
@@ -68,3 +68,195 @@ fn test_highlight_test_with_basic_test() {
         ]
     );
 }
+
+#[test]
+fn test_assertion_with_non_matching_highlight_at_same_position() {
+    // Test that an assertion fails when the highlight at the position does not match
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![Assertion::new(1, 0, 1, false, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(1, 0), Utf8Point::new(1, 5), Highlight(1)), // "variable" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 1);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, vec!["variable".to_string()]);
+}
+
+#[test]
+fn test_assertion_with_exact_matching_highlight() {
+    // Test exact match: assertion and highlight have same start and end
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 5, 3, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_contained_within_highlight() {
+    // Test where assertion is fully contained within a larger highlight
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 3, 2, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 0), Utf8Point::new(0, 10), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_overlapping_highlight_start() {
+    // Test where assertion starts before highlight but overlaps with it
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 3, 4, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 5), Utf8Point::new(0, 10), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertion_with_no_highlights() {
+    // Test that an assertion fails when there are no highlights at all
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, false, String::from("keyword"))];
+    let highlights = vec![];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}
+
+#[test]
+fn test_assertion_with_highlight_ending_before() {
+    // Test where highlight ends before the assertion starts
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 10, 1, false, String::from("keyword"))];
+    let highlights = vec![(Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(0))];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 10);
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}
+
+#[test]
+fn test_negative_assertion_with_non_matching_highlight() {
+    // Test that a negative assertion passes when the specified highlight is NOT present
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, true, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(1)), // "variable" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_negative_assertion_with_matching_highlight() {
+    // Test that a negative assertion fails when the specified highlight IS present
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 0, 1, true, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 5), Highlight(0)), // "keyword" highlight
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 0);
+    assert_eq!(err.expected_highlight, "!keyword");
+    assert_eq!(err.actual_highlights, vec!["keyword".to_string()]);
+}
+
+#[test]
+fn test_multiple_assertions_sequential() {
+    // Test multiple assertions in sequence with non-overlapping highlights
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![
+        Assertion::new(0, 0, 3, false, String::from("keyword")),
+        Assertion::new(0, 10, 1, false, String::from("variable")),
+    ];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 3), Highlight(0)), // "keyword"
+        (Utf8Point::new(0, 10), Utf8Point::new(0, 11), Highlight(1)), // "variable"
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 2);
+}
+
+#[test]
+fn test_multiple_highlights_at_same_position() {
+    // Test where multiple highlights overlap at the assertion position
+    let highlight_names = vec![
+        "keyword".to_string(),
+        "variable".to_string(),
+        "function".to_string(),
+    ];
+    let assertions = vec![Assertion::new(0, 5, 1, false, String::from("variable"))];
+    let highlights = vec![
+        (Utf8Point::new(0, 0), Utf8Point::new(0, 10), Highlight(0)), // "keyword" spans entire range
+        (Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(1)), // "variable" at assertion position
+        (Utf8Point::new(0, 7), Utf8Point::new(0, 12), Highlight(2)), // "function" overlaps
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 1);
+}
+
+#[test]
+fn test_assertions_across_multiple_rows() {
+    // Test assertions on different rows
+    let highlight_names = vec!["keyword".to_string(), "variable".to_string()];
+    let assertions = vec![
+        Assertion::new(0, 5, 3, false, String::from("keyword")),
+        Assertion::new(2, 10, 1, false, String::from("variable")),
+    ];
+    let highlights = vec![
+        (Utf8Point::new(0, 5), Utf8Point::new(0, 8), Highlight(0)), // "keyword" on row 0
+        (Utf8Point::new(2, 10), Utf8Point::new(2, 11), Highlight(1)), // "variable" on row 2
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 2);
+}
+
+#[test]
+fn test_assertion_should_not_match_highlight_on_later_row() {
+    // Test logic for early exit when highlight is on a later row than the assertion
+    let highlight_names = vec!["keyword".to_string()];
+    let assertions = vec![Assertion::new(0, 5, 3, false, String::from("keyword"))];
+    let highlights = vec![
+        (Utf8Point::new(1, 0), Utf8Point::new(1, 5), Highlight(0)), // wrong row
+    ];
+    let result = iterate_assertions(&assertions, &highlights, &highlight_names);
+
+    assert!(result.is_err());
+    let err = result.unwrap_err().downcast::<Failure>().unwrap();
+    assert_eq!(err.row, 0);
+    assert_eq!(err.column, 7); // end_column
+    assert_eq!(err.expected_highlight, "keyword");
+    assert_eq!(err.actual_highlights, Vec::<String>::new());
+}

crates/cli/src/tests/text_provider_test.rs

@@ -30,7 +30,7 @@ fn tree_query<I: AsRef<[u8]>>(tree: &Tree, text: impl TextProvider<I>, language:
     let mut cursor = QueryCursor::new();
     let mut captures = cursor.captures(&query, tree.root_node(), text);
     let (match_, idx) = captures.next().unwrap();
-    let capture = match_.captures[*idx];
+    let capture = match_.captures()[*idx];
     assert_eq!(capture.index as usize, *idx);
     assert_eq!("comment", capture.node.kind());
 }

crates/cli/src/tests/tree_test.rs

@@ -1,5 +1,3 @@
-use std::str;
-
 use tree_sitter::{InputEdit, Parser, Point, Range, Tree};
 
 use super::helpers::fixtures::get_language;
@@ -795,3 +793,44 @@ fn get_changed_ranges(
     *tree = new_tree;
     result
 }
+
+// Regression test for an incremental reparse bug where an external
+// scanner's choice depends on lexer->eof() (and thus on the parser's
+// current included ranges). The cached token at byte 0 was emitted when
+// the included range stopped just after the opener. Widening the range
+// to include a later matching delimiter must invalidate that cached
+// token so the scanner re-runs and emits the open form instead of the
+// unclosed form.
+#[test]
+fn test_reuse_invalidates_scanner_token_when_included_range_expands() {
+    let language = get_test_fixture_language("external_lookahead_eof_boundary");
+    let mut parser = Parser::new();
+    parser.set_language(&language).unwrap();
+
+    let source = "``";
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 1,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 1),
+        }])
+        .unwrap();
+    let tree1 = parser.parse(source, None).unwrap();
+    assert_eq!(tree1.root_node().to_sexp(), "(document (unclosed_delim))");
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 2,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 2),
+        }])
+        .unwrap();
+    let tree2 = parser.parse(source, Some(&tree1)).unwrap();
+    assert_eq!(
+        tree2.root_node().to_sexp(),
+        "(document (span (open_delim) (close_delim)))"
+    );
+}

crates/cli/src/tests/wasm_language_test.rs

@@ -314,6 +314,55 @@ fn test_load_wasm_errors() {
     });
 }
 
+#[test]
+fn test_load_wasm_language_with_reserved_words() {
+    // This test exercises a grammar with multiple reserved word sets loaded via WASM.
+    allocations::record(|| {
+        let store = WasmStore::new(&ENGINE).unwrap();
+        let language = get_test_fixture_language_wasm("reserved_words");
+
+        let mut parser = Parser::new();
+        parser.set_wasm_store(store).unwrap();
+        parser.set_language(&language).unwrap();
+
+        // "if" and "while" are globally reserved, so using them as identifiers
+        // should produce an error recovery.
+        let tree = parser
+            .parse("var a =\n\nif (something) {\n  c();\n}", None)
+            .unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            concat!(
+                "(program ",
+                "(ERROR (identifier)) ",
+                "(if_statement (parenthesized_expression (identifier)) ",
+                "(block (expression_statement (call_expression (identifier))))))",
+            )
+        );
+
+        // "if" and "while" are NOT reserved in the 'property' context, so they
+        // can appear as object keys without error.
+        let tree = parser
+            .parse("var x = {\n  if: a,\n  while: b,\n};", None)
+            .unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            concat!(
+                "(program (var_declaration (identifier) (object ",
+                "(pair (identifier) (identifier)) (pair (identifier) (identifier)))))"
+            )
+        );
+
+        // "var" IS reserved in the 'property' context, so using it as a property
+        // key triggers error recovery.
+        let tree = parser.parse("var x = {\nvar y = z;", None).unwrap();
+        assert_eq!(
+            tree.root_node().to_sexp(),
+            "(program (ERROR (identifier)) (var_declaration (identifier) (identifier)))"
+        );
+    });
+}
+
 #[test]
 fn test_wasm_oom() {
     allocations::record(|| {
@@ -338,3 +387,18 @@ fn test_wasm_oom() {
         );
     });
 }
+
+#[test]
+fn test_lookahead_iterator_outlives_wasm_language() {
+    allocations::record(|| {
+        let mut store = WasmStore::new(&ENGINE).unwrap();
+        let wasm = fs::read(WASM_DIR.join("tree-sitter-ruby.wasm")).unwrap();
+        let language = store.load_language("ruby", &wasm).unwrap();
+
+        let mut lookahead = language.lookahead_iterator(0).unwrap();
+        drop(language);
+
+        // The iterator retains the language, so the names are still live.
+        assert!(lookahead.iter_names().count() > 0);
+    });
+}

crates/cli/src/tree_sitter_cli.rs

@@ -5,6 +5,7 @@ pub mod highlight;
 pub mod init;
 pub mod input;
 pub mod logger;
+pub mod paint;
 pub mod parse;
 pub mod playground;
 pub mod query;

crates/cli/src/util.rs

@@ -71,8 +71,8 @@ pub struct LogSession {
     open_log: bool,
 }
 
-pub fn print_tree_graph(tree: &Tree, path: &str, quiet: bool) -> Result<()> {
-    let session = LogSession::new(path, quiet)?;
+pub fn print_tree_graph(tree: &Tree, path: &str, open_log: bool) -> Result<()> {
+    let session = LogSession::new(path, open_log)?;
     tree.print_dot_graph(session.dot_process_stdin.as_ref().unwrap());
     Ok(())
 }

crates/cli/src/wasm.rs

@@ -15,7 +15,7 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)>
         .unwrap();
     let wasm_filename = format!("tree-sitter-{grammar_name}.wasm");
     let contents = fs::read(language_dir.join(&wasm_filename)).with_context(|| {
-        format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.",)
+        format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.")
     })?;
     Ok((grammar_name, contents))
 }

crates/config/Cargo.toml

@@ -1,17 +1,16 @@
 [package]
-name                   = "tree-sitter-config"
-version.workspace      = true
-description            = "User configuration of tree-sitter's command line programs"
-authors.workspace      = true
-edition.workspace      = true
+name = "tree-sitter-config"
+version.workspace = true
+description = "User configuration of tree-sitter's command line programs"
+authors.workspace = true
+edition.workspace = true
 rust-version.workspace = true
-readme                 = "README.md"
-homepage.workspace     = true
-repository.workspace   = true
-documentation          = "https://docs.rs/tree-sitter-config"
-license.workspace      = true
-keywords.workspace     = true
-categories.workspace   = true
+homepage.workspace = true
+repository.workspace = true
+documentation = "https://docs.rs/tree-sitter-config"
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
 
 [lib]
 path = "src/tree_sitter_config.rs"
@@ -20,8 +19,8 @@ path = "src/tree_sitter_config.rs"
 workspace = true
 
 [dependencies]
-etcetera.workspace   = true
-log.workspace        = true
-serde.workspace      = true
+etcetera.workspace = true
+log.workspace = true
+serde.workspace = true
 serde_json.workspace = true
-thiserror.workspace  = true
+thiserror.workspace = true

crates/config/src/tree_sitter_config.rs

@@ -28,14 +28,24 @@ pub enum ConfigError {
 #[derive(Debug, Error)]
 pub struct IoError {
     pub error: std::io::Error,
-    pub path: Option<String>,
+    pub path: Option<PathBuf>,
 }
 
+impl PartialEq for IoError {
+    fn eq(&self, other: &Self) -> bool {
+        self.path == other.path
+            && self.error.kind() == other.error.kind()
+            && self.error.raw_os_error() == other.error.raw_os_error()
+    }
+}
+
+impl Eq for IoError {}
+
 impl IoError {
     fn new(error: std::io::Error, path: Option<&Path>) -> Self {
         Self {
             error,
-            path: path.map(|p| p.to_string_lossy().to_string()),
+            path: path.map(Path::to_path_buf),
         }
     }
 }
@@ -44,7 +54,7 @@ impl std::fmt::Display for IoError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.error)?;
         if let Some(ref path) = self.path {
-            write!(f, " ({path})")?;
+            write!(f, " ({})", path.display())?;
         }
         Ok(())
     }

crates/generate/Cargo.toml

@@ -1,17 +1,16 @@
 [package]
-name                   = "tree-sitter-generate"
-version.workspace      = true
-description            = "Library for generating C source code from a tree-sitter grammar"
-authors.workspace      = true
-edition.workspace      = true
+name = "tree-sitter-generate"
+version.workspace = true
+description = "Library for generating C source code from a tree-sitter grammar"
+authors.workspace = true
+edition.workspace = true
 rust-version.workspace = true
-readme                 = "README.md"
-homepage.workspace     = true
-repository.workspace   = true
-documentation          = "https://docs.rs/tree-sitter-generate"
-license.workspace      = true
-keywords.workspace     = true
-categories.workspace   = true
+homepage.workspace = true
+repository.workspace = true
+documentation = "https://docs.rs/tree-sitter-generate"
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
 
 [lib]
 path = "src/generate.rs"
@@ -21,24 +20,19 @@ workspace = true
 
 [features]
 default = [ "qjs-rt" ]
-load    = [ "dep:semver" ]
-qjs-rt  = [ "load", "rquickjs", "pathdiff" ]
+load = [ "dep:semver" ]
+qjs-rt = [ "load", "rquickjs", "pathdiff" ]
 
 [dependencies]
-bitflags = "2.10.0"
+bitflags = "2.11.1"
 dunce = "1.0.5"
+hashbrown.workspace = true
 indexmap.workspace = true
-indoc.workspace = true
 log.workspace = true
 pathdiff = { optional = true, version = "0.2.3" }
 regex.workspace = true
 regex-syntax.workspace = true
-rquickjs = { features = [
-  "bindgen",
-  "loader",
-  "macro",
-  "phf",
-], optional = true, version = "0.11.0" }
+rquickjs = { features = [ "bindgen", "loader", "macro", "phf" ], optional = true, version = "0.13" }
 rustc-hash.workspace = true
 semver = { optional = true, workspace = true }
 serde.workspace = true

crates/generate/src/bitvec.rs

@@ -93,6 +93,7 @@ pub struct BitVec {
 }
 
 impl BitVec {
+    #[must_use]
     pub const fn new() -> Self {
         Self {
             data: ptr::NonNull::dangling().as_ptr(),
@@ -101,6 +102,7 @@ impl BitVec {
         }
     }
 
+    #[must_use]
     pub fn with_capacity(n_bits: usize) -> Self {
         let n_words = n_bits.div_ceil(64);
         if n_words == 0 {
@@ -120,6 +122,7 @@ impl BitVec {
 
     /// View the in-use words as a slice.
     #[inline]
+    #[must_use]
     pub const fn as_slice(&self) -> &[u64] {
         let n = self.words_in_use();
         if n == 0 {
@@ -142,18 +145,21 @@ impl BitVec {
         unsafe { std::slice::from_raw_parts_mut(self.data, n) }
     }
 
+    #[must_use]
+    #[allow(clippy::len_without_is_empty)]
     pub const fn len(&self) -> usize {
         self.num_bits as usize
     }
 
-    pub fn get(&self, index: usize) -> Option<bool> {
+    #[must_use]
+    pub const fn get(&self, index: usize) -> Option<bool> {
         if index >= self.num_bits as usize {
             return None;
         }
         Some(self.as_slice()[index / 64] >> (index % 64) & 1 != 0)
     }
 
-    pub fn set(&mut self, index: usize, val: bool) {
+    pub const fn set(&mut self, index: usize, val: bool) {
         let word_idx = index / 64;
         let bit_idx = index % 64;
         let words = self.as_full_slice_mut();
@@ -206,14 +212,15 @@ impl BitVec {
         self.num_bits = new_len as u32;
     }
 
-    pub fn last(&self) -> Option<bool> {
+    #[must_use]
+    pub const fn last(&self) -> Option<bool> {
         if self.num_bits == 0 {
             return None;
         }
         self.get(self.num_bits as usize - 1)
     }
 
-    pub fn pop(&mut self) -> Option<bool> {
+    pub const fn pop(&mut self) -> Option<bool> {
         if self.num_bits == 0 {
             return None;
         }
@@ -234,6 +241,7 @@ impl BitVec {
     }
 
     /// Word-level OR: self |= other. Returns true if any new bits were set.
+    #[inline]
     pub fn insert_all(&mut self, other: &Self) -> bool {
         let other_words = other.words_in_use();
         if other_words == 0 {
@@ -379,6 +387,7 @@ pub struct SetBitsIter<'a> {
 }
 
 impl<'a> SetBitsIter<'a> {
+    #[must_use]
     pub fn new(data: &'a [u64]) -> Self {
         Self {
             data,

crates/generate/src/build_tables.rs

@@ -6,28 +6,31 @@ mod item_set_builder;
 mod minimize_parse_table;
 mod token_conflicts;
 
-use std::collections::{BTreeSet, HashMap};
+use std::collections::BTreeSet;
 
 pub use build_lex_table::LARGE_CHARACTER_RANGE_COUNT;
 use build_parse_table::BuildTableResult;
 pub use build_parse_table::ParseTableBuilderError;
 use log::{debug, info};
+use rustc_hash::FxHashMap;
 
 use self::{
     build_lex_table::build_lex_table,
     build_parse_table::{ParseStateInfo, build_parse_table},
     coincident_tokens::CoincidentTokenIndex,
+    item::ItemKeyMap,
     item_set_builder::ParseItemSetBuilder,
     minimize_parse_table::minimize_parse_table,
     token_conflicts::TokenConflictMap,
 };
 use crate::{
-    OptLevel,
+    Diagnostic, OptLevel,
     grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor},
     node_types::VariableInfo,
     rules::{AliasMap, Symbol, SymbolType, TokenSet},
-    tables::{LexTable, ParseAction, ParseTable, ParseTableEntry},
+    strpool::StrPool,
+    tables::{ActionList, ActionListPool, LexTable, ParseAction, ParseTable, ParseTableEntry},
 };
 
 pub struct Tables {
@@ -37,32 +40,42 @@ pub struct Tables {
     pub large_character_sets: Vec<(Option<Symbol>, CharacterSet)>,
 }
 
+#[expect(
+    clippy::too_many_arguments,
+    reason = "all parameters are required for table building"
+)]
 pub fn build_tables(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     simple_aliases: &AliasMap,
     variable_info: &[VariableInfo],
     inlines: &InlinedProductionMap,
+    str_pool: &StrPool,
     report_symbol_name: Option<&str>,
     optimizations: OptLevel,
+    diagnostics: &mut Vec<Diagnostic>,
 ) -> BuildTableResult<Tables> {
-    let item_set_builder = ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines);
-    let following_tokens =
-        get_following_tokens(syntax_grammar, lexical_grammar, inlines, &item_set_builder);
+    let item_key_map = ItemKeyMap::new(syntax_grammar, str_pool);
+    let item_set_builder =
+        ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines, &item_key_map);
+    let following_tokens = get_following_tokens(syntax_grammar, lexical_grammar, &item_set_builder);
     let (mut parse_table, parse_state_info) = build_parse_table(
         syntax_grammar,
         lexical_grammar,
         item_set_builder,
         variable_info,
+        str_pool,
+        diagnostics,
     )?;
     let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
-    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
+    let coincident_token_index =
+        CoincidentTokenIndex::new(&parse_table, lexical_grammar, syntax_grammar.word_token);
     let keywords = identify_keywords(
         lexical_grammar,
-        &parse_table,
         syntax_grammar.word_token,
         &token_conflict_map,
         &coincident_token_index,
+        str_pool,
     );
     populate_error_state(
         &mut parse_table,
@@ -71,8 +84,10 @@ pub fn build_tables(
         &coincident_token_index,
         &token_conflict_map,
         &keywords,
+        str_pool,
     );
     populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
+    let mut parse_table = ActionListPool::intern_table(parse_table);
     minimize_parse_table(
         &mut parse_table,
         syntax_grammar,
@@ -80,6 +95,7 @@ pub fn build_tables(
         simple_aliases,
         &token_conflict_map,
         &keywords,
+        str_pool,
         optimizations,
     );
     let lex_tables = build_lex_table(
@@ -91,7 +107,10 @@ pub fn build_tables(
         &token_conflict_map,
     );
     populate_external_lex_states(&mut parse_table, syntax_grammar);
-    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
+    mark_fragile_tokens(&mut parse_table, &token_conflict_map);
+    parse_table
+        .action_lists
+        .canonicalize(&mut parse_table.states);
 
     if let Some(report_symbol_name) = report_symbol_name {
         report_state_info(
@@ -99,6 +118,7 @@ pub fn build_tables(
             lexical_grammar,
             &parse_table,
             &parse_state_info,
+            str_pool,
             report_symbol_name,
         );
     }
@@ -118,30 +138,25 @@ pub fn build_tables(
 fn get_following_tokens(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
-    inlines: &InlinedProductionMap,
     builder: &ParseItemSetBuilder,
 ) -> Vec<TokenSet> {
     let n_terminals = lexical_grammar.variables.len();
     let n_externals = syntax_grammar.external_tokens.len();
     let mut result = vec![TokenSet::with_capacity(n_terminals, n_externals); n_terminals];
-    let productions = syntax_grammar
-        .variables
-        .iter()
-        .flat_map(|v| &v.productions)
-        .chain(&inlines.productions);
     let all_tokens = (0..result.len())
         .map(Symbol::terminal)
         .collect::<TokenSet>();
-    for production in productions {
-        for i in 1..production.steps.len() {
-            let left_tokens = builder.last_set(&production.steps[i - 1].symbol);
-            let right_tokens = builder.first_set(&production.steps[i].symbol);
-            let right_reserved_tokens = builder.reserved_first_set(&production.steps[i].symbol);
+    for production in &syntax_grammar.productions {
+        let steps = &syntax_grammar.steps[production.step_range()];
+        for i in 1..steps.len() {
+            let left_tokens = builder.last_set(steps[i - 1].symbol());
+            let right_tokens = builder.first_set(steps[i].symbol());
+            let right_reserved_tokens = builder.reserved_first_set(steps[i].symbol());
             for left_token in left_tokens.iter() {
                 if left_token.is_terminal() {
-                    result[left_token.index].insert_all_terminals(right_tokens);
+                    result[left_token.index as usize].insert_all_terminals(right_tokens);
                     if let Some(reserved_tokens) = right_reserved_tokens {
-                        result[left_token.index].insert_all_terminals(reserved_tokens);
+                        result[left_token.index as usize].insert_all_terminals(reserved_tokens);
                     }
                 }
             }
@@ -152,19 +167,20 @@ fn get_following_tokens(
             for entry in &mut result {
                 entry.insert(*extra);
             }
-            result[extra.index] = all_tokens.clone();
+            result[extra.index as usize] = all_tokens.clone();
         }
     }
     result
 }
 
 fn populate_error_state(
-    parse_table: &mut ParseTable,
+    parse_table: &mut ParseTable<ParseTableEntry>,
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     coincident_token_index: &CoincidentTokenIndex,
     token_conflict_map: &TokenConflictMap,
     keywords: &TokenSet,
+    str_pool: &StrPool,
 ) {
     let state = &mut parse_table.states[0];
     let n = lexical_grammar.variables.len();
@@ -183,7 +199,7 @@ fn populate_error_state(
             } else {
                 debug!(
                     "error recovery - token {} has no conflicts",
-                    lexical_grammar.variables[i].name
+                    str_pool.resolve(lexical_grammar.variables[i].name)
                 );
                 Some(Symbol::terminal(i))
             }
@@ -192,30 +208,31 @@ fn populate_error_state(
 
     let recover_entry = ParseTableEntry {
         reusable: false,
-        actions: vec![ParseAction::Recover],
+        actions: ActionList::One(ParseAction::Recover),
     };
 
     // Exclude from the error-recovery state any token that conflicts with one of
     // the *conflict-free tokens* identified above.
     for i in 0..n {
         let symbol = Symbol::terminal(i);
-        if !conflict_free_tokens.contains(&symbol)
-            && !keywords.contains(&symbol)
+        if !conflict_free_tokens.contains(symbol)
+            && !keywords.contains(symbol)
             && syntax_grammar.word_token != Some(symbol)
             && let Some(t) = conflict_free_tokens.iter().find(|t| {
                 !coincident_token_index.contains(symbol, *t)
-                    && token_conflict_map.does_conflict(symbol.index, t.index)
+                    && token_conflict_map.does_conflict(symbol.index as usize, t.index as usize)
             })
         {
             debug!(
                 "error recovery - exclude token {} because of conflict with {}",
-                lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
+                str_pool.resolve(lexical_grammar.variables[i].name),
+                str_pool.resolve(lexical_grammar.variables[t.index as usize].name)
             );
             continue;
         }
         debug!(
             "error recovery - include token {}",
-            lexical_grammar.variables[i].name
+            str_pool.resolve(lexical_grammar.variables[i].name)
         );
         state
             .terminal_entries
@@ -236,7 +253,7 @@ fn populate_error_state(
 }
 
 fn populate_used_symbols(
-    parse_table: &mut ParseTable,
+    parse_table: &mut ParseTable<ParseTableEntry>,
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
 ) {
@@ -246,13 +263,13 @@ fn populate_used_symbols(
     for state in &parse_table.states {
         for symbol in state.terminal_entries.keys() {
             match symbol.kind {
-                SymbolType::Terminal => terminal_usages[symbol.index] = true,
-                SymbolType::External => external_usages[symbol.index] = true,
+                SymbolType::Terminal => terminal_usages[symbol.index as usize] = true,
+                SymbolType::External => external_usages[symbol.index as usize] = true,
                 _ => {}
             }
         }
         for symbol in state.nonterminal_entries.keys() {
-            non_terminal_usages[symbol.index] = true;
+            non_terminal_usages[symbol.index as usize] = true;
         }
     }
     parse_table.symbols.push(Symbol::end());
@@ -264,7 +281,10 @@ fn populate_used_symbols(
             // ensure that a subtree's symbol can be successfully reassigned to the word token
             // without having to move the subtree to the heap.
             // See https://github.com/tree-sitter/tree-sitter/issues/258
-            if syntax_grammar.word_token.is_some_and(|t| t.index == i) {
+            if syntax_grammar
+                .word_token
+                .is_some_and(|t| t.index as usize == i)
+            {
                 parse_table.symbols.insert(1, Symbol::terminal(i));
             } else {
                 parse_table.symbols.push(Symbol::terminal(i));
@@ -284,7 +304,7 @@ fn populate_used_symbols(
 }
 
 fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &SyntaxGrammar) {
-    let mut external_tokens_by_corresponding_internal_token = HashMap::new();
+    let mut external_tokens_by_corresponding_internal_token = FxHashMap::default();
     for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
         if let Some(symbol) = external_token.corresponding_internal_token {
             external_tokens_by_corresponding_internal_token.insert(symbol.index, i);
@@ -315,16 +335,16 @@ fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &S
             .unwrap_or_else(|| {
                 parse_table.external_lex_states.push(external_tokens);
                 parse_table.external_lex_states.len() - 1
-            });
+            }) as u32;
     }
 }
 
 fn identify_keywords(
     lexical_grammar: &LexicalGrammar,
-    parse_table: &ParseTable,
     word_token: Option<Symbol>,
     token_conflict_map: &TokenConflictMap,
     coincident_token_index: &CoincidentTokenIndex,
+    str_pool: &StrPool,
 ) -> TokenSet {
     if word_token.is_none() {
         return TokenSet::new();
@@ -342,12 +362,12 @@ fn identify_keywords(
         .filter_map(|(i, variable)| {
             cursor.reset(vec![variable.start_state]);
             if all_chars_are_alphabetical(&cursor)
-                && token_conflict_map.does_match_same_string(i, word_token.index)
-                && !token_conflict_map.does_match_different_string(i, word_token.index)
+                && token_conflict_map.does_match_same_string(i, word_token.index as usize)
+                && !token_conflict_map.does_match_different_string(i, word_token.index as usize)
             {
                 debug!(
                     "Keywords - add candidate {}",
-                    lexical_grammar.variables[i].name
+                    str_pool.resolve(lexical_grammar.variables[i].name)
                 );
                 Some(Symbol::terminal(i))
             } else {
@@ -362,12 +382,14 @@ fn identify_keywords(
         .filter(|token| {
             for other_token in keyword_candidates.iter() {
                 if other_token != *token
-                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
+                    && token_conflict_map
+                        .does_match_same_string(other_token.index as usize, token.index as usize)
                 {
                     debug!(
                         "Keywords - exclude {} because it matches the same string as {}",
-                        lexical_grammar.variables[token.index].name,
-                        lexical_grammar.variables[other_token.index].name
+                        str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
+                        str_pool
+                            .resolve(lexical_grammar.variables[other_token.index as usize].name)
                     );
                     return false;
                 }
@@ -383,7 +405,7 @@ fn identify_keywords(
         .iter()
         .filter(|token| {
             for other_index in 0..lexical_grammar.variables.len() {
-                if keyword_candidates.contains(&Symbol::terminal(other_index)) {
+                if keyword_candidates.contains(Symbol::terminal(other_index)) {
                     continue;
                 }
 
@@ -391,26 +413,20 @@ fn identify_keywords(
                 // this keyword candidate, then substituting the word token won't
                 // introduce any new lexical conflicts.
                 if coincident_token_index
-                    .states_with(*token, Symbol::terminal(other_index))
-                    .iter()
-                    .all(|state_id| {
-                        parse_table.states[*state_id]
-                            .terminal_entries
-                            .contains_key(&word_token)
-                    })
+                    .all_coincident_states_have_word(*token, Symbol::terminal(other_index))
                 {
                     continue;
                 }
 
                 if !token_conflict_map.has_same_conflict_status(
-                    token.index,
-                    word_token.index,
+                    token.index as usize,
+                    word_token.index as usize,
                     other_index,
                 ) {
                     debug!(
                         "Keywords - exclude {} because of conflict with {}",
-                        lexical_grammar.variables[token.index].name,
-                        lexical_grammar.variables[other_index].name
+                        str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
+                        str_pool.resolve(lexical_grammar.variables[other_index].name)
                     );
                     return false;
                 }
@@ -418,33 +434,27 @@ fn identify_keywords(
 
             debug!(
                 "Keywords - include {}",
-                lexical_grammar.variables[token.index].name,
+                str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
             );
             true
         })
         .collect()
 }
 
-fn mark_fragile_tokens(
-    parse_table: &mut ParseTable,
-    lexical_grammar: &LexicalGrammar,
-    token_conflict_map: &TokenConflictMap,
-) {
-    let n = lexical_grammar.variables.len();
-    let mut valid_tokens_mask = Vec::with_capacity(n);
+fn mark_fragile_tokens(parse_table: &mut ParseTable, token_conflict_map: &TokenConflictMap) {
+    let mut valid_terminal_indices = Vec::new();
     for state in &mut parse_table.states {
-        valid_tokens_mask.clear();
-        valid_tokens_mask.resize(n, false);
+        valid_terminal_indices.clear();
         for token in state.terminal_entries.keys() {
             if token.is_terminal() {
-                valid_tokens_mask[token.index] = true;
+                valid_terminal_indices.push(token.index);
             }
         }
-        for (token, entry) in &mut state.terminal_entries {
+        for (token, id) in &mut state.terminal_entries {
             if token.is_terminal() {
-                for (i, is_valid) in valid_tokens_mask.iter().enumerate() {
-                    if *is_valid && token_conflict_map.does_overlap(i, token.index) {
-                        entry.reusable = false;
+                for &i in &valid_terminal_indices {
+                    if token_conflict_map.does_overlap(i as usize, token.index as usize) {
+                        id.set_reusable(false);
                         break;
                     }
                 }
@@ -457,7 +467,8 @@ fn report_state_info<'a>(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     parse_table: &ParseTable,
-    parse_state_info: &[ParseStateInfo<'a>],
+    parse_state_info: &ParseStateInfo<'a>,
+    str_pool: &StrPool,
     report_symbol_name: &'a str,
 ) {
     let mut all_state_indices = BTreeSet::new();
@@ -467,8 +478,8 @@ fn report_state_info<'a>(
 
     for (i, state) in parse_table.states.iter().enumerate() {
         all_state_indices.insert(i);
-        let item_set = &parse_state_info[state.id];
-        for entry in &item_set.1.entries {
+        let item_set = parse_state_info.item_set(state.id);
+        for entry in &item_set.entries {
             if !entry.item.is_augmented() {
                 symbols_with_state_indices[entry.item.variable_index as usize]
                     .1
@@ -482,13 +493,13 @@ fn report_state_info<'a>(
     let max_symbol_name_length = syntax_grammar
         .variables
         .iter()
-        .map(|v| v.name.len())
+        .map(|v| str_pool.resolve(v.name).len())
         .max()
         .unwrap();
     for (symbol, states) in &symbols_with_state_indices {
         info!(
             "{:width$}\t{}",
-            syntax_grammar.variables[symbol.index].name,
+            str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name),
             states.len(),
             width = max_symbol_name_length
         );
@@ -501,7 +512,9 @@ fn report_state_info<'a>(
         symbols_with_state_indices
             .iter()
             .find_map(|(symbol, state_indices)| {
-                if syntax_grammar.variables[symbol.index].name == report_symbol_name {
+                if str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name)
+                    == report_symbol_name
+                {
                     Some(state_indices)
                 } else {
                     None
@@ -515,7 +528,8 @@ fn report_state_info<'a>(
 
         for state_index in state_indices {
             let id = parse_table.states[state_index].id;
-            let (preceding_symbols, item_set) = &parse_state_info[id];
+            let preceding_symbols = &parse_state_info.preceding_symbols_by_id[id as usize];
+            let item_set = parse_state_info.item_set(id);
             info!("state index: {state_index}");
             info!("state id: {id}");
             info!(
@@ -524,11 +538,12 @@ fn report_state_info<'a>(
                     .iter()
                     .map(|symbol| {
                         if symbol.is_terminal() {
-                            lexical_grammar.variables[symbol.index].name.clone()
+                            str_pool.resolve(lexical_grammar.variables[symbol.index as usize].name)
                         } else if symbol.is_external() {
-                            syntax_grammar.external_tokens[symbol.index].name.clone()
+                            str_pool
+                                .resolve(syntax_grammar.external_tokens[symbol.index as usize].name)
                         } else {
-                            syntax_grammar.variables[symbol.index].name.clone()
+                            str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name)
                         }
                     })
                     .collect::<Vec<_>>()
@@ -536,7 +551,13 @@ fn report_state_info<'a>(
             );
             info!(
                 "\nitems:\n{}",
-                item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar),
+                item::ParseItemSetDisplay(
+                    item_set,
+                    syntax_grammar,
+                    lexical_grammar,
+                    str_pool,
+                    &parse_state_info.lookaheads
+                ),
             );
         }
     }

crates/generate/src/build_tables/build_lex_table.rs

@@ -1,8 +1,10 @@
 use std::{
-    collections::{HashMap, VecDeque, hash_map::Entry},
+    collections::{VecDeque, hash_map::Entry},
     mem,
 };
 
+use rustc_hash::FxHashMap;
+
 use log::debug;
 
 use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap};
@@ -11,7 +13,7 @@ use crate::{
     grammars::{LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor},
     rules::{Symbol, TokenSet},
-    tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable},
+    tables::{AdvanceAction, LexState, LexStateId, LexTable, ParseStateId, ParseTable},
 };
 
 pub const LARGE_CHARACTER_RANGE_COUNT: usize = 8;
@@ -47,7 +49,7 @@ pub fn build_lex_table(
             .chain(state.reserved_words.iter())
             .filter_map(|token| {
                 if token.is_terminal() {
-                    if keywords.contains(&token) {
+                    if keywords.contains(token) {
                         syntax_grammar.word_token
                     } else {
                         Some(token)
@@ -69,13 +71,13 @@ pub fn build_lex_table(
                 coincident_token_index,
             ) {
                 did_merge = true;
-                entry.1.push(i);
+                entry.1.push(i as u32);
                 break;
             }
         }
 
         if !did_merge {
-            parse_state_ids_by_token_set.push((tokens, vec![i]));
+            parse_state_ids_by_token_set.push((tokens, vec![i as u32]));
         }
     }
 
@@ -83,7 +85,7 @@ pub fn build_lex_table(
     for (tokens, parse_state_ids) in parse_state_ids_by_token_set {
         let lex_state_id = builder.add_state_for_tokens(&tokens);
         for id in parse_state_ids {
-            parse_table.states[id].lex_state_id = lex_state_id;
+            parse_table.states[id as usize].lex_state_id = lex_state_id;
         }
     }
 
@@ -129,7 +131,7 @@ pub fn build_lex_table(
 }
 
 struct QueueEntry {
-    state_id: usize,
+    state_id: LexStateId,
     nfa_states: Vec<u32>,
     eof_valid: bool,
 }
@@ -139,7 +141,7 @@ struct LexTableBuilder<'a> {
     cursor: NfaCursor<'a>,
     table: LexTable,
     state_queue: VecDeque<QueueEntry>,
-    state_ids_by_nfa_state_set: HashMap<(Vec<u32>, bool), usize>,
+    state_ids_by_nfa_state_set: FxHashMap<(Vec<u32>, bool), LexStateId>,
 }
 
 impl<'a> LexTableBuilder<'a> {
@@ -149,7 +151,7 @@ impl<'a> LexTableBuilder<'a> {
             cursor: NfaCursor::new(&lexical_grammar.nfa, vec![]),
             table: LexTable::default(),
             state_queue: VecDeque::new(),
-            state_ids_by_nfa_state_set: HashMap::new(),
+            state_ids_by_nfa_state_set: FxHashMap::default(),
         }
     }
 
@@ -159,13 +161,13 @@ impl<'a> LexTableBuilder<'a> {
         self.state_ids_by_nfa_state_set.clear();
     }
 
-    fn add_state_for_tokens(&mut self, tokens: &TokenSet) -> usize {
+    fn add_state_for_tokens(&mut self, tokens: &TokenSet) -> LexStateId {
         let mut eof_valid = false;
         let nfa_states = tokens
             .iter()
             .filter_map(|token| {
                 if token.is_terminal() {
-                    Some(self.lexical_grammar.variables[token.index].start_state)
+                    Some(self.lexical_grammar.variables[token.index as usize].start_state)
                 } else {
                     eof_valid = true;
                     None
@@ -179,7 +181,7 @@ impl<'a> LexTableBuilder<'a> {
                 "entry point state: {state_id}, tokens: {:?}",
                 tokens
                     .iter()
-                    .map(|t| &self.lexical_grammar.variables[t.index].name)
+                    .map(|t| &self.lexical_grammar.variables[t.index as usize].name)
                     .collect::<Vec<_>>()
             );
         }
@@ -195,7 +197,7 @@ impl<'a> LexTableBuilder<'a> {
         state_id
     }
 
-    fn add_state(&mut self, nfa_states: Vec<u32>, eof_valid: bool) -> (usize, bool) {
+    fn add_state(&mut self, nfa_states: Vec<u32>, eof_valid: bool) -> (LexStateId, bool) {
         self.cursor.reset(nfa_states);
         match self
             .state_ids_by_nfa_state_set
@@ -203,7 +205,7 @@ impl<'a> LexTableBuilder<'a> {
         {
             Entry::Occupied(o) => (*o.get(), false),
             Entry::Vacant(v) => {
-                let state_id = self.table.states.len();
+                let state_id = self.table.states.len() as u32;
                 self.table.states.push(LexState::default());
                 self.state_queue.push_back(QueueEntry {
                     state_id,
@@ -216,7 +218,7 @@ impl<'a> LexTableBuilder<'a> {
         }
     }
 
-    fn populate_state(&mut self, state_id: usize, nfa_states: Vec<u32>, eof_valid: bool) {
+    fn populate_state(&mut self, state_id: LexStateId, nfa_states: Vec<u32>, eof_valid: bool) {
         self.cursor.force_reset(nfa_states);
 
         // The EOF state is represented as an empty list of NFA states.
@@ -234,14 +236,13 @@ impl<'a> LexTableBuilder<'a> {
             completion = Some((id, prec));
         }
 
-        let transitions = self.cursor.transitions();
-        let has_sep = self.cursor.transition_chars().any(|(_, sep)| sep);
+        let (transitions, has_sep) = self.cursor.transitions_and_any_sep();
 
         // If EOF is a valid lookahead token, add a transition predicated on the null
         // character that leads to the empty set of NFA states.
         if eof_valid {
             let (next_state_id, _) = self.add_state(Vec::new(), false);
-            self.table.states[state_id].eof_action = Some(AdvanceAction {
+            self.table.states[state_id as usize].eof_action = Some(AdvanceAction {
                 state: next_state_id,
                 in_main_token: true,
             });
@@ -262,7 +263,7 @@ impl<'a> LexTableBuilder<'a> {
 
             let (next_state_id, _) =
                 self.add_state(transition.states, eof_valid && transition.is_separator);
-            self.table.states[state_id].advance_actions.push((
+            self.table.states[state_id as usize].advance_actions.push((
                 transition.characters,
                 AdvanceAction {
                     state: next_state_id,
@@ -272,9 +273,10 @@ impl<'a> LexTableBuilder<'a> {
         }
 
         if let Some((complete_id, _)) = completion {
-            self.table.states[state_id].accept_action = Some(Symbol::terminal(complete_id));
+            self.table.states[state_id as usize].accept_action =
+                Some(Symbol::terminal(complete_id));
         } else if self.cursor.state_ids.is_empty() {
-            self.table.states[state_id].accept_action = Some(Symbol::end());
+            self.table.states[state_id as usize].accept_action = Some(Symbol::end());
         }
     }
 }
@@ -285,20 +287,27 @@ fn check_token_conflicts(
     token_conflict_map: &TokenConflictMap,
     coincident_token_index: &CoincidentTokenIndex,
 ) -> bool {
-    let symbol = Symbol::terminal(i);
-    for existing_token in set_without_terminal.terminals() {
-        if token_conflict_map.does_conflict(i, existing_token.index)
-            || token_conflict_map.does_match_prefix(i, existing_token.index)
-        {
-            return true;
-        }
-        if !coincident_token_index.contains(symbol, existing_token)
-            && (token_conflict_map.does_overlap(existing_token.index, i)
-                || token_conflict_map.does_overlap(i, existing_token.index))
-        {
+    let wpr = token_conflict_map.row_words;
+    let row_start = i * wpr;
+    let set_bits = set_without_terminal.terminal_bits_words();
+
+    // Does terminal i conflict with or match-prefix any terminal in the set?
+    let conflict_row = &token_conflict_map.conflict_or_prefix_bits[row_start..row_start + wpr];
+    for (&c, &s) in conflict_row.iter().zip(set_bits) {
+        if c & s != 0 {
             return true;
         }
     }
+
+    // Does terminal i overlap (in either direction) with any non-coincident terminal in the set?
+    let overlap_row = &token_conflict_map.overlap_either_bits[row_start..row_start + wpr];
+    let coincident_row = &coincident_token_index.row_bits[row_start..row_start + wpr];
+    for ((&o, &s), &c) in overlap_row.iter().zip(set_bits).zip(coincident_row) {
+        if o & s & !c != 0 {
+            return true;
+        }
+    }
+
     false
 }
 
@@ -310,10 +319,10 @@ fn merge_token_set(
 ) -> bool {
     if tokens
         .terminals()
-        .filter(|terminal| !other.contains_terminal(terminal.index))
+        .filter(|terminal| !other.contains_terminal(terminal.index as usize))
         .any(|terminal| {
             check_token_conflicts(
-                terminal.index,
+                terminal.index as usize,
                 other,
                 token_conflict_map,
                 coincident_token_index,
@@ -325,10 +334,10 @@ fn merge_token_set(
 
     if other
         .terminals()
-        .filter(|terminal| !tokens.contains_terminal(terminal.index))
+        .filter(|terminal| !tokens.contains_terminal(terminal.index as usize))
         .any(|terminal| {
             check_token_conflicts(
-                terminal.index,
+                terminal.index as usize,
                 tokens,
                 token_conflict_map,
                 coincident_token_index,
@@ -345,7 +354,7 @@ fn merge_token_set(
 fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
     // Initially group the states by their accept action and their
     // valid lookahead characters.
-    let mut state_ids_by_signature = HashMap::new();
+    let mut state_ids_by_signature = FxHashMap::default();
     for (i, state) in table.states.iter().enumerate() {
         let signature = (
             i == 0,
@@ -360,7 +369,7 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         state_ids_by_signature
             .entry(signature)
             .or_insert(Vec::new())
-            .push(i);
+            .push(i as u32);
     }
     let mut state_ids_by_group_id = state_ids_by_signature
         .into_iter()
@@ -373,10 +382,10 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         .unwrap();
     state_ids_by_group_id.swap(error_group_index, 0);
 
-    let mut group_ids_by_state_id = vec![0; table.states.len()];
+    let mut group_ids_by_state_id = vec![0u32; table.states.len()];
     for (group_id, state_ids) in state_ids_by_group_id.iter().enumerate() {
         for state_id in state_ids {
-            group_ids_by_state_id[*state_id] = group_id;
+            group_ids_by_state_id[*state_id as usize] = group_id as u32;
         }
     }
 
@@ -391,30 +400,35 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
     let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
     for state_ids in &state_ids_by_group_id {
         let mut new_state = LexState::default();
-        mem::swap(&mut new_state, &mut table.states[state_ids[0]]);
+        mem::swap(&mut new_state, &mut table.states[state_ids[0] as usize]);
 
         for (_, advance_action) in &mut new_state.advance_actions {
-            advance_action.state = group_ids_by_state_id[advance_action.state];
+            advance_action.state = group_ids_by_state_id[advance_action.state as usize];
         }
         if let Some(eof_action) = &mut new_state.eof_action {
-            eof_action.state = group_ids_by_state_id[eof_action.state];
+            eof_action.state = group_ids_by_state_id[eof_action.state as usize];
         }
         new_states.push(new_state);
     }
 
     for state in &mut parse_table.states {
-        state.lex_state_id = group_ids_by_state_id[state.lex_state_id];
+        state.lex_state_id = group_ids_by_state_id[state.lex_state_id as usize];
     }
 
     table.states = new_states;
 }
 
-fn lex_states_differ(left: &LexState, right: &LexState, group_ids_by_state_id: &[usize]) -> bool {
+fn lex_states_differ(
+    left: &LexState,
+    right: &LexState,
+    group_ids_by_state_id: &[LexStateId],
+) -> bool {
     left.advance_actions
         .iter()
         .zip(right.advance_actions.iter())
         .any(|(left, right)| {
-            group_ids_by_state_id[left.1.state] != group_ids_by_state_id[right.1.state]
+            group_ids_by_state_id[left.1.state as usize]
+                != group_ids_by_state_id[right.1.state as usize]
         })
 }
 
@@ -424,9 +438,9 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
     old_ids_by_new_id[1..].sort_by_key(|id| &table.states[*id]);
 
     // Get the inverse mapping
-    let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
+    let mut new_ids_by_old_id = vec![0u32; old_ids_by_new_id.len()];
     for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
-        new_ids_by_old_id[*old_id] = id;
+        new_ids_by_old_id[*old_id] = id as u32;
     }
 
     // Reorder the parse states and update their references to reflect
@@ -437,10 +451,10 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
             let mut state = LexState::default();
             mem::swap(&mut state, &mut table.states[*old_id]);
             for (_, advance_action) in &mut state.advance_actions {
-                advance_action.state = new_ids_by_old_id[advance_action.state];
+                advance_action.state = new_ids_by_old_id[advance_action.state as usize];
             }
             if let Some(eof_action) = &mut state.eof_action {
-                eof_action.state = new_ids_by_old_id[eof_action.state];
+                eof_action.state = new_ids_by_old_id[eof_action.state as usize];
             }
             state
         })
@@ -448,6 +462,6 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
 
     // Update the parse table's lex state references
     for state in &mut parse_table.states {
-        state.lex_state_id = new_ids_by_old_id[state.lex_state_id];
+        state.lex_state_id = new_ids_by_old_id[state.lex_state_id as usize];
     }
 }

crates/generate/src/build_tables/build_parse_table.rs

@@ -1,13 +1,12 @@
 use std::{
     cmp::Ordering,
-    collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
+    collections::{BTreeMap, BTreeSet, VecDeque},
     hash::BuildHasherDefault,
 };
 
 use indexmap::{IndexMap, map::Entry};
-use log::warn;
-use rustc_hash::FxHasher;
-use serde::Serialize;
+use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
+use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
 use super::{
@@ -15,12 +14,15 @@ use super::{
     item_set_builder::ParseItemSetBuilder,
 };
 use crate::{
+    Diagnostic,
+    build_tables::item::{LookaheadSetPool, START_PRODUCTION_ID, prec_display},
     grammars::{LexicalGrammar, PrecedenceEntry, ReservedWordSetId, SyntaxGrammar, VariableType},
     node_types::VariableInfo,
     rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
+    strpool::StrPool,
     tables::{
-        FieldLocation, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable,
-        ParseTableEntry, ProductionInfo, ProductionInfoId,
+        ActionList, ActionListPool, FieldLocation, GotoAction, ParseAction, ParseState,
+        ParseStateId, ParseTable, ParseTableEntry, ProductionInfo, ProductionInfoId,
     },
 };
 
@@ -29,7 +31,19 @@ use crate::{
 type SymbolSequence = Vec<Symbol>;
 
 type AuxiliarySymbolSequence = Vec<AuxiliarySymbolInfo>;
-pub type ParseStateInfo<'a> = (SymbolSequence, ParseItemSet<'a>);
+
+pub struct ParseStateInfo<'a> {
+    pub preceding_symbols_by_id: Vec<SymbolSequence>,
+    item_sets_by_ids: IndexMap<ParseItemSet<'a>, ParseStateId, BuildHasherDefault<FxHasher>>,
+    pub lookaheads: LookaheadSetPool,
+}
+
+impl<'a> ParseStateInfo<'a> {
+    #[must_use]
+    pub fn item_set(&self, id: ParseStateId) -> &ParseItemSet<'a> {
+        self.item_sets_by_ids.get_index(id as usize).unwrap().0
+    }
+}
 
 #[derive(Clone, PartialEq)]
 struct AuxiliarySymbolInfo {
@@ -56,18 +70,20 @@ struct ParseTableBuilder<'a> {
     syntax_grammar: &'a SyntaxGrammar,
     lexical_grammar: &'a LexicalGrammar,
     variable_info: &'a [VariableInfo],
-    core_ids_by_core: HashMap<ParseItemSetCore<'a>, usize>,
+    core_ids_by_core: FxHashMap<ParseItemSetCore<'a>, u32>,
     state_ids_by_item_set: IndexMap<ParseItemSet<'a>, ParseStateId, BuildHasherDefault<FxHasher>>,
-    parse_state_info_by_id: Vec<ParseStateInfo<'a>>,
+    preceding_symbols_by_id: Vec<SymbolSequence>,
+    production_info_ids_by_prod_id: Vec<Option<ProductionInfoId>>,
     parse_state_queue: VecDeque<ParseStateQueueEntry>,
-    non_terminal_extra_states: Vec<(Symbol, usize)>,
-    actual_conflicts: HashSet<Vec<Symbol>>,
-    parse_table: ParseTable,
+    non_terminal_extra_states: Vec<(Symbol, ParseStateId)>,
+    actual_conflicts: FxHashSet<Vec<Symbol>>,
+    parse_table: ParseTable<ParseTableEntry>,
+    str_pool: &'a StrPool,
 }
 
 pub type BuildTableResult<T> = Result<T, ParseTableBuilderError>;
 
-#[derive(Debug, Error, Serialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum ParseTableBuilderError {
     #[error("Unresolved conflict for symbol sequence:\n\n{0}")]
     Conflict(#[from] ConflictError),
@@ -81,7 +97,7 @@ pub enum ParseTableBuilderError {
     StateCount(usize),
 }
 
-#[derive(Default, Debug, Serialize, Error)]
+#[derive(Default, Debug, Serialize, Error, Deserialize, PartialEq, Eq)]
 pub struct ConflictError {
     pub symbol_sequence: Vec<String>,
     pub conflicting_lookahead: String,
@@ -89,7 +105,7 @@ pub struct ConflictError {
     pub possible_resolutions: Vec<Resolution>,
 }
 
-#[derive(Default, Debug, Serialize, Error)]
+#[derive(Default, Debug, Serialize, Error, Deserialize, PartialEq, Eq)]
 pub struct Interpretation {
     pub preceding_symbols: Vec<String>,
     pub variable_name: String,
@@ -99,16 +115,17 @@ pub struct Interpretation {
     pub conflicting_lookahead: String,
     pub precedence: Option<String>,
     pub associativity: Option<String>,
+    pub requires_eof_lookahead: bool,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub enum Resolution {
     Precedence { symbols: Vec<String> },
     Associativity { symbols: Vec<String> },
     AddConflict { symbols: Vec<String> },
 }
 
-#[derive(Debug, Serialize, Error)]
+#[derive(Debug, Serialize, Deserialize, Error, PartialEq, Eq)]
 pub struct AmbiguousExtraError {
     pub parent_symbols: Vec<String>,
 }
@@ -126,16 +143,21 @@ impl std::fmt::Display for ConflictError {
             .iter()
             .map(|i| {
                 let line = i.to_string();
-                let prec_line = if let (Some(precedence), Some(associativity)) =
-                    (&i.precedence, &i.associativity)
-                {
-                    Some(format!(
+                let mut annotations = Vec::new();
+                if let (Some(precedence), Some(associativity)) = (&i.precedence, &i.associativity) {
+                    annotations.push(format!(
                         "(precedence: {precedence}, associativity: {associativity})",
-                    ))
+                    ));
+                } else if let Some(precedence) = &i.precedence {
+                    annotations.push(format!("(precedence: {precedence})"));
+                }
+                if i.requires_eof_lookahead {
+                    annotations.push("(reduces only at end of input)".to_string());
+                }
+                let prec_line = if annotations.is_empty() {
+                    None
                 } else {
-                    i.precedence
-                        .as_ref()
-                        .map(|precedence| format!("(precedence: {precedence})"))
+                    Some(annotations.join("  "))
                 };
 
                 (line, prec_line)
@@ -244,6 +266,7 @@ impl<'a> ParseTableBuilder<'a> {
         lexical_grammar: &'a LexicalGrammar,
         item_set_builder: ParseItemSetBuilder<'a>,
         variable_info: &'a [VariableInfo],
+        str_pool: &'a StrPool,
     ) -> Self {
         Self {
             syntax_grammar,
@@ -252,21 +275,27 @@ impl<'a> ParseTableBuilder<'a> {
             variable_info,
             non_terminal_extra_states: Vec::new(),
             state_ids_by_item_set: IndexMap::default(),
-            core_ids_by_core: HashMap::new(),
-            parse_state_info_by_id: Vec::new(),
+            core_ids_by_core: FxHashMap::default(),
+            preceding_symbols_by_id: Vec::new(),
+            production_info_ids_by_prod_id: vec![None; syntax_grammar.productions.len()],
             parse_state_queue: VecDeque::new(),
             actual_conflicts: syntax_grammar.expected_conflicts.iter().cloned().collect(),
             parse_table: ParseTable {
                 states: Vec::new(),
+                action_lists: ActionListPool::default(),
                 symbols: Vec::new(),
                 external_lex_states: Vec::new(),
                 production_infos: Vec::new(),
                 max_aliased_production_length: 1,
             },
+            str_pool,
         }
     }
 
-    fn build(mut self) -> BuildTableResult<(ParseTable, Vec<ParseStateInfo<'a>>)> {
+    fn build(
+        mut self,
+        diagnostics: &mut Vec<Diagnostic>,
+    ) -> BuildTableResult<(ParseTable<ParseTableEntry>, ParseStateInfo<'a>)> {
         // Ensure that the empty alias sequence has index 0.
         self.parse_table
             .production_infos
@@ -276,13 +305,14 @@ impl<'a> ParseTableBuilder<'a> {
         self.add_parse_state(&Vec::new(), &Vec::new(), ParseItemSet::default());
 
         // Add the starting state at index 1.
+        let end_lookaheads = self.item_set_builder.lookaheads.singleton(Symbol::end());
         self.add_parse_state(
             &Vec::new(),
             &Vec::new(),
             ParseItemSet {
                 entries: vec![ParseItemSetEntry {
-                    item: ParseItem::start(),
-                    lookaheads: std::iter::once(Symbol::end()).collect(),
+                    item: ParseItem::start(self.item_set_builder.key_map),
+                    lookaheads: end_lookaheads,
                     following_reserved_word_set: ReservedWordSetId::default(),
                 }],
             },
@@ -296,33 +326,39 @@ impl<'a> ParseTableBuilder<'a> {
             .iter()
             .filter(|s| s.is_non_terminal())
         {
-            let variable = &self.syntax_grammar.variables[extra_non_terminal.index];
-            for production in &variable.productions {
-                non_terminal_extra_item_sets_by_first_terminal
+            for prod_id in self
+                .syntax_grammar
+                .variable_prod_ids(extra_non_terminal.index as usize)
+            {
+                let production = self.syntax_grammar.production(prod_id);
+                let entry = non_terminal_extra_item_sets_by_first_terminal
                     .entry(production.first_symbol().unwrap())
                     .or_insert_with(ParseItemSet::default)
                     .insert(ParseItem {
-                        variable_index: extra_non_terminal.index as u32,
-                        production,
+                        variable_index: extra_non_terminal.index,
+                        prod_id,
                         step_index: 1,
+                        keys: self.item_set_builder.key_map.keys_for(prod_id),
                         has_preceding_inherited_fields: false,
-                    })
+                    });
+                entry.lookaheads = self
+                    .item_set_builder
                     .lookaheads
-                    .insert(Symbol::end_of_nonterminal_extra());
+                    .insert(entry.lookaheads, Symbol::end_of_nonterminal_extra());
             }
         }
 
         let non_terminal_sets_len = non_terminal_extra_item_sets_by_first_terminal.len();
         self.non_terminal_extra_states
             .reserve(non_terminal_sets_len);
-        self.parse_state_info_by_id.reserve(non_terminal_sets_len);
+        self.preceding_symbols_by_id.reserve(non_terminal_sets_len);
         self.parse_table.states.reserve(non_terminal_sets_len);
         self.parse_state_queue.reserve(non_terminal_sets_len);
         // Add a state for each starting terminal of a non-terminal extra rule.
         for (terminal, item_set) in non_terminal_extra_item_sets_by_first_terminal {
             if terminal.is_non_terminal() {
                 Err(ParseTableBuilderError::ImproperNonTerminalExtra(
-                    self.symbol_name(&terminal),
+                    self.symbol_name(terminal),
                 ))?;
             }
 
@@ -333,12 +369,18 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         while let Some(entry) = self.parse_state_queue.pop_front() {
-            let item_set = self
-                .item_set_builder
-                .transitive_closure(&self.parse_state_info_by_id[entry.state_id].1);
+            // The dedup-map key is each state's kernel (the GOTO result, pre closure).
+            // Two states are identical iff their kernels match.
+            let kernel = self
+                .state_ids_by_item_set
+                .get_index(entry.state_id as usize)
+                // Invariant: `state_id` is the map's insertion index
+                .unwrap()
+                .0;
+            let item_set = self.item_set_builder.transitive_closure(kernel);
 
             self.add_actions(
-                self.parse_state_info_by_id[entry.state_id].0.clone(),
+                self.preceding_symbols_by_id[entry.state_id as usize].clone(),
                 entry.preceding_auxiliary_symbols,
                 entry.state_id,
                 &item_set,
@@ -346,24 +388,23 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         if !self.actual_conflicts.is_empty() {
-            warn!(
-                "unnecessary conflicts:\n  {}",
-                &self
-                    .actual_conflicts
-                    .iter()
-                    .map(|conflict| {
-                        conflict
-                            .iter()
-                            .map(|symbol| format!("`{}`", self.symbol_name(symbol)))
-                            .collect::<Vec<_>>()
-                            .join(", ")
-                    })
-                    .collect::<Vec<_>>()
-                    .join("\n  ")
-            );
+            let mut conflicts = self
+                .actual_conflicts
+                .iter()
+                .map(|conf| conf.iter().map(|&s| self.symbol_name(s)).collect())
+                .collect::<Vec<_>>();
+            conflicts.sort_unstable();
+            diagnostics.push(Diagnostic::UnnecessaryConflicts(conflicts));
         }
 
-        Ok((self.parse_table, self.parse_state_info_by_id))
+        Ok((
+            self.parse_table,
+            ParseStateInfo {
+                preceding_symbols_by_id: self.preceding_symbols_by_id,
+                item_sets_by_ids: self.state_ids_by_item_set,
+                lookaheads: self.item_set_builder.lookaheads,
+            },
+        ))
     }
 
     fn add_parse_state(
@@ -381,12 +422,11 @@ impl<'a> ParseTableBuilder<'a> {
             // parse states to populate.
             Entry::Vacant(v) => {
                 let core = v.key().core();
-                let core_count = self.core_ids_by_core.len();
+                let core_count = self.core_ids_by_core.len() as u32;
                 let core_id = *self.core_ids_by_core.entry(core).or_insert(core_count);
 
-                let state_id = self.parse_table.states.len();
-                self.parse_state_info_by_id
-                    .push((preceding_symbols.clone(), v.key().clone()));
+                let state_id = self.parse_table.states.len() as u32;
+                self.preceding_symbols_by_id.push(preceding_symbols.clone());
 
                 self.parse_table.states.push(ParseState {
                     id: state_id,
@@ -396,6 +436,7 @@ impl<'a> ParseTableBuilder<'a> {
                     nonterminal_entries: IndexMap::default(),
                     reserved_words: TokenSet::default(),
                     core_id,
+                    has_eof_gated_reduce: false,
                 });
                 self.parse_state_queue.push_back(ParseStateQueueEntry {
                     state_id,
@@ -417,7 +458,11 @@ impl<'a> ParseTableBuilder<'a> {
         let mut terminal_successors = BTreeMap::new();
         let mut non_terminal_successors = BTreeMap::new();
         let mut lookaheads_with_conflicts = TokenSet::new();
-        let mut reduction_infos = HashMap::<Symbol, ReductionInfo>::new();
+        let mut reduction_infos = FxHashMap::<Symbol, ReductionInfo>::default();
+
+        // `get_auxiliary_node_info` scans every entry in `item_set`, and the same auxiliary
+        // symbol typically appears across many entries in a state. Memoize per symbol.
+        let mut aux_node_info = FxHashMap::<Symbol, AuxiliarySymbolInfo>::default();
 
         // Each item in the item set contributes to either or a Shift action or a Reduce
         // action in this state.
@@ -430,17 +475,23 @@ impl<'a> ParseTableBuilder<'a> {
             // If the item is unfinished, then this state has a transition for the item's
             // next symbol. Advance the item to its next step and insert the resulting
             // item into the successor item set.
-            if let Some(next_symbol) = item.symbol() {
+            if let Some(next_symbol) = item.symbol(self.syntax_grammar) {
                 let mut successor = item.successor();
                 let successor_set = if next_symbol.is_non_terminal() {
-                    let variable = &self.syntax_grammar.variables[next_symbol.index];
+                    let variable = &self.syntax_grammar.variables[next_symbol.index as usize];
 
                     // Keep track of where auxiliary non-terminals (repeat symbols) are
                     // used within visible symbols. This information may be needed later
                     // for conflict resolution.
                     if variable.is_auxiliary() {
-                        preceding_auxiliary_symbols
-                            .push(self.get_auxiliary_node_info(item_set, next_symbol));
+                        preceding_auxiliary_symbols.push(
+                            aux_node_info
+                                .entry(next_symbol)
+                                .or_insert_with(|| {
+                                    self.get_auxiliary_node_info(item_set, next_symbol)
+                                })
+                                .clone(),
+                        );
                     }
 
                     // For most parse items, the symbols associated with the preceding children
@@ -453,7 +504,9 @@ impl<'a> ParseTableBuilder<'a> {
                     // of its preceding children need to be taken into account when comparing
                     // it with other items.
                     if variable.is_hidden()
-                        && !self.variable_info[next_symbol.index].fields.is_empty()
+                        && !self.variable_info[next_symbol.index as usize]
+                            .fields
+                            .is_empty()
                     {
                         successor.has_preceding_inherited_fields = true;
                     }
@@ -467,7 +520,10 @@ impl<'a> ParseTableBuilder<'a> {
                         .or_insert_with(ParseItemSet::default)
                 };
                 let successor_entry = successor_set.insert(successor);
-                successor_entry.lookaheads.insert_all(lookaheads);
+                successor_entry.lookaheads = self
+                    .item_set_builder
+                    .lookaheads
+                    .union(successor_entry.lookaheads, *lookaheads);
                 successor_entry.following_reserved_word_set = successor_entry
                     .following_reserved_word_set
                     .max(*reserved_lookaheads);
@@ -479,18 +535,37 @@ impl<'a> ParseTableBuilder<'a> {
                 let action = if item.is_augmented() {
                     ParseAction::Accept
                 } else {
+                    // These values are narrowed to u16 to reduce the size of
+                    // ParseAction. No real-world grammar approaches these limits.
+                    debug_assert!(
+                        u16::try_from(item.step_index).is_ok(),
+                        "production step count exceeds u16::MAX"
+                    );
+                    let production_id = self.get_production_id(item);
+                    debug_assert!(
+                        u16::try_from(production_id).is_ok(),
+                        "production info id exceeds u16::MAX"
+                    );
                     ParseAction::Reduce {
                         symbol,
-                        child_count: item.step_index as usize,
-                        dynamic_precedence: item.production.dynamic_precedence,
-                        production_id: self.get_production_id(item),
+                        child_count: item.step_index as u16,
+                        dynamic_precedence: item.production(self.syntax_grammar).dynamic_precedence,
+                        production_id: production_id as u16,
                     }
                 };
 
-                let precedence = item.precedence();
-                let associativity = item.associativity();
-                for lookahead in lookaheads.iter() {
-                    let table_entry = self.parse_table.states[state_id]
+                let precedence = item.precedence(self.syntax_grammar);
+                let associativity = item.associativity(self.syntax_grammar);
+                if item.production(self.syntax_grammar).requires_eof_lookahead {
+                    self.parse_table.states[state_id as usize].has_eof_gated_reduce = true;
+                }
+                for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() {
+                    if item.production(self.syntax_grammar).requires_eof_lookahead
+                        && lookahead != Symbol::end()
+                    {
+                        continue;
+                    }
+                    let table_entry = self.parse_table.states[state_id as usize]
                         .terminal_entries
                         .entry(lookahead)
                         .or_insert_with(ParseTableEntry::new);
@@ -506,24 +581,30 @@ impl<'a> ParseTableBuilder<'a> {
                             self.syntax_grammar,
                             precedence,
                             &[symbol],
-                            &reduction_info.precedence,
+                            reduction_info.precedence,
                             &reduction_info.symbols,
                         ) {
                             Ordering::Greater => {
                                 table_entry.actions.clear();
                                 table_entry.actions.push(action);
-                                lookaheads_with_conflicts.remove(&lookahead);
+                                lookaheads_with_conflicts.remove(lookahead);
                                 *reduction_info = ReductionInfo::default();
                             }
+                            // Two items that reduce identically build the same tree, so
+                            // there is nothing for the user to resolve. Precedence is
+                            // still compared first, because an item that only repeats an
+                            // existing action can still outrank it and clear the entry.
                             Ordering::Equal => {
-                                table_entry.actions.push(action);
-                                lookaheads_with_conflicts.insert(lookahead);
+                                if !table_entry.actions.contains(&action) {
+                                    table_entry.actions.push(action);
+                                    lookaheads_with_conflicts.insert(lookahead);
+                                }
                             }
                             Ordering::Less => continue,
                         }
                     }
 
-                    reduction_info.precedence.clone_from(precedence);
+                    reduction_info.precedence = precedence;
                     if let Err(i) = reduction_info.symbols.binary_search(&symbol) {
                         reduction_info.symbols.insert(i, symbol);
                     }
@@ -550,7 +631,7 @@ impl<'a> ParseTableBuilder<'a> {
             );
             preceding_symbols.pop();
 
-            let entry = self.parse_table.states[state_id]
+            let entry = self.parse_table.states[state_id as usize]
                 .terminal_entries
                 .entry(symbol);
             if let Entry::Occupied(e) = &entry
@@ -576,7 +657,7 @@ impl<'a> ParseTableBuilder<'a> {
                 next_item_set,
             );
             preceding_symbols.pop();
-            self.parse_table.states[state_id]
+            self.parse_table.states[state_id as usize]
                 .nonterminal_entries
                 .insert(symbol, GotoAction::Goto(next_state_id));
         }
@@ -598,7 +679,7 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         // Add actions for the grammar's `extra` symbols.
-        let state = &mut self.parse_table.states[state_id];
+        let state = &mut self.parse_table.states[state_id as usize];
         let is_end_of_non_terminal_extra = state.is_end_of_non_terminal_extra();
 
         // If this state represents the end of a non-terminal extra rule, then make sure that
@@ -616,13 +697,13 @@ impl<'a> ParseTableBuilder<'a> {
                             None
                         }
                     })
-                    .collect::<HashSet<_>>();
+                    .collect::<FxHashSet<_>>();
                 let parent_symbol_names = parent_symbols
                     .iter()
                     .map(|&variable_index| {
-                        self.syntax_grammar.variables[variable_index as usize]
-                            .name
-                            .clone()
+                        self.str_pool
+                            .resolve(self.syntax_grammar.variables[variable_index as usize].name)
+                            .to_string()
                     })
                     .collect::<Vec<_>>();
 
@@ -639,10 +720,10 @@ impl<'a> ParseTableBuilder<'a> {
                     .entry(*terminal)
                     .or_insert(ParseTableEntry {
                         reusable: true,
-                        actions: vec![ParseAction::Shift {
+                        actions: ActionList::One(ParseAction::Shift {
                             state: *state_id,
                             is_repetition: false,
-                        }],
+                        }),
                     });
             }
 
@@ -660,7 +741,7 @@ impl<'a> ParseTableBuilder<'a> {
                         .entry(*extra_token)
                         .or_insert(ParseTableEntry {
                             reusable: true,
-                            actions: vec![ParseAction::ShiftExtra],
+                            actions: ActionList::One(ParseAction::ShiftExtra),
                         });
                 }
             }
@@ -671,13 +752,18 @@ impl<'a> ParseTableBuilder<'a> {
                 .entries
                 .iter()
                 .filter_map(|entry| {
-                    if let Some(next_step) = entry.item.step() {
-                        if next_step.symbol == keyword_capture_token {
-                            Some(next_step.reserved_word_set_id)
+                    if let Some(next_step) = entry.item.step(self.syntax_grammar) {
+                        if next_step.symbol() == keyword_capture_token {
+                            Some(ReservedWordSetId(u32::from(next_step.reserved)))
                         } else {
                             None
                         }
-                    } else if entry.lookaheads.contains(&keyword_capture_token) {
+                    } else if self
+                        .item_set_builder
+                        .lookaheads
+                        .get(entry.lookaheads)
+                        .contains(keyword_capture_token)
+                    {
                         Some(entry.following_reserved_word_set)
                     } else {
                         None
@@ -686,7 +772,7 @@ impl<'a> ParseTableBuilder<'a> {
                 .max();
             if let Some(reserved_word_set_id) = reserved_word_set_id {
                 state.reserved_words =
-                    self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0].clone();
+                    self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0 as usize].clone();
             }
         }
 
@@ -702,7 +788,7 @@ impl<'a> ParseTableBuilder<'a> {
         conflicting_lookahead: Symbol,
         reduction_info: &ReductionInfo,
     ) -> BuildTableResult<()> {
-        let entry = self.parse_table.states[state_id]
+        let entry = self.parse_table.states[state_id as usize]
             .terminal_entries
             .get_mut(&conflicting_lookahead)
             .unwrap();
@@ -714,32 +800,37 @@ impl<'a> ParseTableBuilder<'a> {
         // REDUCE-REDUCE conflicts where all actions have the *same*
         // precedence, and there can still be SHIFT/REDUCE conflicts.
         let mut considered_associativity = false;
-        let mut shift_precedence = Vec::<(&Precedence, Symbol)>::new();
+        let mut shift_precedence = Vec::<(Precedence, Symbol)>::new();
         let mut conflicting_items = BTreeSet::new();
         for ParseItemSetEntry {
             item, lookaheads, ..
         } in &item_set.entries
         {
-            if let Some(step) = item.step() {
+            if let Some(step) = item.step(self.syntax_grammar) {
                 if item.step_index > 0
                     && self
                         .item_set_builder
-                        .first_set(&step.symbol)
-                        .contains(&conflicting_lookahead)
+                        .first_set(step.symbol())
+                        .contains(conflicting_lookahead)
                 {
                     if item.variable_index != u32::MAX {
                         conflicting_items.insert(item);
                     }
 
                     let p = (
-                        item.precedence(),
+                        item.precedence(self.syntax_grammar),
                         Symbol::non_terminal(item.variable_index as usize),
                     );
                     if let Err(i) = shift_precedence.binary_search(&p) {
                         shift_precedence.insert(i, p);
                     }
                 }
-            } else if lookaheads.contains(&conflicting_lookahead) && item.variable_index != u32::MAX
+            } else if self
+                .item_set_builder
+                .lookaheads
+                .get(*lookaheads)
+                .contains(conflicting_lookahead)
+                && item.variable_index != u32::MAX
             {
                 conflicting_items.insert(item);
             }
@@ -764,30 +855,51 @@ impl<'a> ParseTableBuilder<'a> {
 
             // If the SHIFT action has higher precedence, remove all the REDUCE actions.
             let mut shift_is_less = false;
+            let mut shift_is_equal = false;
             let mut shift_is_more = false;
             for p in shift_precedence {
                 match Self::compare_precedence(
                     self.syntax_grammar,
                     p.0,
                     &[p.1],
-                    &reduction_info.precedence,
+                    reduction_info.precedence,
                     &reduction_info.symbols,
                 ) {
                     Ordering::Greater => shift_is_more = true,
                     Ordering::Less => shift_is_less = true,
-                    Ordering::Equal => {}
+                    Ordering::Equal => shift_is_equal = true,
                 }
             }
 
             if shift_is_more && !shift_is_less {
-                entry.actions.drain(0..entry.actions.len() - 1);
+                entry.actions.keep_last();
             }
             // If the REDUCE actions have higher precedence, remove the SHIFT action.
             else if shift_is_less && !shift_is_more {
-                entry.actions.pop();
-                conflicting_items.retain(|item| item.is_done());
+                // Exception: if one SHIFT interpretation ties the REDUCE actions in
+                // precedence while another has lower precedence, and the REDUCE
+                // actions are purely right associative, honor that right
+                // associativity by shifting rather than reducing. The
+                // lower-precedence interpretation coexists with the tying one, so on
+                // its own it must not force a REDUCE that would flip the tie to left
+                // associative.
+                if shift_is_equal
+                    && matches!(
+                        (
+                            reduction_info.has_left_assoc,
+                            reduction_info.has_non_assoc,
+                            reduction_info.has_right_assoc,
+                        ),
+                        (false, false, true)
+                    )
+                {
+                    entry.actions.keep_last();
+                } else {
+                    entry.actions.pop();
+                    conflicting_items.retain(|item| item.is_done());
+                }
             }
-            // If the SHIFT and REDUCE actions have the same predence, consider
+            // If the SHIFT and REDUCE actions have the same precedence, consider
             // the REDUCE actions' associativity.
             else if !shift_is_less && !shift_is_more {
                 considered_associativity = true;
@@ -804,7 +916,7 @@ impl<'a> ParseTableBuilder<'a> {
                         conflicting_items.retain(|item| item.is_done());
                     }
                     (false, false, true) => {
-                        entry.actions.drain(0..entry.actions.len() - 1);
+                        entry.actions.keep_last();
                     }
                     _ => {}
                 }
@@ -812,7 +924,7 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         // If all of the actions but one have been eliminated, then there's no problem.
-        let entry = self.parse_table.states[state_id]
+        let entry = self.parse_table.states[state_id as usize]
             .terminal_entries
             .get_mut(&conflicting_lookahead)
             .unwrap();
@@ -824,7 +936,7 @@ impl<'a> ParseTableBuilder<'a> {
         let mut actual_conflict = Vec::new();
         for item in &conflicting_items {
             let symbol = Symbol::non_terminal(item.variable_index as usize);
-            if self.syntax_grammar.variables[symbol.index].is_auxiliary() {
+            if self.syntax_grammar.variables[symbol.index as usize].is_auxiliary() {
                 actual_conflict.extend(
                     preceding_auxiliary_symbols
                         .iter()
@@ -857,12 +969,12 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         let mut conflict_error = ConflictError::default();
-        for symbol in preceding_symbols {
+        for &symbol in preceding_symbols {
             conflict_error
                 .symbol_sequence
                 .push(self.symbol_name(symbol));
         }
-        conflict_error.conflicting_lookahead = self.symbol_name(&conflicting_lookahead);
+        conflict_error.conflicting_lookahead = self.symbol_name(conflicting_lookahead);
 
         let interpretations = conflicting_items
             .iter()
@@ -870,26 +982,32 @@ impl<'a> ParseTableBuilder<'a> {
                 let preceding_symbols = preceding_symbols
                     .iter()
                     .take(preceding_symbols.len() - item.step_index as usize)
-                    .map(|symbol| self.symbol_name(symbol))
+                    .map(|&symbol| self.symbol_name(symbol))
                     .collect::<Vec<_>>();
 
-                let variable_name = self.syntax_grammar.variables[item.variable_index as usize]
-                    .name
-                    .clone();
+                let variable_name = self
+                    .str_pool
+                    .resolve(self.syntax_grammar.variables[item.variable_index as usize].name)
+                    .to_string();
 
                 let production_step_symbols = item
-                    .production
+                    .production(self.syntax_grammar)
                     .steps
                     .iter()
-                    .map(|step| self.symbol_name(&step.symbol))
+                    .map(|step| self.symbol_name(step.symbol()))
                     .collect::<Vec<_>>();
 
-                let precedence = match item.precedence() {
+                let precedence = match item.precedence(self.syntax_grammar) {
                     Precedence::None => None,
-                    _ => Some(item.precedence().to_string()),
+                    _ => Some(prec_display(
+                        item.precedence(self.syntax_grammar),
+                        self.str_pool,
+                    )),
                 };
 
-                let associativity = item.associativity().map(|assoc| format!("{assoc:?}"));
+                let associativity = item
+                    .associativity(self.syntax_grammar)
+                    .map(|assoc| format!("{assoc:?}"));
 
                 Interpretation {
                     preceding_symbols,
@@ -897,9 +1015,12 @@ impl<'a> ParseTableBuilder<'a> {
                     production_step_symbols,
                     step_index: item.step_index,
                     done: item.is_done(),
-                    conflicting_lookahead: self.symbol_name(&conflicting_lookahead),
+                    conflicting_lookahead: self.symbol_name(conflicting_lookahead),
                     precedence,
                     associativity,
+                    requires_eof_lookahead: item
+                        .production(self.syntax_grammar)
+                        .requires_eof_lookahead,
                 }
             })
             .collect::<Vec<_>>();
@@ -925,7 +1046,7 @@ impl<'a> ParseTableBuilder<'a> {
                     continue;
                 }
                 last_rule_id = Some(item.variable_index);
-                result.push(self.symbol_name(&Symbol::non_terminal(item.variable_index as usize)));
+                result.push(self.symbol_name(Symbol::non_terminal(item.variable_index as usize)));
             }
 
             result
@@ -940,7 +1061,7 @@ impl<'a> ParseTableBuilder<'a> {
             }
 
             for item in &reduce_items {
-                let name = self.symbol_name(&Symbol::non_terminal(item.variable_index as usize));
+                let name = self.symbol_name(Symbol::non_terminal(item.variable_index as usize));
                 conflict_error
                     .possible_resolutions
                     .push(Resolution::Precedence {
@@ -961,7 +1082,7 @@ impl<'a> ParseTableBuilder<'a> {
             .push(Resolution::AddConflict {
                 symbols: actual_conflict
                     .iter()
-                    .map(|s| self.symbol_name(s))
+                    .map(|&s| self.symbol_name(s))
                     .collect(),
             });
 
@@ -972,33 +1093,33 @@ impl<'a> ParseTableBuilder<'a> {
 
     fn compare_precedence(
         grammar: &SyntaxGrammar,
-        left: &Precedence,
+        left: Precedence,
         left_symbols: &[Symbol],
-        right: &Precedence,
+        right: Precedence,
         right_symbols: &[Symbol],
     ) -> Ordering {
         let precedence_entry_matches =
-            |entry: &PrecedenceEntry, precedence: &Precedence, symbols: &[Symbol]| -> bool {
+            |entry: &PrecedenceEntry, precedence: Precedence, symbols: &[Symbol]| -> bool {
                 match entry {
                     PrecedenceEntry::Name(n) => {
                         if let Precedence::Name(p) = precedence {
-                            n == p
+                            *n == p
                         } else {
                             false
                         }
                     }
                     PrecedenceEntry::Symbol(n) => symbols
                         .iter()
-                        .any(|s| &grammar.variables[s.index].name == n),
+                        .any(|s| &grammar.variables[s.index as usize].name == n),
                 }
             };
 
         match (left, right) {
             // Integer precedences can be compared to other integer precedences,
             // and to the default precedence, which is zero.
-            (Precedence::Integer(l), Precedence::Integer(r)) if *l != 0 || *r != 0 => l.cmp(r),
-            (Precedence::Integer(l), Precedence::None) if *l != 0 => l.cmp(&0),
-            (Precedence::None, Precedence::Integer(r)) if *r != 0 => 0.cmp(r),
+            (Precedence::Integer(l), Precedence::Integer(r)) if l != 0 || r != 0 => l.cmp(&r),
+            (Precedence::Integer(l), Precedence::None) if l != 0 => l.cmp(&0),
+            (Precedence::None, Precedence::Integer(r)) if r != 0 => 0.cmp(&r),
 
             // Named precedences can be compared to other named precedences.
             _ => grammar
@@ -1038,7 +1159,7 @@ impl<'a> ParseTableBuilder<'a> {
             .iter()
             .filter_map(|ParseItemSetEntry { item, .. }| {
                 let variable_index = item.variable_index as usize;
-                if item.symbol() == Some(symbol)
+                if item.symbol(self.syntax_grammar) == Some(symbol)
                     && !self.syntax_grammar.variables[variable_index].is_auxiliary()
                 {
                     Some(Symbol::non_terminal(variable_index))
@@ -1054,37 +1175,46 @@ impl<'a> ParseTableBuilder<'a> {
     }
 
     fn get_production_id(&mut self, item: &ParseItem) -> ProductionInfoId {
+        debug_assert_ne!(item.prod_id, START_PRODUCTION_ID);
+        if let Some(id) = self.production_info_ids_by_prod_id[item.prod_id as usize] {
+            return id;
+        }
         let mut production_info = ProductionInfo {
             alias_sequence: Vec::new(),
             field_map: BTreeMap::new(),
         };
 
-        for (i, step) in item.production.steps.iter().enumerate() {
-            production_info.alias_sequence.push(step.alias.clone());
-            if let Some(field_name) = &step.field_name {
+        for (i, step) in item
+            .production(self.syntax_grammar)
+            .steps
+            .iter()
+            .enumerate()
+        {
+            production_info.alias_sequence.push(step.alias());
+            if let Some(field_name) = step.field() {
                 production_info
                     .field_map
-                    .entry(field_name.clone())
+                    .entry(field_name)
                     .or_default()
                     .push(FieldLocation {
-                        index: i,
+                        index: i as u32,
                         inherited: false,
                     });
             }
 
-            if step.symbol.kind == SymbolType::NonTerminal
-                && !self.syntax_grammar.variables[step.symbol.index]
+            if step.symbol().kind == SymbolType::NonTerminal
+                && !self.syntax_grammar.variables[step.symbol().index as usize]
                     .kind
                     .is_visible()
             {
-                let info = &self.variable_info[step.symbol.index];
-                for field_name in info.fields.keys() {
+                let info = &self.variable_info[step.symbol().index as usize];
+                for &field_name in info.fields.keys() {
                     production_info
                         .field_map
-                        .entry(field_name.clone())
+                        .entry(field_name)
                         .or_default()
                         .push(FieldLocation {
-                            index: i,
+                            index: i as u32,
                             inherited: true,
                         });
                 }
@@ -1095,11 +1225,14 @@ impl<'a> ParseTableBuilder<'a> {
             production_info.alias_sequence.pop();
         }
 
-        if item.production.steps.len() > self.parse_table.max_aliased_production_length {
-            self.parse_table.max_aliased_production_length = item.production.steps.len();
+        if item.production(self.syntax_grammar).steps.len()
+            > self.parse_table.max_aliased_production_length
+        {
+            self.parse_table.max_aliased_production_length =
+                item.production(self.syntax_grammar).steps.len();
         }
 
-        if let Some(index) = self
+        let id = if let Some(index) = self
             .parse_table
             .production_infos
             .iter()
@@ -1109,22 +1242,28 @@ impl<'a> ParseTableBuilder<'a> {
         } else {
             self.parse_table.production_infos.push(production_info);
             self.parse_table.production_infos.len() - 1
-        }
+        } as ProductionInfoId;
+        self.production_info_ids_by_prod_id[item.prod_id as usize] = Some(id);
+        id
     }
 
-    fn symbol_name(&self, symbol: &Symbol) -> String {
+    fn symbol_name(&self, symbol: Symbol) -> String {
         match symbol.kind {
             SymbolType::End | SymbolType::EndOfNonTerminalExtra => "EOF".to_string(),
-            SymbolType::External => self.syntax_grammar.external_tokens[symbol.index]
-                .name
-                .clone(),
-            SymbolType::NonTerminal => self.syntax_grammar.variables[symbol.index].name.clone(),
+            SymbolType::External => self
+                .str_pool
+                .resolve(self.syntax_grammar.external_tokens[symbol.index as usize].name)
+                .to_string(),
+            SymbolType::NonTerminal => self
+                .str_pool
+                .resolve(self.syntax_grammar.variables[symbol.index as usize].name)
+                .to_string(),
             SymbolType::Terminal => {
-                let variable = &self.lexical_grammar.variables[symbol.index];
+                let variable = &self.lexical_grammar.variables[symbol.index as usize];
                 if variable.kind == VariableType::Named {
-                    variable.name.clone()
+                    self.str_pool.resolve(variable.name).to_string()
                 } else {
-                    format!("'{}'", variable.name)
+                    format!("'{}'", self.str_pool.resolve(variable.name))
                 }
             }
         }
@@ -1136,12 +1275,15 @@ pub fn build_parse_table<'a>(
     lexical_grammar: &'a LexicalGrammar,
     item_set_builder: ParseItemSetBuilder<'a>,
     variable_info: &'a [VariableInfo],
-) -> BuildTableResult<(ParseTable, Vec<ParseStateInfo<'a>>)> {
+    str_pool: &'a StrPool,
+    diagnostics: &mut Vec<Diagnostic>,
+) -> BuildTableResult<(ParseTable<ParseTableEntry>, ParseStateInfo<'a>)> {
     ParseTableBuilder::new(
         syntax_grammar,
         lexical_grammar,
         item_set_builder,
         variable_info,
+        str_pool,
     )
-    .build()
+    .build(diagnostics)
 }

crates/generate/src/build_tables/coincident_tokens.rs

@@ -1,29 +1,40 @@
-use std::fmt;
+use crate::{grammars::LexicalGrammar, rules::Symbol, strpool::StrPool, tables::ParseTable};
 
-use crate::{
-    grammars::LexicalGrammar,
-    rules::Symbol,
-    tables::{ParseStateId, ParseTable},
-};
-
-pub struct CoincidentTokenIndex<'a> {
-    entries: Vec<Vec<ParseStateId>>,
-    grammar: &'a LexicalGrammar,
+pub struct CoincidentTokenIndex {
+    /// Flat bitset for fast [`contains()`](Self::contains) checks. Indexed as `a * n + b`
+    /// (both `(a,b)` and `(b,a)` bits are set, so no min/max normalization needed).
+    contains_bits: Vec<u64>,
+    /// Flat biset for fast [`Self::all_coincident_states_have_word`] checks.
+    /// Bit `(a, b)` is set iff tokens `a` and `b` are coincident in some parse state
+    /// where the grammar's word token is _not_ a valid lookahead. Allows for fast
+    /// answers to "do all states containing this pair also contain the word token?".
+    without_word_bits: Vec<u64>,
+    /// Word-aligned per-row bitsets for vectorized intersection checks.
+    /// Row `a` spans `[a * row_words .. (a+1) * row_words]`.
+    /// Bit `b` is set iff tokens `a` and `b` are coincident in some parse state.
+    pub(crate) row_bits: Vec<u64>,
     n: usize,
 }
 
-impl<'a> CoincidentTokenIndex<'a> {
-    pub fn new(table: &ParseTable, lexical_grammar: &'a LexicalGrammar) -> Self {
+impl<'a> CoincidentTokenIndex {
+    #[must_use]
+    pub fn new<T>(
+        table: &ParseTable<T>,
+        lexical_grammar: &'a LexicalGrammar,
+        word_token: Option<Symbol>,
+    ) -> Self {
         let n = lexical_grammar.variables.len();
+        let row_words = n.div_ceil(64);
         let mut result = Self {
             n,
-            grammar: lexical_grammar,
-            entries: vec![Vec::new(); n * n],
+            contains_bits: vec![0u64; (n * n).div_ceil(64)],
+            without_word_bits: vec![0u64; (n * n).div_ceil(64)],
+            row_bits: vec![0u64; n * row_words],
         };
         // Pre-collect terminal indices up front rather than continuously recomputing within the
         // loop below.
         let mut terminal_indices = Vec::new();
-        for (i, state) in table.states.iter().enumerate() {
+        for state in &table.states {
             terminal_indices.clear();
             terminal_indices.extend(
                 state
@@ -32,54 +43,64 @@ impl<'a> CoincidentTokenIndex<'a> {
                     .filter(|s| s.is_terminal())
                     .map(|s| s.index),
             );
-            for (j, &a) in terminal_indices.iter().enumerate() {
-                for &b in &terminal_indices[j..] {
-                    let index = result.index(a, b);
-                    if result.entries[index].last().copied() != Some(i) {
-                        result.entries[index].push(i);
+            let has_word = word_token.is_some_and(|w| state.terminal_entries.contains_key(&w));
+            for (i, &a) in terminal_indices.iter().enumerate() {
+                for &b in &terminal_indices[i..] {
+                    let (a, b) = (a as usize, b as usize);
+                    // Set both (a,b) and (b,a) bits so `contains()` needs
+                    // no min/max normalization.
+                    let ab = a * n + b;
+                    let ba = b * n + a;
+                    result.contains_bits[ab / 64] |= 1u64 << (ab % 64);
+                    result.contains_bits[ba / 64] |= 1u64 << (ba % 64);
+                    if !has_word {
+                        result.without_word_bits[ab / 64] |= 1u64 << (ab % 64);
+                        result.without_word_bits[ba / 64] |= 1u64 << (ba % 64);
                     }
+                    // Also populate the word-aligned row bitsets.
+                    result.row_bits[a * row_words + b / 64] |= 1u64 << (b % 64);
+                    result.row_bits[b * row_words + a / 64] |= 1u64 << (a % 64);
                 }
             }
         }
         result
     }
 
-    pub fn states_with(&self, a: Symbol, b: Symbol) -> &[ParseStateId] {
-        &self.entries[self.index(a.index, b.index)]
-    }
-
-    pub fn contains(&self, a: Symbol, b: Symbol) -> bool {
-        !self.entries[self.index(a.index, b.index)].is_empty()
+    #[must_use]
+    pub fn all_coincident_states_have_word(&self, a: Symbol, b: Symbol) -> bool {
+        let bit_index = a.index as usize * self.n + b.index as usize;
+        self.without_word_bits[bit_index / 64] & (1u64 << (bit_index % 64)) == 0
     }
 
     #[must_use]
-    const fn index(&self, a: usize, b: usize) -> usize {
-        if a < b {
-            a * self.n + b
-        } else {
-            b * self.n + a
-        }
+    pub fn contains(&self, a: Symbol, b: Symbol) -> bool {
+        let bit_index = a.index as usize * self.n + b.index as usize;
+        self.contains_bits[bit_index / 64] & (1u64 << (bit_index % 64)) != 0
     }
 }
 
-impl fmt::Debug for CoincidentTokenIndex<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "CoincidentTokenIndex {{")?;
+#[expect(dead_code, reason = "Debugging aid")]
+pub struct CoincidentTokenIndexDisplay<'a>(CoincidentTokenIndex, &'a LexicalGrammar, StrPool);
 
-        writeln!(f, "  entries: {{")?;
-        for i in 0..self.n {
-            writeln!(f, "    {}: {{", self.grammar.variables[i].name)?;
-            for j in 0..self.n {
+impl std::fmt::Debug for CoincidentTokenIndexDisplay<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "CoincidentTokenIndex {{")?;
+        for i in 0..self.0.n {
+            let mut coincident = Vec::new();
+            for j in 0..self.0.n {
+                if self.0.contains(Symbol::terminal(i), Symbol::terminal(j)) {
+                    coincident.push(self.2.resolve(self.1.variables[j].name));
+                }
+            }
+            if !coincident.is_empty() {
                 writeln!(
                     f,
-                    "      {}: {:?},",
-                    self.grammar.variables[j].name,
-                    self.entries[self.index(i, j)].len()
+                    "  {}: {:?},",
+                    self.2.resolve(self.1.variables[i].name),
+                    coincident
                 )?;
             }
-            writeln!(f, "    }},")?;
         }
-        write!(f, "  }},")?;
         write!(f, "}}")?;
         Ok(())
     }

crates/generate/src/build_tables/item.rs

@@ -1,32 +1,247 @@
 use std::{
     cmp::Ordering,
     fmt,
-    hash::{Hash, Hasher},
-    sync::LazyLock,
+    hash::{BuildHasherDefault, Hash, Hasher},
 };
 
+use indexmap::IndexSet;
+use rustc_hash::{FxHashMap, FxHasher};
+
 use crate::{
-    grammars::{
-        LexicalGrammar, NO_RESERVED_WORDS, Production, ProductionStep, ReservedWordSetId,
-        SyntaxGrammar,
-    },
+    grammars::{LexicalGrammar, ProdRef, ProductionStep, ReservedWordSetId, SyntaxGrammar},
     rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
+    strpool::StrPool,
 };
 
-static START_PRODUCTION: LazyLock<Production> = LazyLock::new(|| Production {
-    dynamic_precedence: 0,
-    steps: vec![ProductionStep {
-        symbol: Symbol {
-            index: 0,
-            kind: SymbolType::NonTerminal,
-        },
-        precedence: Precedence::None,
-        associativity: None,
-        alias: None,
-        field_name: None,
-        reserved_word_set_id: NO_RESERVED_WORDS,
-    }],
-});
+const START_STEPS: [ProductionStep; 1] = [ProductionStep {
+    sym_index: 0,
+    prec_val: 0,
+    alias: 0,
+    field: 0,
+    reserved: ProductionStep::NO_RESERVED_WORDS,
+    flags: SymbolType::NonTerminal as u8,
+}];
+
+pub const START_PRODUCTION_ID: u32 = u32::MAX;
+
+const fn start_production() -> ProdRef<'static> {
+    ProdRef {
+        steps: &START_STEPS,
+        dynamic_precedence: 0,
+        requires_eof_lookahead: false,
+    }
+}
+
+/// Precomputed identity keys for one `(production, dot)` pair.
+///
+/// `cmp` is the rank of the content tuple `Ord` compared (dynamic precedence,
+/// `requires_eof_lookahead`, length, precedence/associativity at the dot, then
+/// completed steps' aliases and fields and remaining steps in full). Equal ranks
+/// hold _exactly_ when the tuple is equal, so it doubles as the equality class
+/// for items without preceding inherited fields. `eq_with_syms` subdivides `cmp`
+/// by the completed steps' symbols, which participate in equality only when
+/// `has_preceding_inherited_fields` is set.
+#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
+pub struct DotKeys {
+    pub cmp: u32,
+    pub eq_with_syms: u32,
+}
+
+/// Identity keys for every `(production, dot)` in a grammar (all grammar productions,
+/// every inlined production, and the augmented start production).
+pub struct ItemKeyMap {
+    keys: Vec<Box<[DotKeys]>>,
+    start: Box<[DotKeys]>,
+}
+
+impl ItemKeyMap {
+    pub fn new(grammar: &SyntaxGrammar, str_pool: &StrPool) -> Self {
+        let prod = |slot: usize| -> ProdRef {
+            if slot == 0 {
+                start_production()
+            } else {
+                grammar.production(slot as u32 - 1)
+            }
+        };
+        let slot_count = grammar.productions.len() + 1;
+
+        let mut contents: Vec<(u32, u32)> = Vec::with_capacity(slot_count);
+        for pi in 0..slot_count {
+            for dot in 0..=prod(pi).steps.len() {
+                contents.push((pi as u32, dot as u32));
+            }
+        }
+        let content = |&(pi, dot): &(u32, u32)| ItemContent {
+            production: prod(pi as usize),
+            dot: dot as usize,
+            str_pool,
+        };
+        contents.sort_unstable_by(|a, b| content(a).cmp(&content(b)));
+
+        let mut slot_keys: Vec<Box<[DotKeys]>> = (0..slot_count)
+            .map(|pi| vec![DotKeys::default(); prod(pi).steps.len() + 1].into_boxed_slice())
+            .collect::<Vec<_>>();
+        // Dense ids in sorted order: equal content shares an id, distinct content gets the next up
+        let mut cmp_id = 0;
+        let mut prev = None;
+        for &(pi, dot) in &contents {
+            if let Some(p) = prev
+                && content(&p) != content(&(pi, dot))
+            {
+                cmp_id += 1;
+            }
+            slot_keys[pi as usize][dot as usize].cmp = cmp_id;
+            prev = Some((pi, dot));
+        }
+
+        // Refine each cmp class by preceding symbols:  read only under
+        // `has_preceding_inherited_fields`.
+        let mut sym_classes = FxHashMap::default();
+        for &(pi, dot) in &contents {
+            let syms = prod(pi as usize).steps[..dot as usize]
+                .iter()
+                .map(|s| s.symbol())
+                .collect::<Vec<_>>();
+            let next = sym_classes.len() as u32;
+            let keys = &mut slot_keys[pi as usize][dot as usize];
+            keys.eq_with_syms = *sym_classes.entry((keys.cmp, syms)).or_insert(next);
+        }
+
+        let start = slot_keys.remove(0);
+        Self {
+            keys: slot_keys,
+            start,
+        }
+    }
+
+    /// The keys slice (indexed by `dot`) for a production of this grammar.
+    pub fn keys_for(&self, id: u32) -> &[DotKeys] {
+        if id == START_PRODUCTION_ID {
+            &self.start
+        } else {
+            &self.keys[id as usize]
+        }
+    }
+
+    pub fn start_keys(&self) -> &[DotKeys] {
+        &self.start
+    }
+}
+
+/// The content tuple that `ParseItem`'s `Ord` (and flagless `Eq`) observe, evaluated on
+/// `(production, dot)` directly. Ranking the set of all keys by this order makes
+/// the dense `cmp` order-preserving. Item ordering only ever compares same-dot pairs,
+/// and the dot comparison keeps the order total across dots so ranks are well defined.
+struct ItemContent<'a> {
+    production: ProdRef<'a>,
+    dot: usize,
+    str_pool: &'a StrPool,
+}
+
+impl Eq for ItemContent<'_> {}
+
+impl ItemContent<'_> {
+    const fn prec(&self) -> Precedence {
+        if self.dot > 0 {
+            self.production.steps[self.dot - 1].precedence()
+        } else {
+            Precedence::None
+        }
+    }
+
+    const fn assoc(&self) -> Option<Associativity> {
+        if self.dot > 0 {
+            self.production.steps[self.dot - 1].associativity()
+        } else {
+            None
+        }
+    }
+
+    fn prec_cmp(&self, a: Precedence, b: Precedence) -> Ordering {
+        match (a, b) {
+            (Precedence::None, Precedence::None) => Ordering::Equal,
+            (Precedence::Integer(a), Precedence::Integer(b)) => a.cmp(&b),
+            (Precedence::Name(a), Precedence::Name(b)) => {
+                self.str_pool.resolve(a).cmp(self.str_pool.resolve(b))
+            }
+            (Precedence::None, _) | (Precedence::Integer(_), Precedence::Name(_)) => Ordering::Less,
+            (_, Precedence::None) | (Precedence::Name(_), Precedence::Integer(_)) => {
+                Ordering::Greater
+            }
+        }
+    }
+
+    fn alias_cmp(&self, a: ProductionStep, b: ProductionStep) -> Ordering {
+        let key = |step: ProductionStep| {
+            step.alias()
+                .map(|alias| (self.str_pool.resolve(alias.value), alias.is_named))
+        };
+        key(a).cmp(&key(b))
+    }
+
+    fn field_cmp(&self, a: ProductionStep, b: ProductionStep) -> Ordering {
+        let key = |step: ProductionStep| step.field().map(|field| self.str_pool.resolve(field));
+        key(a).cmp(&key(b))
+    }
+
+    fn step_cmp(&self, a: ProductionStep, b: ProductionStep) -> Ordering {
+        a.symbol()
+            .cmp(&b.symbol())
+            .then_with(|| self.prec_cmp(a.precedence(), b.precedence()))
+            .then_with(|| a.associativity().cmp(&b.associativity()))
+            .then_with(|| self.alias_cmp(a, b))
+            .then_with(|| self.field_cmp(a, b))
+            .then_with(|| a.reserved.cmp(&b.reserved))
+    }
+}
+
+impl Ord for ItemContent<'_> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.production
+            .dynamic_precedence
+            .cmp(&other.production.dynamic_precedence)
+            .then_with(|| {
+                self.production
+                    .requires_eof_lookahead
+                    .cmp(&other.production.requires_eof_lookahead)
+            })
+            .then_with(|| {
+                self.production
+                    .steps
+                    .len()
+                    .cmp(&other.production.steps.len())
+            })
+            .then_with(|| self.prec_cmp(self.prec(), other.prec()))
+            .then_with(|| self.assoc().cmp(&other.assoc()))
+            .then_with(|| self.dot.cmp(&other.dot))
+            .then_with(|| {
+                let steps = self.production.steps.iter().zip(other.production.steps);
+                for (i, (&sa, &sb)) in steps.enumerate() {
+                    let o = if i < self.dot {
+                        self.alias_cmp(sa, sb).then_with(|| self.field_cmp(sa, sb))
+                    } else {
+                        self.step_cmp(sa, sb)
+                    };
+                    if o != Ordering::Equal {
+                        return o;
+                    }
+                }
+                Ordering::Equal
+            })
+    }
+}
+
+impl PartialOrd for ItemContent<'_> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for ItemContent<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == Ordering::Equal
+    }
+}
 
 /// A [`ParseItem`] represents an in-progress match of a single production in a grammar.
 #[derive(Clone, Copy, Debug)]
@@ -35,8 +250,10 @@ pub struct ParseItem<'a> {
     pub variable_index: u32,
     /// The number of symbols that have already been matched.
     pub step_index: u32,
-    /// The production being matched.
-    pub production: &'a Production,
+    /// The id of the production being matched.
+    pub prod_id: u32,
+    /// The `production`'s identity keys, indexed by `step_index`.
+    pub keys: &'a [DotKeys],
     /// A boolean indicating whether any of the already-matched children were
     /// hidden nodes and had fields. Ordinarily, a parse item's behavior is not
     /// affected by the symbols of its preceding children; it only needs to
@@ -53,9 +270,97 @@ pub struct ParseItem<'a> {
     pub has_preceding_inherited_fields: bool,
 }
 
-/// A [`ParseItemSet`] represents a set of in-progress matches of productions in a
-/// grammar, and for each in-progress match, a set of "lookaheads" - tokens that
-/// are allowed to *follow* the in-progress rule. This object corresponds directly
+/// Interned lookahead set. An index into a [`LookaheadPool`]. Ids are canonical,
+/// so id equality is equivalent to underlying [`TokenSet`] equality.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct LookaheadSetId(u32);
+
+impl Default for LookaheadSetId {
+    fn default() -> Self {
+        LookaheadSetPool::EMPTY
+    }
+}
+
+/// Interner for lookahead [`TokenSet`]s. Item-set entries store ids.
+pub struct LookaheadSetPool {
+    sets: IndexSet<TokenSet, BuildHasherDefault<FxHasher>>,
+    union_memo: FxHashMap<(LookaheadSetId, LookaheadSetId), LookaheadSetId>,
+    insert_memo: FxHashMap<(LookaheadSetId, Symbol), LookaheadSetId>,
+}
+
+impl LookaheadSetPool {
+    pub const EMPTY: LookaheadSetId = LookaheadSetId(0);
+
+    #[must_use]
+    pub fn new() -> Self {
+        let mut pool = Self {
+            sets: IndexSet::default(),
+            union_memo: FxHashMap::default(),
+            insert_memo: FxHashMap::default(),
+        };
+        let empty = pool.intern(TokenSet::new());
+        debug_assert_eq!(empty, Self::EMPTY);
+        pool
+    }
+
+    #[must_use]
+    pub fn get(&self, id: LookaheadSetId) -> &TokenSet {
+        &self.sets[id.0 as usize]
+    }
+
+    pub fn intern(&mut self, set: TokenSet) -> LookaheadSetId {
+        let (index, _) = self.sets.insert_full(set);
+        LookaheadSetId(index as u32)
+    }
+
+    pub fn intern_ref(&mut self, set: &TokenSet) -> LookaheadSetId {
+        if let Some(index) = self.sets.get_index_of(set) {
+            return LookaheadSetId(index as u32);
+        }
+        self.intern(set.clone())
+    }
+
+    pub fn singleton(&mut self, symbol: Symbol) -> LookaheadSetId {
+        self.insert(Self::EMPTY, symbol)
+    }
+
+    pub fn insert(&mut self, id: LookaheadSetId, symbol: Symbol) -> LookaheadSetId {
+        if self.get(id).contains(symbol) {
+            return id;
+        }
+        if let Some(&result) = self.insert_memo.get(&(id, symbol)) {
+            return result;
+        }
+        let mut set = self.get(id).clone();
+        set.insert(symbol);
+        let result = self.intern(set);
+        self.insert_memo.insert((id, symbol), result);
+        result
+    }
+
+    pub fn union(&mut self, left: LookaheadSetId, right: LookaheadSetId) -> LookaheadSetId {
+        if left == right || right == Self::EMPTY {
+            return left;
+        }
+        if left == Self::EMPTY {
+            return right;
+        }
+        let key = (left.min(right), left.max(right));
+        if let Some(&result) = self.union_memo.get(&key) {
+            return result;
+        }
+        let mut set = self.get(key.0).clone();
+        set.insert_all(self.get(key.1));
+        let result = self.intern(set);
+        self.union_memo.insert(key, result);
+        result
+    }
+}
+
+/// Represents a set of in-progress matches of productions in a grammar.
+///
+/// For each in-progress match, a set of "lookaheads" (tokens that are allowed to
+/// *follow* the in-progress rule) are included. This object corresponds directly
 /// to a state in the final parse table.
 #[derive(Clone, Debug, PartialEq, Eq, Default)]
 pub struct ParseItemSet<'a> {
@@ -65,7 +370,7 @@ pub struct ParseItemSet<'a> {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ParseItemSetEntry<'a> {
     pub item: ParseItem<'a>,
-    pub lookaheads: TokenSet,
+    pub lookaheads: LookaheadSetId,
     pub following_reserved_word_set: ReservedWordSetId,
 }
 
@@ -80,50 +385,75 @@ pub struct ParseItemDisplay<'a>(
     pub &'a ParseItem<'a>,
     pub &'a SyntaxGrammar,
     pub &'a LexicalGrammar,
+    pub &'a StrPool,
 );
 
 pub struct TokenSetDisplay<'a>(
     pub &'a TokenSet,
     pub &'a SyntaxGrammar,
     pub &'a LexicalGrammar,
+    pub &'a StrPool,
 );
 
 pub struct ParseItemSetDisplay<'a>(
     pub &'a ParseItemSet<'a>,
     pub &'a SyntaxGrammar,
     pub &'a LexicalGrammar,
+    pub &'a StrPool,
+    pub &'a LookaheadSetPool,
 );
 
 impl<'a> ParseItem<'a> {
-    pub fn start() -> Self {
+    #[must_use]
+    pub fn start(key_map: &'a ItemKeyMap) -> Self {
         ParseItem {
             variable_index: u32::MAX,
-            production: &START_PRODUCTION,
+            prod_id: START_PRODUCTION_ID,
+            keys: key_map.start_keys(),
             step_index: 0,
             has_preceding_inherited_fields: false,
         }
     }
 
-    pub fn step(&self) -> Option<&'a ProductionStep> {
-        self.production.steps.get(self.step_index as usize)
+    /// The production being matched.
+    #[must_use]
+    pub fn production<'g>(&self, grammar: &'g SyntaxGrammar) -> ProdRef<'g> {
+        if self.prod_id == START_PRODUCTION_ID {
+            start_production()
+        } else {
+            grammar.production(self.prod_id)
+        }
     }
 
-    pub fn symbol(&self) -> Option<Symbol> {
-        self.step().map(|step| step.symbol)
+    #[must_use]
+    pub fn step(&self, grammar: &SyntaxGrammar) -> Option<ProductionStep> {
+        self.production(grammar)
+            .steps
+            .get(self.step_index as usize)
+            .copied()
     }
 
-    pub fn associativity(&self) -> Option<Associativity> {
-        self.prev_step().and_then(|step| step.associativity)
+    #[must_use]
+    pub fn symbol(&self, grammar: &SyntaxGrammar) -> Option<Symbol> {
+        self.step(grammar).map(ProductionStep::symbol)
     }
 
-    pub fn precedence(&self) -> &Precedence {
-        self.prev_step()
-            .map_or(&Precedence::None, |step| &step.precedence)
+    #[must_use]
+    pub fn associativity(&self, grammar: &SyntaxGrammar) -> Option<Associativity> {
+        self.prev_step(grammar)
+            .and_then(ProductionStep::associativity)
     }
 
-    pub fn prev_step(&self) -> Option<&'a ProductionStep> {
+    #[must_use]
+    pub fn precedence(&self, grammar: &SyntaxGrammar) -> Precedence {
+        self.prev_step(grammar)
+            .map_or(Precedence::None, ProductionStep::precedence)
+    }
+
+    #[must_use]
+    pub fn prev_step(&self, grammar: &SyntaxGrammar) -> Option<ProductionStep> {
         if self.step_index > 0 {
-            Some(&self.production.steps[self.step_index as usize - 1])
+            Some(self.production(grammar).steps[self.step_index as usize - 1])
         } else {
             None
         }
@@ -131,7 +461,7 @@ impl<'a> ParseItem<'a> {
 
     #[must_use]
     pub const fn is_done(&self) -> bool {
-        self.step_index as usize == self.production.steps.len()
+        self.step_index as usize + 1 == self.keys.len()
     }
 
     #[must_use]
@@ -144,7 +474,8 @@ impl<'a> ParseItem<'a> {
     pub const fn successor(&self) -> Self {
         ParseItem {
             variable_index: self.variable_index,
-            production: self.production,
+            prod_id: self.prod_id,
+            keys: self.keys,
             step_index: self.step_index + 1,
             has_preceding_inherited_fields: self.has_preceding_inherited_fields,
         }
@@ -152,11 +483,19 @@ impl<'a> ParseItem<'a> {
 
     /// Create an item identical to this one, but with a different production.
     /// This is used when dynamically "inlining" certain symbols in a production.
-    pub const fn substitute_production(&self, production: &'a Production) -> Self {
+    #[must_use]
+    pub const fn substitute_production(&self, prod_id: u32, keys: &'a [DotKeys]) -> Self {
         let mut result = *self;
-        result.production = production;
+        result.prod_id = prod_id;
+        result.keys = keys;
         result
     }
+
+    /// This item's identity keys at the current dot.
+    #[must_use]
+    const fn dot_keys(&self) -> DotKeys {
+        self.keys[self.step_index as usize]
+    }
 }
 
 impl<'a> ParseItemSet<'a> {
@@ -168,7 +507,7 @@ impl<'a> ParseItemSet<'a> {
                     i,
                     ParseItemSetEntry {
                         item,
-                        lookaheads: TokenSet::new(),
+                        lookaheads: LookaheadSetPool::EMPTY,
                         following_reserved_word_set: ReservedWordSetId::default(),
                     },
                 );
@@ -178,6 +517,7 @@ impl<'a> ParseItemSet<'a> {
         }
     }
 
+    #[must_use]
     pub fn core(&self) -> ParseItemSetCore<'a> {
         ParseItemSetCore {
             entries: self.entries.iter().map(|e| e.item).collect(),
@@ -185,6 +525,14 @@ impl<'a> ParseItemSet<'a> {
     }
 }
 
+pub fn prec_display(prec: Precedence, str_pool: &StrPool) -> String {
+    match prec {
+        Precedence::None => "none".to_string(),
+        Precedence::Integer(i) => i.to_string(),
+        Precedence::Name(sid) => format!("'{}'", str_pool.resolve(sid)),
+    }
+}
+
 impl fmt::Display for ParseItemDisplay<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
         if self.0.is_augmented() {
@@ -193,60 +541,76 @@ impl fmt::Display for ParseItemDisplay<'_> {
             write!(
                 f,
                 "{} →",
-                self.1.variables[self.0.variable_index as usize].name
+                self.3
+                    .resolve(self.1.variables[self.0.variable_index as usize].name)
             )?;
         }
 
-        for (i, step) in self.0.production.steps.iter().enumerate() {
+        let production = self.0.production(self.1);
+        for (i, step) in production.steps.iter().enumerate() {
+            let symbol = step.symbol();
             if i == self.0.step_index as usize {
                 write!(f, " •")?;
-                if !step.precedence.is_none()
-                    || step.associativity.is_some()
-                    || step.reserved_word_set_id != ReservedWordSetId::default()
+                if step.precedence() != Precedence::None
+                    || step.associativity().is_some()
+                    || step.reserved != 0
                 {
                     write!(f, " (")?;
-                    if !step.precedence.is_none() {
-                        write!(f, " {}", step.precedence)?;
+                    if step.precedence() != Precedence::None {
+                        write!(f, " {}", prec_display(step.precedence(), self.3))?;
                     }
-                    if let Some(associativity) = step.associativity {
+                    if let Some(associativity) = step.associativity() {
                         write!(f, " {associativity:?}")?;
                     }
-                    if step.reserved_word_set_id != ReservedWordSetId::default() {
-                        write!(f, "reserved: {}", step.reserved_word_set_id)?;
+                    if step.reserved != 0 {
+                        write!(f, "reserved: {}", step.reserved)?;
                     }
                     write!(f, " )")?;
                 }
             }
 
             write!(f, " ")?;
-            if step.symbol.is_terminal() {
-                if let Some(variable) = self.2.variables.get(step.symbol.index) {
-                    write!(f, "{}", variable.name)?;
+            if symbol.is_terminal() {
+                if let Some(variable) = self.2.variables.get(symbol.index as usize) {
+                    write!(f, "{}", self.3.resolve(variable.name))?;
                 } else {
-                    write!(f, "terminal-{}", step.symbol.index)?;
+                    write!(f, "terminal-{}", symbol.index)?;
                 }
-            } else if step.symbol.is_external() {
-                write!(f, "{}", self.1.external_tokens[step.symbol.index].name)?;
+            } else if symbol.is_external() {
+                write!(
+                    f,
+                    "{}",
+                    self.3
+                        .resolve(self.1.external_tokens[symbol.index as usize].name)
+                )?;
             } else {
-                write!(f, "{}", self.1.variables[step.symbol.index].name)?;
+                write!(
+                    f,
+                    "{}",
+                    self.3.resolve(self.1.variables[symbol.index as usize].name)
+                )?;
             }
 
-            if let Some(alias) = &step.alias {
-                write!(f, "@{}", alias.value)?;
+            if let Some(alias) = &step.alias() {
+                write!(f, "@{}", self.3.resolve(alias.value))?;
             }
         }
 
         if self.0.is_done() {
             write!(f, " •")?;
-            if let Some(step) = self.0.production.steps.last() {
-                if let Some(associativity) = step.associativity {
-                    if step.precedence.is_none() {
+            if let Some(&step) = production.steps.last() {
+                if let Some(associativity) = step.associativity() {
+                    if step.precedence() == Precedence::None {
                         write!(f, " ({associativity:?})")?;
                     } else {
-                        write!(f, " ({} {associativity:?})", step.precedence)?;
+                        write!(
+                            f,
+                            " ({} {associativity:?})",
+                            prec_display(step.precedence(), self.3)
+                        )?;
                     }
-                } else if !step.precedence.is_none() {
-                    write!(f, " ({})", step.precedence)?;
+                } else if step.precedence() != Precedence::None {
+                    write!(f, " ({})", prec_display(step.precedence(), self.3))?;
                 }
             }
         }
@@ -290,15 +654,28 @@ impl fmt::Display for TokenSetDisplay<'_> {
             }
 
             if symbol.is_terminal() {
-                if let Some(variable) = self.2.variables.get(symbol.index) {
-                    write!(f, "{}", display_variable_name(&variable.name))?;
+                if let Some(variable) = self.2.variables.get(symbol.index as usize) {
+                    write!(
+                        f,
+                        "{}",
+                        display_variable_name(self.3.resolve(variable.name))
+                    )?;
                 } else {
                     write!(f, "terminal-{}", symbol.index)?;
                 }
             } else if symbol.is_external() {
-                write!(f, "{}", self.1.external_tokens[symbol.index].name)?;
+                write!(
+                    f,
+                    "{}",
+                    self.3
+                        .resolve(self.1.external_tokens[symbol.index as usize].name)
+                )?;
             } else {
-                write!(f, "{}", self.1.variables[symbol.index].name)?;
+                write!(
+                    f,
+                    "{}",
+                    self.3.resolve(self.1.variables[symbol.index as usize].name)
+                )?;
             }
         }
         write!(f, "]")?;
@@ -312,8 +689,8 @@ impl fmt::Display for ParseItemSetDisplay<'_> {
             write!(
                 f,
                 "{}\t{}",
-                ParseItemDisplay(&entry.item, self.1, self.2),
-                TokenSetDisplay(&entry.lookaheads, self.1, self.2),
+                ParseItemDisplay(&entry.item, self.1, self.2, self.3),
+                TokenSetDisplay(self.4.get(entry.lookaheads), self.1, self.2, self.3),
             )?;
             if entry.following_reserved_word_set != ReservedWordSetId::default() {
                 write!(
@@ -332,11 +709,6 @@ impl Hash for ParseItem<'_> {
     fn hash<H: Hasher>(&self, hasher: &mut H) {
         hasher.write_u32(self.variable_index);
         hasher.write_u32(self.step_index);
-        hasher.write_i32(self.production.dynamic_precedence);
-        hasher.write_usize(self.production.steps.len());
-        hasher.write_i32(i32::from(self.has_preceding_inherited_fields));
-        self.precedence().hash(hasher);
-        self.associativity().hash(hasher);
 
         // The already-matched children don't play any role in the parse state for
         // this item, unless any of the following are true:
@@ -344,15 +716,14 @@ impl Hash for ParseItem<'_> {
         //   * the children have aliases
         //   * the children are hidden and represent rules that have fields.
         // See the docs for `has_preceding_inherited_fields`.
-        for step in &self.production.steps[0..self.step_index as usize] {
-            step.alias.hash(hasher);
-            step.field_name.hash(hasher);
-            if self.has_preceding_inherited_fields {
-                step.symbol.hash(hasher);
-            }
-        }
-        for step in &self.production.steps[self.step_index as usize..] {
-            step.hash(hasher);
+        // Preceding symbols participate only in `eq_with_syms`.
+        let keys = self.dot_keys();
+        if self.has_preceding_inherited_fields {
+            hasher.write_u8(1);
+            hasher.write_u32(keys.eq_with_syms);
+        } else {
+            hasher.write_u8(0);
+            hasher.write_u32(keys.cmp);
         }
     }
 }
@@ -362,36 +733,16 @@ impl PartialEq for ParseItem<'_> {
     fn eq(&self, other: &Self) -> bool {
         if self.variable_index != other.variable_index
             || self.step_index != other.step_index
-            || self.production.dynamic_precedence != other.production.dynamic_precedence
-            || self.production.steps.len() != other.production.steps.len()
-            || self.precedence() != other.precedence()
-            || self.associativity() != other.associativity()
             || self.has_preceding_inherited_fields != other.has_preceding_inherited_fields
         {
             return false;
         }
 
-        for (i, step) in self.production.steps.iter().enumerate() {
-            // See the previous comment (in the `Hash::hash` impl) regarding comparisons
-            // of parse items' already-completed steps.
-            if i < self.step_index as usize {
-                if step.alias != other.production.steps[i].alias {
-                    return false;
-                }
-                if step.field_name != other.production.steps[i].field_name {
-                    return false;
-                }
-                if self.has_preceding_inherited_fields
-                    && step.symbol != other.production.steps[i].symbol
-                {
-                    return false;
-                }
-            } else if *step != other.production.steps[i] {
-                return false;
-            }
+        if self.has_preceding_inherited_fields {
+            self.dot_keys().eq_with_syms == other.dot_keys().eq_with_syms
+        } else {
+            self.dot_keys().cmp == other.dot_keys().cmp
         }
-
-        true
     }
 }
 
@@ -401,37 +752,19 @@ impl Ord for ParseItem<'_> {
         self.step_index
             .cmp(&other.step_index)
             .then_with(|| self.variable_index.cmp(&other.variable_index))
+            .then_with(|| self.dot_keys().cmp.cmp(&other.dot_keys().cmp))
             .then_with(|| {
-                self.production
-                    .dynamic_precedence
-                    .cmp(&other.production.dynamic_precedence)
+                self.has_preceding_inherited_fields
+                    .cmp(&other.has_preceding_inherited_fields)
             })
             .then_with(|| {
-                self.production
-                    .steps
-                    .len()
-                    .cmp(&other.production.steps.len())
-            })
-            .then_with(|| self.precedence().cmp(other.precedence()))
-            .then_with(|| self.associativity().cmp(&other.associativity()))
-            .then_with(|| {
-                for (i, step) in self.production.steps.iter().enumerate() {
-                    // See the previous comment (in the `Hash::hash` impl) regarding comparisons
-                    // of parse items' already-completed steps.
-                    let o = if i < self.step_index as usize {
-                        step.alias
-                            .cmp(&other.production.steps[i].alias)
-                            .then_with(|| {
-                                step.field_name.cmp(&other.production.steps[i].field_name)
-                            })
-                    } else {
-                        step.cmp(&other.production.steps[i])
-                    };
-                    if o != Ordering::Equal {
-                        return o;
-                    }
+                if self.has_preceding_inherited_fields {
+                    self.dot_keys()
+                        .eq_with_syms
+                        .cmp(&other.dot_keys().eq_with_syms)
+                } else {
+                    Ordering::Equal
                 }
-                Ordering::Equal
             })
     }
 }

crates/generate/src/build_tables/item_set_builder.rs

@@ -1,20 +1,35 @@
-use std::{
-    collections::{HashMap, HashSet},
-    fmt,
-};
+use std::fmt;
 
-use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, ParseItemSetEntry, TokenSetDisplay};
+use rustc_hash::{FxHashMap, FxHashSet};
+
+use super::item::{
+    ItemKeyMap, ParseItem, ParseItemDisplay, ParseItemSet, ParseItemSetEntry, TokenSetDisplay,
+};
 use crate::{
+    build_tables::item::{LookaheadSetId, LookaheadSetPool},
     grammars::{InlinedProductionMap, LexicalGrammar, ReservedWordSetId, SyntaxGrammar},
     rules::{Symbol, SymbolType, TokenSet},
+    strpool::StrPool,
 };
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 struct TransitiveClosureAddition<'a> {
     item: ParseItem<'a>,
-    info: FollowSetInfo,
+    info: AdditionInfo,
 }
 
+/// [`FollowSetInfo`] with an interned lookahead set and word-token membership precomputed.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct AdditionInfo {
+    lookaheads: LookaheadSetId,
+    reserved_lookaheads: ReservedWordSetId,
+    propagates_lookaheads: bool,
+    contains_word: bool,
+}
+
+/// the mutable accumulator for a non-terminal's follow set. `lookaheads` is a
+/// working `TokenSet` that can be unioned incrementally during traversal, to be
+/// interned into [`AdditionInfo`] once complete.
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 struct FollowSetInfo {
     lookaheads: TokenSet,
@@ -24,11 +39,14 @@ struct FollowSetInfo {
 
 pub struct ParseItemSetBuilder<'a> {
     syntax_grammar: &'a SyntaxGrammar,
-    lexical_grammar: &'a LexicalGrammar,
-    first_sets: HashMap<Symbol, TokenSet>,
-    reserved_first_sets: HashMap<Symbol, ReservedWordSetId>,
-    last_sets: HashMap<Symbol, TokenSet>,
+    first_sets: FxHashMap<Symbol, TokenSet>,
+    /// Each FIRST set interned, for closure propagation by id
+    first_set_ids: FxHashMap<Symbol, LookaheadSetId>,
+    reserved_first_sets: FxHashMap<Symbol, ReservedWordSetId>,
+    last_sets: FxHashMap<Symbol, TokenSet>,
     inlines: &'a InlinedProductionMap,
+    pub key_map: &'a ItemKeyMap,
+    pub lookaheads: LookaheadSetPool,
     transitive_closure_additions: Vec<Vec<TransitiveClosureAddition<'a>>>,
 }
 
@@ -39,18 +57,22 @@ fn find_or_push<T: Eq>(vector: &mut Vec<T>, value: T) {
 }
 
 impl<'a> ParseItemSetBuilder<'a> {
+    #[must_use]
     pub fn new(
         syntax_grammar: &'a SyntaxGrammar,
         lexical_grammar: &'a LexicalGrammar,
         inlines: &'a InlinedProductionMap,
+        key_map: &'a ItemKeyMap,
     ) -> Self {
         let mut result = Self {
             syntax_grammar,
-            lexical_grammar,
-            first_sets: HashMap::new(),
-            reserved_first_sets: HashMap::new(),
-            last_sets: HashMap::new(),
+            first_sets: FxHashMap::default(),
+            first_set_ids: FxHashMap::default(),
+            reserved_first_sets: FxHashMap::default(),
+            last_sets: FxHashMap::default(),
             inlines,
+            key_map,
+            lookaheads: LookaheadSetPool::new(),
             transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()],
         };
 
@@ -92,7 +114,7 @@ impl<'a> ParseItemSetBuilder<'a> {
         // Rather than computing these sets using recursion, we use an explicit stack
         // called `symbols_to_process`.
         let mut symbols_to_process = Vec::new();
-        let mut processed_non_terminals = HashSet::new();
+        let mut processed_non_terminals = FxHashSet::default();
         for i in 0..syntax_grammar.variables.len() {
             let symbol = Symbol::non_terminal(i);
             let first_set = result.first_sets.entry(symbol).or_default();
@@ -102,14 +124,16 @@ impl<'a> ParseItemSetBuilder<'a> {
             symbols_to_process.clear();
             symbols_to_process.push(symbol);
             while let Some(sym) = symbols_to_process.pop() {
-                for production in &syntax_grammar.variables[sym.index].productions {
-                    if let Some(step) = production.steps.first() {
-                        if step.symbol.is_terminal() || step.symbol.is_external() {
-                            first_set.insert(step.symbol);
-                        } else if processed_non_terminals.insert(step.symbol) {
-                            symbols_to_process.push(step.symbol);
+                for prod_id in syntax_grammar.variable_prod_ids(sym.index as usize) {
+                    if let Some(step) = syntax_grammar.production(prod_id).steps.first() {
+                        let symbol = step.symbol();
+                        if symbol.is_terminal() || symbol.is_external() {
+                            first_set.insert(symbol);
+                        } else if processed_non_terminals.insert(symbol) {
+                            symbols_to_process.push(symbol);
                         }
-                        *reserved_first_set = (*reserved_first_set).max(step.reserved_word_set_id);
+                        *reserved_first_set =
+                            (*reserved_first_set).max(ReservedWordSetId(u32::from(step.reserved)));
                     }
                 }
             }
@@ -120,18 +144,26 @@ impl<'a> ParseItemSetBuilder<'a> {
             symbols_to_process.clear();
             symbols_to_process.push(symbol);
             while let Some(sym) = symbols_to_process.pop() {
-                for production in &syntax_grammar.variables[sym.index].productions {
-                    if let Some(step) = production.steps.last() {
-                        if step.symbol.is_terminal() || step.symbol.is_external() {
-                            last_set.insert(step.symbol);
-                        } else if processed_non_terminals.insert(step.symbol) {
-                            symbols_to_process.push(step.symbol);
+                for prod_id in syntax_grammar.variable_prod_ids(sym.index as usize) {
+                    if let Some(step) = syntax_grammar.production(prod_id).steps.last() {
+                        let symbol = step.symbol();
+                        if symbol.is_terminal() || symbol.is_external() {
+                            last_set.insert(symbol);
+                        } else if processed_non_terminals.insert(symbol) {
+                            symbols_to_process.push(symbol);
                         }
                     }
                 }
             }
         }
 
+        // Intern each FIRST set so closure propagation can union by id.
+        for (symbol, set) in &result.first_sets {
+            result
+                .first_set_ids
+                .insert(*symbol, result.lookaheads.intern_ref(set));
+        }
+
         // To compute an item set's transitive closure, we find each item in the set
         // whose next symbol is a non-terminal, and we add new items to the set for
         // each of that symbol's productions. These productions might themselves begin
@@ -152,8 +184,8 @@ impl<'a> ParseItemSetBuilder<'a> {
         //   * `lookaheads` - the set of possible lookahead tokens that can always come after `item`
         //     in an expansion of symbol `i`.
         //
-        //   * `reserved_lookaheads` - the set of reserved lookahead lookahead tokens that can
-        //     always come after `item` in the expansion of symbol `i`.
+        //   * `reserved_lookaheads` - the set of reserved lookahead tokens that can always come
+        //     after `item` in the expansion of symbol `i`.
         //
         //   * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the
         //     *end* of the expansion of symbol `i`, so that i's own current lookahead tokens can
@@ -161,8 +193,10 @@ impl<'a> ParseItemSetBuilder<'a> {
         //
         // Rather than computing these additions recursively, we use an explicit stack.
         let empty_lookaheads = TokenSet::new();
+        let mut eof_lookaheads = TokenSet::new();
+        eof_lookaheads.insert(Symbol::end());
         let mut stack = Vec::new();
-        let mut follow_set_info_by_non_terminal = HashMap::<usize, FollowSetInfo>::new();
+        let mut follow_set_info_by_non_terminal = FxHashMap::<usize, FollowSetInfo>::default();
         for i in 0..syntax_grammar.variables.len() {
             // First, build up a map whose keys are all of the non-terminals that can
             // appear at the beginning of non-terminal `i`, and whose values store
@@ -186,20 +220,28 @@ impl<'a> ParseItemSetBuilder<'a> {
                     continue;
                 }
 
-                for production in &syntax_grammar.variables[sym_ix].productions {
+                for prod_id in syntax_grammar.variable_prod_ids(sym_ix) {
+                    let production = syntax_grammar.production(prod_id);
                     if let Some(symbol) = production.first_symbol()
                         && symbol.is_non_terminal()
                     {
                         if let Some(next_step) = production.steps.get(1) {
                             stack.push((
-                                symbol.index,
-                                &result.first_sets[&next_step.symbol],
-                                result.reserved_first_sets[&next_step.symbol],
+                                symbol.index as usize,
+                                &result.first_sets[&next_step.symbol()],
+                                result.reserved_first_sets[&next_step.symbol()],
+                                false,
+                            ));
+                        } else if production.requires_eof_lookahead {
+                            stack.push((
+                                symbol.index as usize,
+                                &eof_lookaheads,
+                                ReservedWordSetId::default(),
                                 false,
                             ));
                         } else {
                             stack.push((
-                                symbol.index,
+                                symbol.index as usize,
                                 lookaheads,
                                 reserved_word_set_id,
                                 propagates_lookaheads,
@@ -211,40 +253,60 @@ impl<'a> ParseItemSetBuilder<'a> {
 
             // Store all of those non-terminals' productions, along with their associated
             // lookahead info, as *additions* associated with non-terminal `i`.
-            let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
             for (&variable_index, follow_set_info) in &follow_set_info_by_non_terminal {
-                let variable = &syntax_grammar.variables[variable_index];
                 let non_terminal = Symbol::non_terminal(variable_index);
-                let variable_index = variable_index as u32;
                 if syntax_grammar.variables_to_inline.contains(&non_terminal) {
                     continue;
                 }
-                for production in &variable.productions {
+                let info = AdditionInfo {
+                    lookaheads: result.lookaheads.intern_ref(&follow_set_info.lookaheads),
+                    reserved_lookaheads: follow_set_info.reserved_lookaheads,
+                    propagates_lookaheads: follow_set_info.propagates_lookaheads,
+                    contains_word: syntax_grammar
+                        .word_token
+                        .is_some_and(|w| follow_set_info.lookaheads.contains(w)),
+                };
+                let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
+                for prod_id in syntax_grammar.variable_prod_ids(variable_index) {
                     let item = ParseItem {
-                        variable_index,
-                        production,
+                        variable_index: variable_index as u32,
+                        prod_id,
+                        keys: key_map.keys_for(prod_id),
                         step_index: 0,
                         has_preceding_inherited_fields: false,
                     };
 
-                    if let Some(inlined_productions) =
-                        inlines.inlined_productions(item.production, item.step_index)
-                    {
-                        for production in inlined_productions {
+                    if let Some(ids) = inlines.inlined_prod_ids(item.prod_id, item.step_index) {
+                        for &id in ids {
+                            let mut item_info = info;
+                            if syntax_grammar.production(id).requires_eof_lookahead {
+                                item_info.lookaheads =
+                                    result.lookaheads.intern_ref(&eof_lookaheads);
+                                item_info.reserved_lookaheads = ReservedWordSetId::default();
+                                item_info.propagates_lookaheads = false;
+                                item_info.contains_word = false;
+                            }
                             find_or_push(
                                 additions_for_non_terminal,
                                 TransitiveClosureAddition {
-                                    item: item.substitute_production(production),
-                                    info: follow_set_info.clone(),
+                                    item: item.substitute_production(id, key_map.keys_for(id)),
+                                    info: item_info,
                                 },
                             );
                         }
                     } else {
+                        let mut item_info = info;
+                        if syntax_grammar.production(prod_id).requires_eof_lookahead {
+                            item_info.lookaheads = result.lookaheads.intern_ref(&eof_lookaheads);
+                            item_info.reserved_lookaheads = ReservedWordSetId::default();
+                            item_info.propagates_lookaheads = false;
+                            item_info.contains_word = false;
+                        }
                         find_or_push(
                             additions_for_non_terminal,
                             TransitiveClosureAddition {
                                 item,
-                                info: follow_set_info.clone(),
+                                info: item_info,
                             },
                         );
                     }
@@ -255,19 +317,22 @@ impl<'a> ParseItemSetBuilder<'a> {
         result
     }
 
-    pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
+    #[must_use]
+    pub fn transitive_closure(&mut self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
         let mut result = ParseItemSet::default();
         for entry in &item_set.entries {
-            if let Some(productions) = self
+            if let Some(ids) = self
                 .inlines
-                .inlined_productions(entry.item.production, entry.item.step_index)
+                .inlined_prod_ids(entry.item.prod_id, entry.item.step_index)
             {
-                for production in productions {
+                for &id in ids {
                     self.add_item(
                         &mut result,
                         &ParseItemSetEntry {
-                            item: entry.item.substitute_production(production),
-                            lookaheads: entry.lookaheads.clone(),
+                            item: entry
+                                .item
+                                .substitute_production(id, self.key_map.keys_for(id)),
+                            lookaheads: entry.lookaheads,
                             following_reserved_word_set: entry.following_reserved_word_set,
                         },
                     );
@@ -279,114 +344,133 @@ impl<'a> ParseItemSetBuilder<'a> {
         result
     }
 
-    pub fn first_set(&self, symbol: &Symbol) -> &TokenSet {
-        &self.first_sets[symbol]
+    #[must_use]
+    pub fn first_set(&self, symbol: Symbol) -> &TokenSet {
+        &self.first_sets[&symbol]
     }
 
-    pub fn reserved_first_set(&self, symbol: &Symbol) -> Option<&TokenSet> {
-        let id = *self.reserved_first_sets.get(symbol)?;
-        Some(&self.syntax_grammar.reserved_word_sets[id.0])
+    #[must_use]
+    pub fn reserved_first_set(&self, symbol: Symbol) -> Option<&TokenSet> {
+        let id = *self.reserved_first_sets.get(&symbol)?;
+        Some(&self.syntax_grammar.reserved_word_sets[id.0 as usize])
     }
 
-    pub fn last_set(&self, symbol: &Symbol) -> &TokenSet {
-        &self.last_sets[symbol]
+    #[must_use]
+    pub fn last_set(&self, symbol: Symbol) -> &TokenSet {
+        &self.last_sets[&symbol]
     }
 
-    fn add_item(&self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
-        if let Some(step) = entry.item.step()
-            && step.symbol.is_non_terminal()
+    fn add_item(&mut self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
+        if let Some(step) = entry.item.step(self.syntax_grammar)
+            && step.symbol().is_non_terminal()
         {
-            let next_step = entry.item.successor().step();
+            let next_step = entry.item.successor().step(self.syntax_grammar);
 
             // Determine which tokens can follow this non-terminal.
             let (following_tokens, following_reserved_tokens) = if let Some(next_step) = next_step {
                 (
-                    self.first_sets.get(&next_step.symbol).unwrap(),
-                    *self.reserved_first_sets.get(&next_step.symbol).unwrap(),
+                    self.first_set_ids[&next_step.symbol()],
+                    self.reserved_first_sets[&next_step.symbol()],
                 )
             } else {
-                (&entry.lookaheads, entry.following_reserved_word_set)
+                (entry.lookaheads, entry.following_reserved_word_set)
             };
 
             // Use the pre-computed *additions* to expand the non-terminal.
-            for addition in &self.transitive_closure_additions[step.symbol.index] {
-                let entry = set.insert(addition.item);
-                entry.lookaheads.insert_all(&addition.info.lookaheads);
+            for addition in &self.transitive_closure_additions[step.symbol().index as usize] {
+                let e = set.insert(addition.item);
+                e.lookaheads = self
+                    .lookaheads
+                    .union(e.lookaheads, addition.info.lookaheads);
 
-                if let Some(word_token) = self.syntax_grammar.word_token
-                    && addition.info.lookaheads.contains(&word_token)
-                {
-                    entry.following_reserved_word_set = entry
+                if addition.info.contains_word {
+                    e.following_reserved_word_set = e
                         .following_reserved_word_set
                         .max(addition.info.reserved_lookaheads);
                 }
 
                 if addition.info.propagates_lookaheads {
-                    entry.lookaheads.insert_all(following_tokens);
+                    e.lookaheads = self.lookaheads.union(e.lookaheads, following_tokens);
 
                     if let Some(word_token) = self.syntax_grammar.word_token
-                        && following_tokens.contains(&word_token)
+                        && self.lookaheads.get(following_tokens).contains(word_token)
                     {
-                        entry.following_reserved_word_set = entry
-                            .following_reserved_word_set
-                            .max(following_reserved_tokens);
+                        e.following_reserved_word_set =
+                            e.following_reserved_word_set.max(following_reserved_tokens);
                     }
                 }
             }
         }
 
         let e = set.insert(entry.item);
-        e.lookaheads.insert_all(&entry.lookaheads);
+        e.lookaheads = self.lookaheads.union(e.lookaheads, entry.lookaheads);
         e.following_reserved_word_set = e
             .following_reserved_word_set
             .max(entry.following_reserved_word_set);
     }
 }
 
-impl fmt::Debug for ParseItemSetBuilder<'_> {
+#[expect(dead_code, reason = "Debugging aid")]
+struct ParseItemSetBuilderDisplay<'a>(
+    pub &'a ParseItemSetBuilder<'a>,
+    pub &'a LexicalGrammar,
+    pub &'a StrPool,
+);
+
+impl fmt::Debug for ParseItemSetBuilderDisplay<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln!(f, "ParseItemSetBuilder {{")?;
 
         writeln!(f, "  first_sets: {{")?;
-        for (symbol, first_set) in &self.first_sets {
+        for (symbol, first_set) in &self.0.first_sets {
+            let symbol_index = symbol.index as usize;
             let name = match symbol.kind {
-                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
-                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
-                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
+                SymbolType::NonTerminal => self
+                    .2
+                    .resolve(self.0.syntax_grammar.variables[symbol_index].name),
+                SymbolType::External => self
+                    .2
+                    .resolve(self.0.syntax_grammar.external_tokens[symbol_index].name),
+                SymbolType::Terminal => self.2.resolve(self.1.variables[symbol_index].name),
                 SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
             };
             writeln!(
                 f,
                 "    first({name:?}): {}",
-                TokenSetDisplay(first_set, self.syntax_grammar, self.lexical_grammar)
+                TokenSetDisplay(first_set, self.0.syntax_grammar, self.1, self.2)
             )?;
         }
         writeln!(f, "  }}")?;
 
         writeln!(f, "  last_sets: {{")?;
-        for (symbol, last_set) in &self.last_sets {
+        for (symbol, last_set) in &self.0.last_sets {
+            let symbol_index = symbol.index as usize;
             let name = match symbol.kind {
-                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
-                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
-                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
+                SymbolType::NonTerminal => self
+                    .2
+                    .resolve(self.0.syntax_grammar.variables[symbol_index].name),
+                SymbolType::External => self
+                    .2
+                    .resolve(self.0.syntax_grammar.external_tokens[symbol_index].name),
+                SymbolType::Terminal => self.2.resolve(self.1.variables[symbol_index].name),
                 SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
             };
             writeln!(
                 f,
                 "    last({name:?}): {}",
-                TokenSetDisplay(last_set, self.syntax_grammar, self.lexical_grammar)
+                TokenSetDisplay(last_set, self.0.syntax_grammar, self.1, self.2)
             )?;
         }
         writeln!(f, "  }}")?;
 
         writeln!(f, "  additions: {{")?;
-        for (i, variable) in self.syntax_grammar.variables.iter().enumerate() {
-            writeln!(f, "    {}: {{", variable.name)?;
-            for addition in &self.transitive_closure_additions[i] {
+        for (i, variable) in self.0.syntax_grammar.variables.iter().enumerate() {
+            writeln!(f, "    {}: {{", self.2.resolve(variable.name))?;
+            for addition in &self.0.transitive_closure_additions[i] {
                 writeln!(
                     f,
                     "      {}",
-                    ParseItemDisplay(&addition.item, self.syntax_grammar, self.lexical_grammar)
+                    ParseItemDisplay(&addition.item, self.0.syntax_grammar, self.1, self.2)
                 )?;
             }
             writeln!(f, "    }},")?;

crates/generate/src/build_tables/minimize_parse_table.rs

@@ -1,7 +1,6 @@
-use std::{
-    collections::{HashMap, HashSet},
-    mem,
-};
+use std::{cmp::Ordering, mem};
+
+use rustc_hash::{FxHashMap, FxHashSet};
 
 use log::debug;
 
@@ -10,10 +9,75 @@ use crate::{
     OptLevel,
     dedup::split_state_id_groups,
     grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
-    rules::{AliasMap, Symbol, TokenSet},
-    tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
+    rules::{AliasMap, Symbol, SymbolType, TokenSet},
+    strpool::StrPool,
+    tables::{
+        ActionList, ActionListId, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable,
+    },
 };
 
+/// Index into [`SyntaxGrammar::variables`]. All nonterminal [`Symbol`]s share
+/// the same `kind`, so storing the index alone is sufficient for ordering.
+type NonterminalIndex = u32;
+
+/// A [`Symbol`] packed into a `u64` for O(1) sort-key comparison.
+///
+/// Layout: high 3 bits = `kind` discriminant (5 variants fit in 3 bits), low 61 bits = `index`.
+/// This preserves [`Symbol`]'s derived [`Ord`] ordering (kind first, then index) as a single
+/// integer comparison, and halves each entry's size vs storing a full `(Symbol, _)` tuple.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct SymbolKey(u64);
+
+const KEY_TAG_SHIFT: u32 = 61;
+const KEY_INDEX_MASK: u64 = (1u64 << KEY_TAG_SHIFT) - 1;
+
+impl SymbolKey {
+    #[inline]
+    fn new(sym: Symbol) -> Self {
+        debug_assert!(
+            u64::from(sym.index) <= KEY_INDEX_MASK,
+            "symbol index too large"
+        );
+        Self((sym.kind as u64) << KEY_TAG_SHIFT | u64::from(sym.index))
+    }
+
+    #[inline]
+    const fn symbol(self) -> Symbol {
+        let kind = match self.0 >> KEY_TAG_SHIFT {
+            0 => SymbolType::External,
+            1 => SymbolType::End,
+            2 => SymbolType::EndOfNonTerminalExtra,
+            3 => SymbolType::Terminal,
+            _ => SymbolType::NonTerminal,
+        };
+        Symbol {
+            kind,
+            index: self.index(),
+        }
+    }
+
+    #[inline]
+    const fn index(self) -> u32 {
+        (self.0 & KEY_INDEX_MASK) as u32
+    }
+
+    #[inline]
+    #[expect(dead_code)]
+    const fn is_terminal(self) -> bool {
+        (self.0 >> KEY_TAG_SHIFT) == SymbolType::Terminal as u64
+    }
+
+    #[inline]
+    #[expect(dead_code)]
+    const fn is_non_terminal(self) -> bool {
+        (self.0 >> KEY_TAG_SHIFT) == SymbolType::NonTerminal as u64
+    }
+}
+
+#[expect(
+    clippy::too_many_arguments,
+    reason = "all parameters are required for parse table minimization"
+)]
 pub fn minimize_parse_table(
     parse_table: &mut ParseTable,
     syntax_grammar: &SyntaxGrammar,
@@ -21,6 +85,7 @@ pub fn minimize_parse_table(
     simple_aliases: &AliasMap,
     token_conflict_map: &TokenConflictMap,
     keywords: &TokenSet,
+    str_pool: &StrPool,
     optimizations: OptLevel,
 ) {
     let mut minimizer = Minimizer {
@@ -30,6 +95,7 @@ pub fn minimize_parse_table(
         token_conflict_map,
         keywords,
         simple_aliases,
+        str_pool,
     };
     if optimizations.contains(OptLevel::MergeStates) {
         minimizer.merge_compatible_states();
@@ -39,34 +105,67 @@ pub fn minimize_parse_table(
     minimizer.reorder_states_by_descending_size();
 }
 
+/// Word-aligned bitsets precomputed for `token_conflicts`, all indexed by terminal
+/// index. `state_terminals` and `conflict_rows` are flattened two dimensional vectors:
+/// row `i` spans `[i * row_words, (i + 1) * row_words)`.
+struct ConflictBits {
+    row_words: usize,
+    /// Per state: which terminals have entries
+    state_terminals: Vec<u64>,
+    /// Per token: which terminals it lexically conflicts with
+    conflict_rows: Vec<u64>,
+    /// The keyword set.
+    keywords: Vec<u64>,
+    /// Tokens that are also external tokens.
+    internal_external: Vec<u64>,
+}
+
+impl ConflictBits {
+    #[inline]
+    fn get_conflict_row(&self, token: usize) -> &[u64] {
+        let base = token * self.row_words;
+        &self.conflict_rows[base..base + self.row_words]
+    }
+
+    #[inline]
+    fn get_state_row(&self, state: usize) -> &[u64] {
+        let base = state * self.row_words;
+        &self.state_terminals[base..base + self.row_words]
+    }
+}
+
 struct Minimizer<'a> {
     parse_table: &'a mut ParseTable,
     syntax_grammar: &'a SyntaxGrammar,
     lexical_grammar: &'a LexicalGrammar,
-    token_conflict_map: &'a TokenConflictMap<'a>,
+    token_conflict_map: &'a TokenConflictMap,
     keywords: &'a TokenSet,
     simple_aliases: &'a AliasMap,
+    str_pool: &'a StrPool,
 }
 
 impl Minimizer<'_> {
     fn remove_unit_reductions(&mut self) {
-        let mut aliased_symbols = HashSet::new();
-        for variable in &self.syntax_grammar.variables {
-            for production in &variable.productions {
-                for step in &production.steps {
-                    if step.alias.is_some() {
-                        aliased_symbols.insert(step.symbol);
+        let mut aliased_symbols = FxHashSet::default();
+        for i in 0..self.syntax_grammar.variables.len() {
+            for prod_id in self.syntax_grammar.variable_prod_ids(i) {
+                for step in self.syntax_grammar.production(prod_id).steps {
+                    if step.alias().is_some() {
+                        aliased_symbols.insert(step.symbol());
                     }
                 }
             }
         }
 
-        let mut unit_reduction_symbols_by_state = HashMap::new();
+        let mut unit_reduction_symbols_by_state = FxHashMap::default();
         for (i, state) in self.parse_table.states.iter().enumerate() {
+            if state.has_eof_gated_reduce {
+                continue;
+            }
             let mut only_unit_reductions = true;
             let mut unit_reduction_symbol = None;
-            for (_, entry) in &state.terminal_entries {
-                for action in &entry.actions {
+            for (_, id) in &state.terminal_entries {
+                for action in self.parse_table.action_lists.get(*id) {
                     match action {
                         ParseAction::ShiftExtra => continue,
                         ParseAction::Reduce {
@@ -78,7 +177,7 @@ impl Minimizer<'_> {
                             && !self.syntax_grammar.supertype_symbols.contains(symbol)
                             && !self.syntax_grammar.extra_symbols.contains(symbol)
                             && !aliased_symbols.contains(symbol)
-                            && self.syntax_grammar.variables[symbol.index].kind
+                            && self.syntax_grammar.variables[symbol.index as usize].kind
                                 != VariableType::Named
                             && (unit_reduction_symbol.is_none()
                                 || unit_reduction_symbol == Some(symbol)) =>
@@ -100,15 +199,27 @@ impl Minimizer<'_> {
             if let Some(symbol) = unit_reduction_symbol
                 && only_unit_reductions
             {
-                unit_reduction_symbols_by_state.insert(i, *symbol);
+                unit_reduction_symbols_by_state.insert(i as u32, *symbol);
             }
         }
 
-        for state in &mut self.parse_table.states {
+        if unit_reduction_symbols_by_state.is_empty() {
+            return;
+        }
+
+        let mut action_list_ids = FxHashMap::default();
+        for state_index in 0..self.parse_table.states.len() {
             let mut done = false;
             while !done {
                 done = true;
-                state.update_referenced_states(|other_state_id, state| {
+                let ParseTable {
+                    states,
+                    action_lists,
+                    ..
+                } = self.parse_table;
+                let state = &mut states[state_index];
+
+                state.update_nonterminal_references(|other_state_id, state| {
                     unit_reduction_symbols_by_state.get(&other_state_id).map_or(
                         other_state_id,
                         |symbol| {
@@ -120,6 +231,32 @@ impl Minimizer<'_> {
                         },
                     )
                 });
+
+                for i in 0..state.terminal_entries.len() {
+                    let old_id = state.terminal_entries.get_index(i).unwrap().1;
+                    let mut actions = ActionList::from_slice(action_lists.get(*old_id));
+                    let mut changed = false;
+                    for action in &mut *actions {
+                        // A Shift onto a unit-reduction state (one whose only action reduces a
+                        // single `symbol`) can skip it. Shift then reduce then goto is equivalent
+                        // to shifting straight to the the goto target for `symbol` in this case.
+                        if let ParseAction::Shift { state: target, .. } = action
+                            && let Some(symbol) = unit_reduction_symbols_by_state.get(target)
+                            && let Some(GotoAction::Goto(new_target)) =
+                                state.nonterminal_entries.get(symbol)
+                            && *new_target != *target
+                        {
+                            *target = *new_target;
+                            changed = true;
+                            done = false;
+                        }
+                    }
+                    if changed {
+                        let index = action_lists.intern(&mut action_list_ids, actions);
+                        *state.terminal_entries.get_index_mut(i).unwrap().1 =
+                            ActionListId::new(index, old_id.reusable());
+                    }
+                }
             }
         }
     }
@@ -135,26 +272,144 @@ impl Minimizer<'_> {
 
         // Initially group the states by their parse item set core.
         let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
-        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
+        // Pre-allocate for the maximum possible number of groups (one per state) to
+        // avoid reallocs as split_state_id_groups pushes new groups.
+        let mut state_ids_by_group_id = Vec::with_capacity(self.parse_table.states.len());
+        state_ids_by_group_id.resize(core_count as usize, Vec::new());
         for (i, state) in self.parse_table.states.iter().enumerate() {
-            state_ids_by_group_id[state.core_id].push(i);
+            state_ids_by_group_id[state.core_id as usize].push(i as u32);
             group_ids_by_state_id.push(state.core_id);
         }
 
+        // Precompute sorted terminal entry references for merge-join in states_conflict.
+        // entry_maps[state_id][i] = (symbol_key, action_list_id). Keys are packed u64s
+        // (symbol_key) for easy comparison.
+        let entry_maps = self
+            .parse_table
+            .states
+            .iter()
+            .map(|state| {
+                let mut entries = state
+                    .terminal_entries
+                    .iter()
+                    .map(|(sym, id)| (SymbolKey::new(*sym), *id))
+                    .collect::<Vec<(SymbolKey, ActionListId)>>();
+                entries.sort_unstable_by_key(|&(key, _)| key);
+                entries
+            })
+            .collect::<Vec<_>>();
+
+        // Precompute word-aligned bitsets so `token_conflicts` can test a candidate
+        // token against a whole state's terminals.
+        //   - per state: which terminal indices have entries
+        //   - per token: which terminal indices it lexically conflicts with
+        //   - the keyword set as bits
+        let n_terminals = self.lexical_grammar.variables.len();
+        let row_words = n_terminals.div_ceil(64);
+        let set = |bits: &mut [u64], index: usize| bits[index / 64] |= 1 << (index % 64);
+
+        let mut state_terminals = vec![0u64; self.parse_table.states.len() * row_words];
+        for (s, state) in self.parse_table.states.iter().enumerate() {
+            let base = s * row_words;
+            let row = &mut state_terminals[base..base + row_words];
+            for symbol in state.terminal_entries.keys() {
+                if symbol.is_terminal() {
+                    set(row, symbol.index as usize);
+                }
+            }
+        }
+
+        let mut conflict_rows = vec![0u64; n_terminals * row_words];
+        for i in 0..n_terminals {
+            let base = i * row_words;
+            let row = &mut conflict_rows[base..base + row_words];
+            for j in 0..n_terminals {
+                if self.token_conflict_map.does_conflict(i, j) {
+                    set(row, j);
+                }
+            }
+        }
+
+        let mut keywords = vec![0u64; row_words];
+        for symbol in self.keywords.iter() {
+            if symbol.is_terminal() {
+                set(&mut keywords, symbol.index as usize);
+            }
+        }
+
+        let mut internal_external = vec![0u64; row_words];
+        for external in &self.syntax_grammar.external_tokens {
+            if let Some(token) = external.corresponding_internal_token {
+                set(&mut internal_external, token.index as usize);
+            }
+        }
+
+        let bits = ConflictBits {
+            row_words,
+            state_terminals,
+            conflict_rows,
+            keywords,
+            internal_external,
+        };
+
         split_state_id_groups(
             &self.parse_table.states,
             &mut state_ids_by_group_id,
             &mut group_ids_by_state_id,
             0,
-            |left, right, groups| self.states_conflict(left, right, groups),
+            |left, right, groups| self.states_conflict(left, right, groups, &entry_maps, &bits),
         );
 
+        // Precompute per-state sorted shift actions and nonterminal goto actions.
+        // State actions are stable across loop iterations; only group assignments change.
+        // Keys are packed u64s (symbol_key) for single-instruction comparison.
+        let shift_maps = self
+            .parse_table
+            .states
+            .iter()
+            .map(|state| {
+                let mut shifts = state
+                    .terminal_entries
+                    .iter()
+                    .filter_map(|(sym, entry)| {
+                        let action = self.parse_table.action_lists.get(*entry).last()?;
+                        if let ParseAction::Shift { state: s, .. } = action {
+                            Some((SymbolKey::new(*sym), *s))
+                        } else {
+                            None
+                        }
+                    })
+                    .collect::<Vec<(SymbolKey, ParseStateId)>>();
+                shifts.sort_unstable_by_key(|&(key, _)| key);
+                shifts
+            })
+            .collect::<Vec<_>>();
+
+        // Store only the symbol index: all nonterminal entries share the same kind,
+        // so index alone is sufficient for sorting and comparison.
+        let nonterminal_maps = self
+            .parse_table
+            .states
+            .iter()
+            .map(|state| {
+                let mut entries = state
+                    .nonterminal_entries
+                    .iter()
+                    .map(|(sym, action)| (sym.index, *action))
+                    .collect::<Vec<(NonterminalIndex, GotoAction)>>();
+                entries.sort_unstable_by_key(|&(idx, _)| idx);
+                entries
+            })
+            .collect::<Vec<_>>();
+
         while split_state_id_groups(
             &self.parse_table.states,
             &mut state_ids_by_group_id,
             &mut group_ids_by_state_id,
             0,
-            |left, right, groups| self.state_successors_differ(left, right, groups),
+            |left, right, groups| {
+                self.state_successors_differ(left, right, groups, &shift_maps, &nonterminal_maps)
+            },
         ) {}
 
         let error_group_index = state_ids_by_group_id
@@ -172,13 +427,14 @@ impl Minimizer<'_> {
         let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
         for state_ids in &state_ids_by_group_id {
             // Initialize the new state based on the first old state in the group.
-            let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0]]);
+            let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0] as usize]);
 
             // Extend the new state with all of the actions from the other old states
             // in the group.
             for state_id in &state_ids[1..] {
-                let other_parse_state = mem::take(&mut self.parse_table.states[*state_id]);
+                let other_parse_state = mem::take(&mut self.parse_table.states[*state_id as usize]);
 
+                parse_state.has_eof_gated_reduce |= other_parse_state.has_eof_gated_reduce;
                 parse_state
                     .terminal_entries
                     .extend(other_parse_state.terminal_entries);
@@ -188,50 +444,88 @@ impl Minimizer<'_> {
                 parse_state
                     .reserved_words
                     .insert_all(&other_parse_state.reserved_words);
-                for symbol in parse_state.terminal_entries.keys() {
+                for &symbol in parse_state.terminal_entries.keys() {
                     parse_state.reserved_words.remove(symbol);
                 }
             }
 
             // Update the new state's outgoing references using the new grouping.
-            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
+            parse_state.update_nonterminal_references(|state_id, _| {
+                group_ids_by_state_id[state_id as usize]
+            });
             new_states.push(parse_state);
         }
 
         self.parse_table.states = new_states;
+        self.parse_table
+            .remap_terminal_references(|state_id| group_ids_by_state_id[state_id as usize]);
     }
 
     fn states_conflict(
         &self,
-        left_state: &ParseState,
-        right_state: &ParseState,
+        state1: &ParseState,
+        state2: &ParseState,
         group_ids_by_state_id: &[ParseStateId],
+        entry_maps: &[Vec<(SymbolKey, ActionListId)>],
+        bits: &ConflictBits,
     ) -> bool {
-        for (token, left_entry) in &left_state.terminal_entries {
-            if let Some(right_entry) = right_state.terminal_entries.get(token) {
-                if self.entries_conflict(
-                    left_state.id,
-                    right_state.id,
-                    token,
-                    left_entry,
-                    right_entry,
-                    group_ids_by_state_id,
-                ) {
-                    return true;
+        let entries1 = &entry_maps[state1.id as usize];
+        let entries2 = &entry_maps[state2.id as usize];
+        let len1 = entries1.len();
+        let len2 = entries2.len();
+        let mut i = 0;
+        let mut j = 0;
+        while i < len1 || j < len2 {
+            // SAFETY: each branch only accesses entries1[i] when i < len1
+            // and entries2[j] when j < len2, both of which hold by construction.
+            let ord = if i < len1 && j < len2 {
+                unsafe { entries1.get_unchecked(i) }
+                    .0
+                    .cmp(&unsafe { entries2.get_unchecked(j) }.0)
+            } else if i < len1 {
+                Ordering::Less
+            } else {
+                Ordering::Greater
+            };
+            match ord {
+                Ordering::Equal => {
+                    // SAFETY: Equal is only reachable when i < len1 && j < len2.
+                    let e1 = unsafe { entries1.get_unchecked(i) };
+                    let e2 = unsafe { entries2.get_unchecked(j) };
+                    let token = e1.0.symbol();
+                    if self.entries_conflict(
+                        state1.id,
+                        state2.id,
+                        token,
+                        e1.1,
+                        e2.1,
+                        group_ids_by_state_id,
+                    ) {
+                        return true;
+                    }
+                    i += 1;
+                    j += 1;
+                }
+                Ordering::Less => {
+                    // SAFETY: Less is only reachable when i < len1.
+                    let e1 = unsafe { entries1.get_unchecked(i) };
+                    let token = e1.0.symbol();
+                    if self.token_conflicts(state1.id, state2.id, state2, bits, token) {
+                        return true;
+                    }
+                    i += 1;
+                }
+                Ordering::Greater => {
+                    // SAFETY: Greater is only reachable when j < len2.
+                    let e2 = unsafe { entries2.get_unchecked(j) };
+                    let token = e2.0.symbol();
+                    if self.token_conflicts(state1.id, state2.id, state1, bits, token) {
+                        return true;
+                    }
+                    j += 1;
                 }
-            } else if self.token_conflicts(left_state.id, right_state.id, right_state, *token) {
-                return true;
             }
         }
-
-        for token in right_state.terminal_entries.keys() {
-            if !left_state.terminal_entries.contains_key(token)
-                && self.token_conflicts(left_state.id, right_state.id, left_state, *token)
-            {
-                return true;
-            }
-        }
-
         false
     }
 
@@ -240,44 +534,70 @@ impl Minimizer<'_> {
         state1: &ParseState,
         state2: &ParseState,
         group_ids_by_state_id: &[ParseStateId],
+        shift_maps: &[Vec<(SymbolKey, ParseStateId)>],
+        nonterminal_maps: &[Vec<(NonterminalIndex, GotoAction)>],
     ) -> bool {
-        for (token, entry1) in &state1.terminal_entries {
-            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap()
-                && let Some(entry2) = state2.terminal_entries.get(token)
-                && let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap()
-            {
-                let group1 = group_ids_by_state_id[*s1];
-                let group2 = group_ids_by_state_id[*s2];
-                if group1 != group2 {
-                    debug!(
-                        "split states {} {} - successors for {} are split: {s1} {s2}",
-                        state1.id,
-                        state2.id,
-                        self.symbol_name(token),
-                    );
-                    return true;
+        let shifts1 = &shift_maps[state1.id as usize];
+        let shifts2 = &shift_maps[state2.id as usize];
+        let mut i = 0;
+        let mut j = 0;
+        while i < shifts1.len() && j < shifts2.len() {
+            // SAFETY: loop condition ensures i < shifts1.len() and j < shifts2.len().
+            let (k1, s1) = *unsafe { shifts1.get_unchecked(i) };
+            let (k2, s2) = *unsafe { shifts2.get_unchecked(j) };
+            match k1.cmp(&k2) {
+                Ordering::Less => i += 1,
+                Ordering::Greater => j += 1,
+                Ordering::Equal => {
+                    let group1 = group_ids_by_state_id[s1 as usize];
+                    let group2 = group_ids_by_state_id[s2 as usize];
+                    if group1 != group2 {
+                        debug!(
+                            "split states {} {} - successors for {} are split: {s1} {s2}",
+                            state1.id,
+                            state2.id,
+                            self.symbol_name(k1.symbol()),
+                        );
+                        return true;
+                    }
+                    i += 1;
+                    j += 1;
                 }
             }
         }
 
-        for (symbol, s1) in &state1.nonterminal_entries {
-            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
-                match (s1, s2) {
-                    (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => {}
-                    (GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
-                        let group1 = group_ids_by_state_id[*s1];
-                        let group2 = group_ids_by_state_id[*s2];
-                        if group1 != group2 {
-                            debug!(
-                                "split states {} {} - successors for {} are split: {s1} {s2}",
-                                state1.id,
-                                state2.id,
-                                self.symbol_name(symbol),
-                            );
-                            return true;
+        let nonterms1 = &nonterminal_maps[state1.id as usize];
+        let nonterms2 = &nonterminal_maps[state2.id as usize];
+        let mut i = 0;
+        let mut j = 0;
+        while i < nonterms1.len() && j < nonterms2.len() {
+            // SAFETY: loop condition ensures i < nonterms1.len() and j < nonterms2.len().
+            let (idx1, s1) = *unsafe { nonterms1.get_unchecked(i) };
+            let (idx2, s2) = *unsafe { nonterms2.get_unchecked(j) };
+            match idx1.cmp(&idx2) {
+                Ordering::Less => i += 1,
+                Ordering::Greater => j += 1,
+                Ordering::Equal => {
+                    match (s1, s2) {
+                        (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => {}
+                        (GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
+                            let group1 = group_ids_by_state_id[s1 as usize];
+                            let group2 = group_ids_by_state_id[s2 as usize];
+                            if group1 != group2 {
+                                debug!(
+                                    "split states {} {} - successors for {} are split: {s1} {s2}",
+                                    state1.id,
+                                    state2.id,
+                                    self.str_pool
+                                        .resolve(self.syntax_grammar.variables[idx1 as usize].name),
+                                );
+                                return true;
+                            }
                         }
+                        _ => return true,
                     }
-                    _ => return true,
+                    i += 1;
+                    j += 1;
                 }
             }
         }
@@ -289,14 +609,17 @@ impl Minimizer<'_> {
         &self,
         state_id1: ParseStateId,
         state_id2: ParseStateId,
-        token: &Symbol,
-        entry1: &ParseTableEntry,
-        entry2: &ParseTableEntry,
+        token: Symbol,
+        id1: ActionListId,
+        id2: ActionListId,
         group_ids_by_state_id: &[ParseStateId],
     ) -> bool {
         // To be compatible, entries need to have the same actions.
-        let actions1 = &entry1.actions;
-        let actions2 = &entry2.actions;
+        if id1.index() == id2.index() {
+            return false;
+        }
+        let actions1 = self.parse_table.action_lists.get(id1);
+        let actions2 = self.parse_table.action_lists.get(id2);
         if actions1.len() != actions2.len() {
             debug!(
                 "split states {state_id1} {state_id2} - differing action counts for token {}",
@@ -318,8 +641,8 @@ impl Minimizer<'_> {
                 },
             ) = (action1, action2)
             {
-                let group1 = group_ids_by_state_id[*s1];
-                let group2 = group_ids_by_state_id[*s2];
+                let group1 = group_ids_by_state_id[*s1 as usize];
+                let group2 = group_ids_by_state_id[*s2 as usize];
                 if group1 == group2 && is_repetition1 == is_repetition2 {
                     continue;
                 }
@@ -346,10 +669,11 @@ impl Minimizer<'_> {
         left_id: ParseStateId,
         right_id: ParseStateId,
         right_state: &ParseState,
+        bits: &ConflictBits,
         new_token: Symbol,
     ) -> bool {
         if new_token == Symbol::end_of_nonterminal_extra() {
-            debug!("split states {left_id} {right_id} - end of non-terminal extra",);
+            debug!("split states {left_id} {right_id} - end of non-terminal extra");
             return true;
         }
 
@@ -358,52 +682,58 @@ impl Minimizer<'_> {
         if new_token.is_external() {
             debug!(
                 "split states {left_id} {right_id} - external token {}",
-                self.symbol_name(&new_token),
+                self.symbol_name(new_token),
             );
             return true;
         }
 
-        if right_state.reserved_words.contains(&new_token) {
+        if right_state.reserved_words.contains(new_token) {
             return false;
         }
 
         // Do not add tokens which are both internal and external. Their validity could
-        // influence the behavior of the external scanner.
-        if self
-            .syntax_grammar
-            .external_tokens
-            .iter()
-            .any(|external| external.corresponding_internal_token == Some(new_token))
+        // influence the behavior of the external scanner. `bits.internal_external` is
+        // indexed by terminal index only, so gate on the kind
+        if new_token.is_terminal()
+            && bits.internal_external[new_token.index as usize / 64]
+                & (1 << (new_token.index as usize % 64))
+                != 0
         {
             debug!(
                 "split states {left_id} {right_id} - internal/external token {}",
-                self.symbol_name(&new_token),
+                self.symbol_name(new_token),
             );
             return true;
         }
 
-        // Do not add a token if it conflicts with an existing token.
-        for token in right_state.terminal_entries.keys().copied() {
-            if !token.is_terminal() {
-                continue;
-            }
-            if self.syntax_grammar.word_token == Some(token) && self.keywords.contains(&new_token) {
-                continue;
-            }
-            if self.syntax_grammar.word_token == Some(new_token) && self.keywords.contains(&token) {
-                continue;
-            }
-
-            if self
-                .token_conflict_map
-                .does_conflict(new_token.index, token.index)
+        let word_token = self.syntax_grammar.word_token;
+        let new_token_is_word = word_token == Some(new_token);
+        let new_token_is_keyword = word_token.is_some() && self.keywords.contains(new_token);
+        // Do not add a token if it conflicts with an existing token. Test the candidate's
+        // conflict row against the state's terminal bits, masking out the word/keyword
+        // exemptions.
+        let row = bits.get_conflict_row(new_token.index as usize);
+        let right_terminal_bits = bits.get_state_row(right_state.id as usize);
+        for (w, &row_word) in row.iter().enumerate() {
+            let mut candidates = right_terminal_bits[w] & row_word;
+            if new_token_is_keyword
+                && let Some(word) = word_token
+                && word.index as usize / 64 == w
             {
+                candidates &= !(1u64 << (word.index as usize % 64));
+            }
+            if new_token_is_word {
+                candidates &= !bits.keywords[w];
+            }
+            if candidates != 0 {
                 debug!(
                     "split states {} {} - token {} conflicts with {}",
                     left_id,
                     right_id,
-                    self.symbol_name(&new_token),
-                    self.symbol_name(&token),
+                    self.symbol_name(new_token),
+                    self.symbol_name(Symbol::terminal(
+                        w * 64 + candidates.trailing_zeros() as usize
+                    )),
                 );
                 return true;
             }
@@ -412,13 +742,17 @@ impl Minimizer<'_> {
         false
     }
 
-    fn symbol_name(&self, symbol: &Symbol) -> &String {
+    fn symbol_name(&self, symbol: Symbol) -> &str {
+        let symbol_index = symbol.index as usize;
         if symbol.is_non_terminal() {
-            &self.syntax_grammar.variables[symbol.index].name
+            self.str_pool
+                .resolve(self.syntax_grammar.variables[symbol_index].name)
         } else if symbol.is_external() {
-            &self.syntax_grammar.external_tokens[symbol.index].name
+            self.str_pool
+                .resolve(self.syntax_grammar.external_tokens[symbol_index].name)
         } else {
-            &self.lexical_grammar.variables[symbol.index].name
+            self.str_pool
+                .resolve(self.lexical_grammar.variables[symbol_index].name)
         }
     }
 
@@ -429,14 +763,14 @@ impl Minimizer<'_> {
         state_usage_map[1] = true;
 
         for state in &self.parse_table.states {
-            for referenced_state in state.referenced_states() {
-                state_usage_map[referenced_state] = true;
+            for referenced_state in state.referenced_states(&self.parse_table.action_lists) {
+                state_usage_map[referenced_state as usize] = true;
             }
         }
         let mut removed_predecessor_count = 0;
         let mut state_replacement_map = vec![0; self.parse_table.states.len()];
         for state_id in 0..self.parse_table.states.len() {
-            state_replacement_map[state_id] = state_id - removed_predecessor_count;
+            state_replacement_map[state_id] = (state_id - removed_predecessor_count) as u32;
             if !state_usage_map[state_id] {
                 removed_predecessor_count += 1;
             }
@@ -445,22 +779,24 @@ impl Minimizer<'_> {
         let mut original_state_id = 0;
         while state_id < self.parse_table.states.len() {
             if state_usage_map[original_state_id] {
-                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
-                    state_replacement_map[other_state_id]
-                });
+                self.parse_table.states[state_id].update_nonterminal_references(
+                    |other_state_id, _| state_replacement_map[other_state_id as usize],
+                );
                 state_id += 1;
             } else {
                 self.parse_table.states.remove(state_id);
             }
             original_state_id += 1;
         }
+        self.parse_table
+            .remap_terminal_references(|state_id| state_replacement_map[state_id as usize]);
     }
 
     fn reorder_states_by_descending_size(&mut self) {
         // Get a mapping of old state index -> new_state_index
         let mut old_ids_by_new_id = (0..self.parse_table.states.len()).collect::<Vec<_>>();
         old_ids_by_new_id.sort_unstable_by_key(|i| {
-            // Don't changes states 0 (the error state) or 1 (the start state).
+            // Don't change states 0 (the error state) or 1 (the start state).
             if *i <= 1 {
                 return *i as i64 - 1_000_000;
             }
@@ -473,7 +809,7 @@ impl Minimizer<'_> {
         // Get the inverse mapping
         let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
         for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
-            new_ids_by_old_id[*old_id] = id;
+            new_ids_by_old_id[*old_id] = id as u32;
         }
 
         // Reorder the parse states and update their references to reflect
@@ -483,9 +819,11 @@ impl Minimizer<'_> {
             .map(|old_id| {
                 let mut state = ParseState::default();
                 mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
-                state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
+                state.update_nonterminal_references(|id, _| new_ids_by_old_id[id as usize]);
                 state
             })
             .collect();
+        self.parse_table
+            .remap_terminal_references(|id| new_ids_by_old_id[id as usize]);
     }
 }

crates/generate/src/build_tables/token_conflicts.rs

@@ -1,65 +1,126 @@
-use std::{cmp::Ordering, collections::HashSet, fmt};
+use std::cmp::Ordering;
+
+use rustc_hash::FxHashSet;
+
+use bitflags::bitflags;
 
 use crate::{
     build_tables::item::TokenSetDisplay,
     grammars::{LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor, NfaTransition},
     rules::TokenSet,
+    strpool::StrPool,
 };
 
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-struct TokenConflictStatus {
-    matches_prefix: bool,
-    does_match_continuation: bool,
-    does_match_valid_continuation: bool,
-    does_match_separators: bool,
-    matches_same_string: bool,
-    matches_different_string: bool,
+bitflags! {
+    /// Per-(i,j) token conflict status, packed into a single byte.
+    #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+    struct TokenConflictStatus: u8 {
+        const MATCHES_PREFIX              = 1 << 0;
+        const DOES_MATCH_CONTINUATION     = 1 << 1;
+        const DOES_MATCH_VALID_CONT       = 1 << 2;
+        const DOES_MATCH_SEPARATORS       = 1 << 3;
+        const MATCHES_SAME_STRING         = 1 << 4;
+        const MATCHES_DIFFERENT_STRING    = 1 << 5;
+    }
 }
 
-pub struct TokenConflictMap<'a> {
+pub struct TokenConflictMap {
     n: usize,
     status_matrix: Vec<TokenConflictStatus>,
+    #[expect(dead_code, reason = "Debugging aid")]
     following_tokens: Vec<TokenSet>,
+    #[allow(dead_code, reason = "Debugging/test aid")]
     starting_chars_by_index: Vec<CharacterSet>,
+    #[expect(dead_code, reason = "Debugging aid")]
     following_chars_by_index: Vec<CharacterSet>,
-    grammar: &'a LexicalGrammar,
+    /// Per-row bitsets for fast batch conflict checks.
+    /// Row `i` spans `[i * row_words .. (i+1) * row_words]`.
+    /// Bit `j` is set iff [`does_conflict(i, j)`](Self::does_conflict) || [`does_match_prefix(i, j)`](Self::does_match_prefix).
+    pub(crate) conflict_or_prefix_bits: Vec<u64>,
+    /// Bit `j` is set iff [`does_overlap(i, j)`](Self::does_overlap) || [`does_overlap(j, i)`](Self::does_overlap).
+    pub(crate) overlap_either_bits: Vec<u64>,
+    pub(crate) row_words: usize,
 }
 
-impl<'a> TokenConflictMap<'a> {
+impl TokenConflictMap {
     /// Create a token conflict map based on a lexical grammar, which describes the structure
-    /// of each token, and a `following_token` map, which indicates which tokens may be appear
+    /// of each token, and a `following_token` map, which indicates which tokens may appear
     /// immediately after each other token.
     ///
     /// This analyzes the possible kinds of overlap between each pair of tokens and stores
     /// them in a matrix.
-    pub fn new(grammar: &'a LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
+    #[must_use]
+    pub fn new(grammar: &LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
         let mut cursor = NfaCursor::new(&grammar.nfa, Vec::new());
         let starting_chars = get_starting_chars(&mut cursor, grammar);
         let following_chars = get_following_chars(&starting_chars, &following_tokens);
 
+        // Pre-compute O(1) lookup: NFA state ID->owning variable index.
+        // Replaces repeated O(log n) binary searches in compute_conflict_status.
+        let nfa_state_to_var = (0..grammar.nfa.states.len())
+            .map(|id| grammar.variable_index_for_nfa_state(id as u32))
+            .collect::<Vec<usize>>();
+
         let n = grammar.variables.len();
-        let mut status_matrix = vec![TokenConflictStatus::default(); n * n];
+        let mut status_matrix = vec![TokenConflictStatus::empty(); n * n];
         for i in 0..grammar.variables.len() {
             for j in 0..i {
-                let status = compute_conflict_status(&mut cursor, grammar, &following_chars, i, j);
+                let status = compute_conflict_status(
+                    &mut cursor,
+                    grammar,
+                    &following_chars,
+                    &nfa_state_to_var,
+                    i,
+                    j,
+                );
                 status_matrix[matrix_index(n, i, j)] = status.0;
                 status_matrix[matrix_index(n, j, i)] = status.1;
             }
         }
 
-        TokenConflictMap {
+        // Precompute per-row bitsets for vectorized check_token_conflicts.
+        let row_words = n.div_ceil(64);
+        let conflict_mask = TokenConflictStatus::DOES_MATCH_VALID_CONT
+            | TokenConflictStatus::DOES_MATCH_SEPARATORS
+            | TokenConflictStatus::MATCHES_SAME_STRING
+            | TokenConflictStatus::MATCHES_PREFIX;
+        let overlap_mask = TokenConflictStatus::DOES_MATCH_SEPARATORS
+            | TokenConflictStatus::MATCHES_PREFIX
+            | TokenConflictStatus::MATCHES_SAME_STRING
+            | TokenConflictStatus::DOES_MATCH_CONTINUATION;
+
+        let mut conflict_or_prefix_bits = vec![0u64; n * row_words];
+        let mut overlap_either_bits = vec![0u64; n * row_words];
+        for i in 0..n {
+            let row_base = i * row_words;
+            for j in 0..n {
+                let entry_ij = status_matrix[matrix_index(n, i, j)];
+                if entry_ij.intersects(conflict_mask) {
+                    conflict_or_prefix_bits[row_base + j / 64] |= 1u64 << (j % 64);
+                }
+                let entry_ji = status_matrix[matrix_index(n, j, i)];
+                if entry_ij.intersects(overlap_mask) || entry_ji.intersects(overlap_mask) {
+                    overlap_either_bits[row_base + j / 64] |= 1u64 << (j % 64);
+                }
+            }
+        }
+
+        Self {
             n,
             status_matrix,
             following_tokens,
             starting_chars_by_index: starting_chars,
             following_chars_by_index: following_chars,
-            grammar,
+            conflict_or_prefix_bits,
+            overlap_either_bits,
+            row_words,
         }
     }
 
     /// Does token `i` match any strings that token `j` also matches, such that token `i`
     /// is preferred over token `j`?
+    #[must_use]
     pub fn has_same_conflict_status(&self, a: usize, b: usize, other: usize) -> bool {
         let left = &self.status_matrix[matrix_index(self.n, a, other)];
         let right = &self.status_matrix[matrix_index(self.n, b, other)];
@@ -67,47 +128,63 @@ impl<'a> TokenConflictMap<'a> {
     }
 
     /// Does token `i` match any strings that token `j` does *not* match?
+    #[must_use]
     pub fn does_match_different_string(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_different_string
+        self.status_matrix[matrix_index(self.n, i, j)]
+            .contains(TokenConflictStatus::MATCHES_DIFFERENT_STRING)
     }
 
     /// Does token `i` match any strings that token `j` also matches, where
     /// token `i` is preferred over token `j`?
     #[inline]
+    #[must_use]
     pub fn does_match_same_string(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_same_string
+        self.status_matrix[matrix_index(self.n, i, j)]
+            .contains(TokenConflictStatus::MATCHES_SAME_STRING)
     }
 
     #[inline]
+    #[must_use]
     pub fn does_conflict(&self, i: usize, j: usize) -> bool {
-        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
-        entry.does_match_valid_continuation
-            || entry.does_match_separators
-            || entry.matches_same_string
+        debug_assert!(i < self.n && j < self.n, "token indices out of bounds");
+        // Safety: i < n and j < n imply n*i+j < n*n == status_matrix.len().
+        let entry = unsafe { *self.status_matrix.get_unchecked(matrix_index(self.n, i, j)) };
+        entry.intersects(
+            TokenConflictStatus::DOES_MATCH_VALID_CONT
+                | TokenConflictStatus::DOES_MATCH_SEPARATORS
+                | TokenConflictStatus::MATCHES_SAME_STRING,
+        )
     }
 
     /// Does token `i` match any strings that are *prefixes* of strings matched by `j`?
     #[inline]
+    #[must_use]
+    #[allow(dead_code)]
     pub fn does_match_prefix(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_prefix
+        self.status_matrix[matrix_index(self.n, i, j)].contains(TokenConflictStatus::MATCHES_PREFIX)
     }
 
+    #[must_use]
     pub fn does_match_shorter_or_longer(&self, i: usize, j: usize) -> bool {
-        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
-        let reverse_entry = &self.status_matrix[matrix_index(self.n, j, i)];
-        (entry.does_match_valid_continuation || entry.does_match_separators)
-            && !reverse_entry.does_match_separators
+        let entry = self.status_matrix[matrix_index(self.n, i, j)];
+        let reverse_entry = self.status_matrix[matrix_index(self.n, j, i)];
+        entry.intersects(
+            TokenConflictStatus::DOES_MATCH_VALID_CONT | TokenConflictStatus::DOES_MATCH_SEPARATORS,
+        ) && !reverse_entry.contains(TokenConflictStatus::DOES_MATCH_SEPARATORS)
     }
 
     #[inline]
+    #[must_use]
     pub fn does_overlap(&self, i: usize, j: usize) -> bool {
-        let status = &self.status_matrix[matrix_index(self.n, i, j)];
-        status.does_match_separators
-            || status.matches_prefix
-            || status.matches_same_string
-            || status.does_match_continuation
+        self.status_matrix[matrix_index(self.n, i, j)].intersects(
+            TokenConflictStatus::DOES_MATCH_SEPARATORS
+                | TokenConflictStatus::MATCHES_PREFIX
+                | TokenConflictStatus::MATCHES_SAME_STRING
+                | TokenConflictStatus::DOES_MATCH_CONTINUATION,
+        )
     }
 
+    #[must_use]
     pub fn prefer_token(grammar: &LexicalGrammar, left: (i32, usize), right: (i32, usize)) -> bool {
         match left.0.cmp(&right.0) {
             Ordering::Less => false,
@@ -123,6 +200,7 @@ impl<'a> TokenConflictMap<'a> {
         }
     }
 
+    #[must_use]
     pub fn prefer_transition(
         grammar: &LexicalGrammar,
         t: &NfaTransition,
@@ -149,52 +227,59 @@ impl<'a> TokenConflictMap<'a> {
     }
 }
 
-impl fmt::Debug for TokenConflictMap<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+#[expect(dead_code, reason = "Debugging aid")]
+struct TokenConflictMapDisplay<'a>(
+    pub &'a TokenConflictMap,
+    pub &'a LexicalGrammar,
+    pub &'a StrPool,
+);
+
+impl std::fmt::Debug for TokenConflictMapDisplay<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         writeln!(f, "TokenConflictMap {{")?;
 
         let syntax_grammar = SyntaxGrammar::default();
 
         writeln!(f, "  following_tokens: {{")?;
-        for (i, following_tokens) in self.following_tokens.iter().enumerate() {
+        for (i, following_tokens) in self.0.following_tokens.iter().enumerate() {
             writeln!(
                 f,
                 "    follow({:?}): {},",
-                self.grammar.variables[i].name,
-                TokenSetDisplay(following_tokens, &syntax_grammar, self.grammar)
+                self.1.variables[i].name,
+                TokenSetDisplay(following_tokens, &syntax_grammar, self.1, self.2)
             )?;
         }
         writeln!(f, "  }},")?;
 
         writeln!(f, "  starting_characters: {{")?;
-        for i in 0..self.n {
+        for i in 0..self.0.n {
             writeln!(
                 f,
                 "    {:?}: {:?},",
-                self.grammar.variables[i].name, self.starting_chars_by_index[i]
+                self.1.variables[i].name, self.0.starting_chars_by_index[i]
             )?;
         }
         writeln!(f, "  }},")?;
 
         writeln!(f, "  following_characters: {{")?;
-        for i in 0..self.n {
+        for i in 0..self.0.n {
             writeln!(
                 f,
                 "    {:?}: {:?},",
-                self.grammar.variables[i].name, self.following_chars_by_index[i]
+                self.1.variables[i].name, self.0.following_chars_by_index[i]
             )?;
         }
         writeln!(f, "  }},")?;
 
         writeln!(f, "  status_matrix: {{")?;
-        for i in 0..self.n {
-            writeln!(f, "    {:?}: {{", self.grammar.variables[i].name)?;
-            for j in 0..self.n {
+        for i in 0..self.0.n {
+            writeln!(f, "    {:?}: {{", self.1.variables[i].name)?;
+            for j in 0..self.0.n {
                 writeln!(
                     f,
                     "      {:?}: {:?},",
-                    self.grammar.variables[j].name,
-                    self.status_matrix[matrix_index(self.n, i, j)]
+                    self.1.variables[j].name,
+                    self.0.status_matrix[matrix_index(self.0.n, i, j)]
                 )?;
             }
             writeln!(f, "    }},")?;
@@ -233,7 +318,7 @@ fn get_following_chars(
             let mut chars = CharacterSet::empty();
             for token in following_tokens.iter() {
                 if token.is_terminal() {
-                    chars = chars.add(&starting_chars[token.index]);
+                    chars = chars.add(&starting_chars[token.index as usize]);
                 }
             }
             chars
@@ -241,51 +326,76 @@ fn get_following_chars(
         .collect()
 }
 
+/// Hash a sorted slice of NFA state IDs to a single `u64` for use as a
+/// visited-set key.
+///
+/// NOTE: This trades exact equality for speed. Two distinct state sets that
+/// hash to the same `u64` will be treated as "already visited," potentially
+/// causing the BFS to skip a state set. In practice the collision probability
+/// is negligible (1 in 2^64 per comparison), but this is technically a
+/// probabilistic optimization rather than an exact one.
+#[inline]
+fn hash_state_set(states: &[u32]) -> u64 {
+    use std::hash::{Hash, Hasher};
+    let mut h = rustc_hash::FxHasher::default();
+    states.hash(&mut h);
+    h.finish()
+}
+
 fn compute_conflict_status(
     cursor: &mut NfaCursor,
     grammar: &LexicalGrammar,
     following_chars: &[CharacterSet],
+    nfa_state_to_var: &[usize],
     i: usize,
     j: usize,
 ) -> (TokenConflictStatus, TokenConflictStatus) {
-    let mut visited_state_sets = HashSet::new();
-    let mut state_set_queue = vec![vec![
+    let mut visited_state_sets = FxHashSet::<u64>::default();
+    let mut state_set_queue = Vec::with_capacity(4);
+    state_set_queue.push(vec![
         grammar.variables[i].start_state,
         grammar.variables[j].start_state,
-    ]];
-    let mut result = (
-        TokenConflictStatus::default(),
-        TokenConflictStatus::default(),
-    );
+    ]);
+    let mut result = (TokenConflictStatus::empty(), TokenConflictStatus::empty());
 
     while let Some(state_set) = state_set_queue.pop() {
-        let mut live_variable_indices = grammar.variable_indices_for_nfa_states(&state_set);
-
         // If only one of the two tokens could possibly match from this state, then
         // there is no reason to analyze any of its successors. Just record the fact
         // that the token matches a string that the other token does not match.
-        let first_live_variable_index = live_variable_indices.next().unwrap();
-        if live_variable_indices.count() == 0 {
+        let first_live_variable_index = nfa_state_to_var[state_set[0] as usize];
+        if state_set
+            .iter()
+            .all(|&s| nfa_state_to_var[s as usize] == first_live_variable_index)
+        {
             if first_live_variable_index == i {
-                result.0.matches_different_string = true;
+                result
+                    .0
+                    .insert(TokenConflictStatus::MATCHES_DIFFERENT_STRING);
             } else {
-                result.1.matches_different_string = true;
+                result
+                    .1
+                    .insert(TokenConflictStatus::MATCHES_DIFFERENT_STRING);
             }
             continue;
         }
 
         // Don't pursue states where there's no potential for conflict.
         cursor.reset(state_set);
-        let within_separator = cursor.transition_chars().any(|(_, sep)| sep);
+
+        // Compute lazily: most BFS states have no completions, so
+        // `within_separator` is never needed in those iterations.
+        let mut within_separator: Option<bool> = None;
 
         // Examine each possible completed token in this state.
         let mut completion = None;
         for (id, precedence) in cursor.completions() {
-            if within_separator {
+            let sep = *within_separator
+                .get_or_insert_with(|| cursor.transition_chars().any(|(_, sep)| sep));
+            if sep {
                 if id == i {
-                    result.0.does_match_separators = true;
+                    result.0.insert(TokenConflictStatus::DOES_MATCH_SEPARATORS);
                 } else {
-                    result.1.does_match_separators = true;
+                    result.1.insert(TokenConflictStatus::DOES_MATCH_SEPARATORS);
                 }
             }
 
@@ -310,9 +420,9 @@ fn compute_conflict_status(
                 }
 
                 if preferred_id == i {
-                    result.0.matches_same_string = true;
+                    result.0.insert(TokenConflictStatus::MATCHES_SAME_STRING);
                 } else {
-                    result.1.matches_same_string = true;
+                    result.1.insert(TokenConflictStatus::MATCHES_SAME_STRING);
                 }
             } else {
                 completion = Some((id, precedence));
@@ -329,12 +439,18 @@ fn compute_conflict_status(
             if let Some((completed_id, completed_precedence)) = completion {
                 let mut advanced_id = None;
                 let mut successor_contains_completed_id = false;
-                for variable_id in grammar.variable_indices_for_nfa_states(&transition.states) {
-                    if variable_id == completed_id {
+                let mut prev_var = None;
+                for &state_id in &transition.states {
+                    let var_id = nfa_state_to_var[state_id as usize];
+                    if prev_var == Some(var_id) {
+                        continue;
+                    }
+                    prev_var = Some(var_id);
+                    if var_id == completed_id {
                         successor_contains_completed_id = true;
                         break;
                     }
-                    advanced_id = Some(variable_id);
+                    advanced_id = Some(var_id);
                 }
 
                 // Determine which action is preferred: matching the already complete
@@ -345,29 +461,33 @@ fn compute_conflict_status(
                         &transition,
                         completed_id,
                         completed_precedence,
-                        within_separator,
+                        within_separator.unwrap_or(false),
                     ) {
                         can_advance = true;
                         if advanced_id == i {
-                            result.0.does_match_continuation = true;
+                            result
+                                .0
+                                .insert(TokenConflictStatus::DOES_MATCH_CONTINUATION);
                             if transition.characters.does_intersect(&following_chars[j]) {
-                                result.0.does_match_valid_continuation = true;
+                                result.0.insert(TokenConflictStatus::DOES_MATCH_VALID_CONT);
                             }
                         } else {
-                            result.1.does_match_continuation = true;
+                            result
+                                .1
+                                .insert(TokenConflictStatus::DOES_MATCH_CONTINUATION);
                             if transition.characters.does_intersect(&following_chars[i]) {
-                                result.1.does_match_valid_continuation = true;
+                                result.1.insert(TokenConflictStatus::DOES_MATCH_VALID_CONT);
                             }
                         }
                     } else if completed_id == i {
-                        result.0.matches_prefix = true;
+                        result.0.insert(TokenConflictStatus::MATCHES_PREFIX);
                     } else {
-                        result.1.matches_prefix = true;
+                        result.1.insert(TokenConflictStatus::MATCHES_PREFIX);
                     }
                 }
             }
 
-            if can_advance && visited_state_sets.insert(transition.states.clone()) {
+            if can_advance && visited_state_sets.insert(hash_state_set(&transition.states)) {
                 state_set_queue.push(transition.states);
             }
         }
@@ -379,29 +499,36 @@ fn compute_conflict_status(
 mod tests {
     use super::*;
     use crate::{
-        grammars::{Variable, VariableType},
-        prepare_grammar::{ExtractedLexicalGrammar, expand_tokens},
-        rules::{Precedence, Rule, Symbol},
+        grammars::VariableType,
+        prepare_grammar::{LexicalToken, expand_tokens},
+        rules::{Precedence, RulePool, Symbol},
     };
 
     #[test]
     fn test_starting_characters() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: Vec::new(),
-            variables: vec![
-                Variable {
-                    name: "token_0".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("[a-f]1|0x\\d", ""),
-                },
-                Variable {
-                    name: "token_1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("d*ef", ""),
-                },
-            ],
-        })
-        .unwrap();
+        let mut pool = RulePool::default();
+        let empty = pool.intern("");
+        let t0 = {
+            let v = pool.intern("[a-f]1|0x\\d");
+            pool.pattern(v, empty)
+        };
+        let t1 = {
+            let v = pool.intern("d*ef");
+            pool.pattern(v, empty)
+        };
+        let vars = vec![
+            LexicalToken {
+                name: pool.intern("token_0"),
+                kind: VariableType::Named,
+                root: t0,
+            },
+            LexicalToken {
+                name: pool.intern("token_1"),
+                kind: VariableType::Named,
+                root: t1,
+            },
+        ];
+        let grammar = expand_tokens(&mut pool, &vars, &[]).unwrap();
 
         let token_map = TokenConflictMap::new(&grammar, Vec::new());
 
@@ -417,29 +544,40 @@ mod tests {
 
     #[test]
     fn test_token_conflicts() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: Vec::new(),
-            variables: vec![
-                Variable {
-                    name: "in".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("in"),
-                },
-                Variable {
-                    name: "identifier".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("\\w+", ""),
-                },
-                Variable {
-                    name: "instanceof".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("instanceof"),
-                },
-            ],
-        })
-        .unwrap();
+        let mut pool = RulePool::default();
+        let empty = pool.intern("");
+        let in_tok = {
+            let s = pool.intern("in");
+            pool.string(s)
+        };
+        let ident = {
+            let v = pool.intern("\\w+");
+            pool.pattern(v, empty)
+        };
+        let instanceof = {
+            let s = pool.intern("instanceof");
+            pool.string(s)
+        };
+        let vars = vec![
+            LexicalToken {
+                name: pool.intern("in"),
+                kind: VariableType::Named,
+                root: in_tok,
+            },
+            LexicalToken {
+                name: pool.intern("identifier"),
+                kind: VariableType::Named,
+                root: ident,
+            },
+            LexicalToken {
+                name: pool.intern("instanceof"),
+                kind: VariableType::Named,
+                root: instanceof,
+            },
+        ];
+        let grammar = expand_tokens(&mut pool, &vars, &[]).unwrap();
 
-        let var = |name| index_of_var(&grammar, name);
+        let var = |name| index_of_var(&pool, &grammar, name);
 
         let token_map = TokenConflictMap::new(
             &grammar,
@@ -456,40 +594,51 @@ mod tests {
             ],
         );
 
-        // Given the string "in", the `in` token is preferred over the `identifier` token
+        // Given the string "in", the `in` token is preferrred over the `identifier` token
         assert!(token_map.does_match_same_string(var("in"), var("identifier")));
         assert!(!token_map.does_match_same_string(var("identifier"), var("in")));
 
         // Depending on what character follows, the string "in" may be treated as part of an
-        // `identifier` token.
+        // `identifier` token
         assert!(token_map.does_conflict(var("identifier"), var("in")));
 
-        // Depending on what character follows, the string "instanceof" may be treated as part of
-        // an `identifier` token.
+        // Depending on what character follows, the string "instanceof" may be treated as part
+        // of an `identifier` token
         assert!(token_map.does_conflict(var("identifier"), var("instanceof")));
         assert!(token_map.does_conflict(var("instanceof"), var("in")));
     }
 
     #[test]
     fn test_token_conflicts_with_separators() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: vec![Rule::pattern("\\s", "")],
-            variables: vec![
-                Variable {
-                    name: "x".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("x"),
-                },
-                Variable {
-                    name: "newline".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("\n"),
-                },
-            ],
-        })
-        .unwrap();
+        let mut pool = RulePool::default();
+        let empty = pool.intern("");
+        let sep = {
+            let v = pool.intern("\\s");
+            pool.pattern(v, empty)
+        };
+        let x = {
+            let s = pool.intern("x");
+            pool.string(s)
+        };
+        let newline = {
+            let s = pool.intern("\n");
+            pool.string(s)
+        };
+        let vars = vec![
+            LexicalToken {
+                name: pool.intern("x"),
+                kind: VariableType::Named,
+                root: x,
+            },
+            LexicalToken {
+                name: pool.intern("newline"),
+                kind: VariableType::Named,
+                root: newline,
+            },
+        ];
+        let grammar = expand_tokens(&mut pool, &vars, &[sep]).unwrap();
 
-        let var = |name| index_of_var(&grammar, name);
+        let var = |name| index_of_var(&pool, &grammar, name);
 
         let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
 
@@ -499,24 +648,36 @@ mod tests {
 
     #[test]
     fn test_token_conflicts_with_open_ended_tokens() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: vec![Rule::pattern("\\s", "")],
-            variables: vec![
-                Variable {
-                    name: "x".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("x"),
-                },
-                Variable {
-                    name: "anything".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*", "")),
-                },
-            ],
-        })
-        .unwrap();
+        let mut pool = RulePool::default();
+        let empty = pool.intern("");
+        let sep = {
+            let v = pool.intern("\\s");
+            pool.pattern(v, empty)
+        };
+        let x = {
+            let s = pool.intern("x");
+            pool.string(s)
+        };
+        let anything = {
+            let v = pool.intern(".*");
+            let pat = pool.pattern(v, empty);
+            pool.prec(Precedence::Integer(-1), pat)
+        };
+        let vars = vec![
+            LexicalToken {
+                name: pool.intern("x"),
+                kind: VariableType::Named,
+                root: x,
+            },
+            LexicalToken {
+                name: pool.intern("anything"),
+                kind: VariableType::Named,
+                root: anything,
+            },
+        ];
+        let grammar = expand_tokens(&mut pool, &vars, &[sep]).unwrap();
 
-        let var = |name| index_of_var(&grammar, name);
+        let var = |name| index_of_var(&pool, &grammar, name);
 
         let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
 
@@ -524,11 +685,11 @@ mod tests {
         assert!(!token_map.does_match_shorter_or_longer(var("x"), var("anything")));
     }
 
-    fn index_of_var(grammar: &LexicalGrammar, name: &str) -> usize {
+    fn index_of_var(pool: &RulePool, grammar: &LexicalGrammar, name: &str) -> usize {
         grammar
             .variables
             .iter()
-            .position(|v| v.name == name)
+            .position(|v| pool.resolve(v.name) == name)
             .unwrap()
     }
 }

crates/generate/src/dedup.rs

@@ -1,22 +1,25 @@
 #[inline]
 pub fn split_state_id_groups<S>(
     states: &[S],
-    state_ids_by_group_id: &mut Vec<Vec<usize>>,
-    group_ids_by_state_id: &mut [usize],
-    start_group_id: usize,
-    mut should_split: impl FnMut(&S, &S, &[usize]) -> bool,
+    state_ids_by_group_id: &mut Vec<Vec<u32>>,
+    group_ids_by_state_id: &mut [u32],
+    start_group_id: u32,
+    mut should_split: impl FnMut(&S, &S, &[u32]) -> bool,
 ) -> bool {
     let mut result = false;
 
+    // Use a flat bitset instead of Vec::contains for O(1) membership tests.
+    let mut is_split = vec![false; states.len()];
+
     let mut group_id = start_group_id;
-    while group_id < state_ids_by_group_id.len() {
-        let state_ids = &state_ids_by_group_id[group_id];
+    while (group_id as usize) < state_ids_by_group_id.len() {
+        let state_ids = &state_ids_by_group_id[group_id as usize];
         let mut split_state_ids = Vec::new();
 
         let mut i = 0;
         while i < state_ids.len() {
-            let left_state_id = state_ids[i];
-            if split_state_ids.contains(&left_state_id) {
+            let left_state_id = state_ids[i] as usize;
+            if is_split[left_state_id] {
                 i += 1;
                 continue;
             }
@@ -27,15 +30,16 @@ pub fn split_state_id_groups<S>(
             // this state.
             let mut j = i + 1;
             while j < state_ids.len() {
-                let right_state_id = state_ids[j];
-                if split_state_ids.contains(&right_state_id) {
+                let right_state_id = state_ids[j] as usize;
+                if is_split[right_state_id] {
                     j += 1;
                     continue;
                 }
                 let right_state = &states[right_state_id];
 
                 if should_split(left_state, right_state, group_ids_by_state_id) {
-                    split_state_ids.push(right_state_id);
+                    split_state_ids.push(right_state_id as u32);
+                    is_split[right_state_id] = true;
                 }
 
                 j += 1;
@@ -47,11 +51,12 @@ pub fn split_state_id_groups<S>(
         // If any states were removed from the group, add them all as a new group.
         if !split_state_ids.is_empty() {
             result = true;
-            state_ids_by_group_id[group_id].retain(|i| !split_state_ids.contains(i));
+            state_ids_by_group_id[group_id as usize].retain(|i| !is_split[*i as usize]);
 
-            let new_group_id = state_ids_by_group_id.len();
+            let new_group_id = state_ids_by_group_id.len() as u32;
             for id in &split_state_ids {
-                group_ids_by_state_id[*id] = new_group_id;
+                group_ids_by_state_id[*id as usize] = new_group_id;
+                is_split[*id as usize] = false;
             }
 
             state_ids_by_group_id.push(split_state_ids);