0.24.4

(cherry picked from commit ac4595e808)

.dockerignore

@@ -0,0 +1,2 @@
+target
+.git

.editorconfig

@@ -10,9 +10,6 @@ insert_final_newline = true
 [*.rs]
 indent_size = 4
 
-[*.{zig,zon}]
-indent_size = 4
-
 [Makefile]
 indent_style = tab
 indent_size = 8

.envrc

@@ -1,4 +0,0 @@
-source_up_if_exists
-if [ -z "${IN_NIX_SHELL:-}" ]; then
-  use flake .
-fi

.gitattributes

@@ -3,4 +3,5 @@
 /lib/src/unicode/*.h linguist-vendored
 /lib/src/unicode/LICENSE linguist-vendored
 
+/cli/src/generate/prepare_grammar/*.json -diff
 Cargo.lock -diff

.github/FUNDING.yml

@@ -1,15 +0,0 @@
-# These are supported funding model platforms
-
-github: tree-sitter
-patreon: # Replace with a single Patreon username
-open_collective: tree-sitter # Replace with a single Open Collective username
-ko_fi: amaanq
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-polar: # Replace with a single Polar username
-buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
-thanks_dev: # Replace with a single thanks.dev username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,15 +1,7 @@
 name: Bug Report
 description: Report a problem
-type: Bug
+labels: [bug]
 body:
-  - type: checkboxes
-    attributes:
-      label: AI Policy
-      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
-      options:
-        - label: I have read the AI Policy and this issue complies with it.
-          required: true
-
   - type: textarea
     attributes:
       label: "Problem"
@@ -21,11 +13,9 @@ body:
     attributes:
       label: "Steps to reproduce"
       placeholder: |
-        ```sh
         git clone --depth=1 https://github.com/tree-sitter/tree-sitter-ruby
         cd tree-sitter-ruby
         tree-sitter generate
-        ```
     validations:
       required: true
 

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,15 +1,7 @@
 name: Feature request
 description: Request an enhancement
-type: Feature
+labels: [enhancement]
 body:
-  - type: checkboxes
-    attributes:
-      label: AI Policy
-      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
-      options:
-        - label: I have read the AI Policy and this issue complies with it.
-          required: true
-
   - type: markdown
     attributes:
       value: |

.github/actions/cache/action.yml

@@ -1,29 +1,24 @@
-name: Cache
-
-description: This action caches fixtures
-
+name: 'Cache'
+description: "This action caches fixtures"
 outputs:
   cache-hit:
-    description: Cache hit
-    value: ${{ steps.cache.outputs.cache-hit }}
-
+    description: 'Cache hit'
+    value: ${{ steps.cache_output.outputs.cache-hit }}
 runs:
-  using: composite
+  using: "composite"
   steps:
-    - uses: actions/cache@v5
-      id: cache
+    - uses: actions/cache@v4
+      id: cache_fixtures
       with:
         path: |
           test/fixtures/grammars
           target/release/tree-sitter-*.wasm
         key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles(
-          'crates/generate/src/**',
-          'lib/src/parser.h',
-          'lib/src/array.h',
-          'lib/src/alloc.h',
-          'lib/src/wasm-stdlib/external_scanner_stdlib.h',
-          'crates/loader/wasi-sdk-version',
-          'crates/loader/binaryen-version',
+          'cli/src/generate/**',
+          'script/generate-fixtures*',
           'test/fixtures/grammars/*/**/src/*.c',
-          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}
+
+    - run: echo "cache-hit=${{ steps.cache_fixtures.outputs.cache-hit }}" >> $GITHUB_OUTPUT
+      shell: bash
+      id: cache_output

.github/dependabot.yml

@@ -4,8 +4,6 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    cooldown:
-      default-days: 3
     commit-message:
       prefix: "build(deps)"
     labels:
@@ -14,16 +12,10 @@ updates:
     groups:
       cargo:
         patterns: ["*"]
-    ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-major", "version-update:semver-minor"]
-
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
-    cooldown:
-      default-days: 3
     commit-message:
       prefix: "ci"
     labels:
@@ -32,22 +24,3 @@ updates:
     groups:
       actions:
         patterns: ["*"]
-
-  - package-ecosystem: "npm"
-    versioning-strategy: increase
-    directories:
-      - "/crates/npm"
-      - "/crates/eslint"
-      - "/lib/binding_web"
-    schedule:
-      interval: "weekly"
-    cooldown:
-      default-days: 3
-    commit-message:
-      prefix: "build(deps)"
-    labels:
-      - "dependencies"
-      - "npm"
-    groups:
-      npm:
-        patterns: ["*"]

.github/pull_request_template.md

@@ -1,6 +0,0 @@
-### AI Policy
-
-- [ ] I have read the [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy) and this PR complies with it.
-- [ ] If AI tools were used: I have disclosed the tool and extent of usage below.
-
-<!-- If you used AI tools, state which tool and how it was used. Delete this section if not applicable. -->

.github/scripts/close_spam.js

@@ -1,29 +0,0 @@
-module.exports = async ({ github, context }) => {
-  let target = context.payload.issue;
-  if (target) {
-    await github.rest.issues.update({
-      ...context.repo,
-      issue_number: target.number,
-      state: "closed",
-      state_reason: "not_planned",
-      title: "[spam]",
-      body: "",
-      type: null,
-    });
-  } else {
-    target = context.payload.pull_request;
-    await github.rest.pulls.update({
-      ...context.repo,
-      pull_number: target.number,
-      state: "closed",
-      title: "[spam]",
-      body: "",
-    });
-  }
-
-  await github.rest.issues.lock({
-    ...context.repo,
-    issue_number: target.number,
-    lock_reason: "spam",
-  });
-};

.github/scripts/cross.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# set -x
+set -e
+
+if [ "$BUILD_CMD" != "cross" ]; then
+    echo "cross.sh - is a helper to assist only in cross compiling environments" >&2
+    echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2
+    exit 111
+fi
+
+if [ -z "$CROSS_IMAGE" ]; then
+    echo "The CROSS_IMAGE env var should be provided" >&2
+    exit 111
+fi
+
+docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/scripts/make.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# set -x
+set -e
+
+if [ "$BUILD_CMD" == "cross" ]; then
+    if [ -z "$CC" ]; then
+        echo "make.sh: CC is not set" >&2
+        exit 111
+    fi
+    if [ -z "$AR" ]; then
+        echo "make.sh: AR is not set" >&2
+        exit 111
+    fi
+
+    cross.sh make CC=$CC AR=$AR "$@"
+else
+    make "$@"
+fi

.github/scripts/reviewers_remove.js

@@ -0,0 +1,16 @@
+module.exports = async ({ github, context }) => {
+  const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    pull_number: context.issue.number,
+  });
+
+  const reviewers = requestedReviewers.data.users.map((e) => e.login);
+
+  github.rest.pulls.removeRequestedReviewers({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    pull_number: context.issue.number,
+    reviewers: reviewers,
+  });
+};

.github/scripts/tree-sitter.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# set -x
+set -e
+
+if [ -z "$ROOT" ]; then
+    echo "The ROOT env var should be set to absolute path of a repo root folder" >&2
+    exit 111
+fi
+
+if [ -z "$TARGET" ]; then
+    echo "The TARGET env var should be equal to a \`cargo build --target <TARGET>\` command value" >&2
+    exit 111
+fi
+
+tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
+
+if [ "$BUILD_CMD" == "cross" ]; then
+    if [ -z "$CROSS_RUNNER" ]; then
+        echo "The CROSS_RUNNER env var should be set to a CARGO_TARGET_*_RUNNER env var value" >&2
+        echo "that is available in a docker image used by the cross tool under the hood" >&2
+        exit 111
+    fi
+
+    cross.sh $CROSS_RUNNER "$tree_sitter" "$@"
+else
+    "$tree_sitter" "$@"
+fi

.github/scripts/wasm_stdlib.js

@@ -1,35 +0,0 @@
-module.exports = async ({ github, context, core }) => {
-  if (context.eventName !== 'pull_request') return;
-
-  const prNumber = context.payload.pull_request.number;
-  const owner = context.repo.owner;
-  const repo = context.repo.repo;
-
-  const files = await github.paginate(github.rest.pulls.listFiles, {
-    owner,
-    repo,
-    pull_number: prNumber,
-    per_page: 100
-  });
-
-  const changedFiles = files.map(file => file.filename);
-
-  const wasmStdLibSources = [
-    'lib/src/wasm-stdlib/external_scanner_allocator.c',
-    'lib/src/wasm-stdlib/imports.txt',
-    'lib/src/wasm-stdlib/libc.c',
-    'lib/src/wasm-stdlib/stdio.c'
-  ];
-  const dirChanged = changedFiles.some(file =>
-    wasmStdLibSources.includes(file) ||
-    file.startsWith('lib/src/wasm-stdlib/libc/ctype/') ||
-    file.startsWith('lib/src/wasm-stdlib/libc/string/')
-  );
-
-  if (!dirChanged) return;
-
-  const wasmStdLibHeader = 'lib/src/wasm-stdlib/external_scanner_stdlib.h';
-  const requiredChanged = changedFiles.includes(wasmStdLibHeader);
-
-  if (!requiredChanged) core.setFailed(`Changes detected in the Wasm stdlib sources but ${wasmStdLibHeader} was not modified.`);
-};

.github/workflows/backport.yml

@@ -1,33 +1,27 @@
-name: Backport Pull Request
-
+name: backport
 on:
   pull_request_target:
     types: [closed, labeled]
-
-permissions:
-  contents: write
-  pull-requests: write
-
 jobs:
   backport:
+    permissions:
+      contents: write
+      pull-requests: write
+    name: Backport Pull Request
     if: github.event.pull_request.merged
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-        with:
-          persist-credentials: true
-          ref: ${{ github.event.pull_request.base.ref }}
+      - uses: actions/checkout@v4
 
-      - name: Create app token
-        uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v1
         id: app-token
         with:
           app-id: ${{ vars.BACKPORT_APP }}
           private-key: ${{ secrets.BACKPORT_KEY }}
 
       - name: Create backport PR
-        uses: korthout/backport-action@v4.6.0
+        id: backport
+        uses: korthout/backport-action@v3
         with:
           pull_title: "${pull_title}"
           label_pattern: "^ci:backport ([^ ]+)$"

.github/workflows/bindgen.yml

@@ -1,30 +0,0 @@
-name: Check Bindgen Output
-
-on:
-  pull_request:
-    paths:
-      - lib/include/tree_sitter/api.h
-      - lib/binding_rust/bindings.rs
-  push:
-    branches: [master]
-    paths:
-      - lib/include/tree_sitter/api.h
-      - lib/binding_rust/bindings.rs
-
-jobs:
-  check-bindgen:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Set up stable Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Generate bindings
-        run: cargo xtask generate-bindings
-
-      - name: Check if the bindgen output changed
-        run: git diff --exit-code lib/binding_rust/bindings.rs

.github/workflows/build.yml

@@ -1,9 +1,14 @@
 name: Build & Test
 
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+  CROSS_DEBUG: 1
+
 on:
   workflow_call:
     inputs:
-      run-test:
+      run_test:
         default: true
         type: boolean
 
@@ -16,323 +21,168 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - linux-arm64
-          - linux-arm
-          - linux-x64
-          - linux-x86
-          - linux-powerpc64
-          - windows-arm64
-          - windows-x64
-          - windows-x86
-          - macos-arm64
-          - macos-x64
-          - illumos-x64
-          - wasm32
+        - linux-arm64       #
+        - linux-arm         #
+        - linux-x64         #
+        - linux-x86         #
+        - linux-powerpc64   #
+        - windows-arm64     #
+        - windows-x64       # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler
+        - windows-x86       #     -- // --
+        - macos-arm64       #
+        - macos-x64         #
 
         include:
-          # When adding a new `target`:
-          # 1. Define a new platform alias above
-          # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
-          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
-          - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
-          - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
-          - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
-          - { platform: linux-x86       , target: i686-unknown-linux-gnu        , os: ubuntu-24.04     }
-          - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu   , os: ubuntu-24.04     }
-          - { platform: windows-arm64   , target: aarch64-pc-windows-msvc       , os: windows-11-arm   }
-          - { platform: windows-x64     , target: x86_64-pc-windows-msvc        , os: windows-2025     }
-          - { platform: windows-x86     , target: i686-pc-windows-msvc          , os: windows-2025     }
-          - { platform: macos-arm64     , target: aarch64-apple-darwin          , os: macos-15         }
-          - { platform: macos-x64       , target: x86_64-apple-darwin           , os: macos-15-intel   }
-          - { platform: wasm32          , target: wasm32-unknown-unknown        , os: ubuntu-24.04     }
+        # When adding a new `target`:
+        # 1. Define a new platform alias above
+        # 2. Add a new record to a matrix map in `cli/npm/install.js`
+        - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu           , os: ubuntu-latest  , use-cross: true    }
+        - { platform: linux-arm         , target: arm-unknown-linux-gnueabi           , os: ubuntu-latest  , use-cross: true    }
+        - { platform: linux-x64         , target: x86_64-unknown-linux-gnu            , os: ubuntu-20.04   , cli_features: wasm } #2272
+        - { platform: linux-x86         , target: i686-unknown-linux-gnu              , os: ubuntu-latest  , use-cross: true    }
+        - { platform: linux-powerpc64   , target: powerpc64-unknown-linux-gnu         , os: ubuntu-latest  , use-cross: true    }
+        - { platform: windows-arm64     , target: aarch64-pc-windows-msvc             , os: windows-latest                      }
+        - { platform: windows-x64       , target: x86_64-pc-windows-msvc              , os: windows-latest , cli_features: wasm }
+        - { platform: windows-x86       , target: i686-pc-windows-msvc                , os: windows-latest                      }
+        - { platform: macos-arm64       , target: aarch64-apple-darwin                , os: macos-14       , cli_features: wasm }
+        - { platform: macos-x64         , target: x86_64-apple-darwin                 , os: macos-12       , cli_features: wasm }
 
-          # illumos is not supported OOTB, it runs in a vm
-          - { platform: illumos-x64     , target: x86_64-unknown-illumos        , os: ubuntu-24.04     , vm: true , no-run: true }
+        # Cross compilers for C library
+        - { platform: linux-arm64       , cc: aarch64-linux-gnu-gcc             , ar: aarch64-linux-gnu-ar           }
+        - { platform: linux-arm         , cc: arm-linux-gnueabi-gcc             , ar: arm-linux-gnueabi-ar           }
+        - { platform: linux-x86         , cc: i686-linux-gnu-gcc                , ar: i686-linux-gnu-ar              }
+        - { platform: linux-powerpc64   , cc: powerpc64-linux-gnu-gcc           , ar: powerpc64-linux-gnu-ar         }
 
-          # Extra features
-          - { platform: linux-arm64     , features: wasm }
-          - { platform: linux-x64       , features: wasm , run-wasm-test: true }
-          - { platform: macos-arm64     , features: wasm , run-wasm-test: true }
-          - { platform: macos-x64       , features: wasm }
+        # See #2041 tree-sitter issue
+        - { platform: windows-x64   , rust-test-threads: 1 }
+        - { platform: windows-x86   , rust-test-threads: 1 }
 
-          # Cross-compilation
-          - { platform: linux-arm       , cross: true }
-          - { platform: linux-x86       , cross: true }
-          - { platform: linux-powerpc64 , cross: true }
-
-          # Compile-only
-          - { platform: wasm32          , no-run: true }
+        # CLI only build
+        - { platform: windows-arm64 , cli-only: true }
 
     env:
-      CARGO_TERM_COLOR: always
-      RUSTFLAGS: -D warnings
+      BUILD_CMD: cargo
+      EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
 
     defaults:
       run:
         shell: bash
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v7.0.1
+    - uses: actions/checkout@v4
 
-    - name: Set up cross-compilation
-      if: matrix.cross
-      run: |
-        for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do
-          camel_target=${target//-/_}; target_cc=${target/-unknown/}
-          printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
-          printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
-          printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
-        done >> $GITHUB_ENV
-        {
-          printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
-          printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
-        } >> $GITHUB_ENV
-
-    - name: Get emscripten version
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
-
-    - name: Cache Emscripten SDK
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      uses: actions/cache@v6
-      with:
-        path: emsdk
-        key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
+    - name: Read Emscripten version
+      run: echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV
 
     - name: Install Emscripten
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      if: ${{ !matrix.cli-only && !matrix.use-cross }}
+      uses: mymindstorm/setup-emsdk@v14
+      with:
+        version: ${{ env.EMSCRIPTEN_VERSION }}
+
+    - run: rustup toolchain install stable --profile minimal
+    - run: rustup target add ${{ matrix.target }}
+    - uses: Swatinem/rust-cache@v2
+
+    - name: Install cross
+      if: ${{ matrix.use-cross }}
+      uses: taiki-e/install-action@v2
+      with:
+        tool: cross
+
+    - name: Build custom cross image
+      if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }}
       run: |
-        if [[ ! -d emsdk ]]; then
-          git clone --depth 1 https://github.com/emscripten-core/emsdk.git
+        target="${{ matrix.target }}"
+        image=ghcr.io/cross-rs/$target:custom
+        echo "CROSS_IMAGE=$image"                              >> $GITHUB_ENV
+
+        echo "[target.$target]"                                >> Cross.toml
+        echo "image = \"$image\""                              >> Cross.toml
+        echo "CROSS_CONFIG=$PWD/Cross.toml"                    >> $GITHUB_ENV
+
+        echo "FROM ghcr.io/cross-rs/$target:edge"                            >> Dockerfile
+        echo "RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -" >> Dockerfile
+        echo "RUN apt-get update && apt-get -y install nodejs"               >> Dockerfile
+        docker build -t $image .
+
+    - name: Setup env extras
+      env:
+        RUST_TEST_THREADS: ${{ matrix.rust-test-threads }}
+        USE_CROSS: ${{ matrix.use-cross }}
+        TARGET: ${{ matrix.target }}
+        CC: ${{ matrix.cc }}
+        AR: ${{ matrix.ar }}
+      run: |
+        PATH="$PWD/.github/scripts:$PATH"
+        echo "$PWD/.github/scripts" >> $GITHUB_PATH
+
+        echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
+        echo "TARGET=$TARGET" >> $GITHUB_ENV
+        echo "ROOT=$PWD" >> $GITHUB_ENV
+
+        [ -n "$RUST_TEST_THREADS" ] && \
+        echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV
+
+        [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
+        [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
+
+        if [ "$USE_CROSS" == "true" ]; then
+          echo "BUILD_CMD=cross" >> $GITHUB_ENV
+          runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
+          [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
         fi
-        cd emsdk
-        ./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
-        ./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
-        echo "$PWD" >> "$GITHUB_PATH"
-        echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
-        echo "EMSDK=$PWD" >> "$GITHUB_ENV"
-        echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
 
-    - name: Set up Node.js
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      uses: actions/setup-node@v7.0.0
-      with:
-        node-version: 24
-        cache: npm
-        cache-dependency-path: lib/binding_web/package-lock.json
+    - name: Build C library
+      if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
+      run: make.sh -j CFLAGS="-Werror"
 
-    - name: Set up Rust
-      if: ${{ !matrix.vm }}
-      uses: actions-rust-lang/setup-rust-toolchain@v1
-      with:
-        target: ${{ matrix.target }}
+    - name: Build wasm library
+      if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times
+      run: script/build-wasm
 
-    - name: Install Rust Wasm test target
-      if: matrix.run-wasm-test
-      run: rustup toolchain install nightly --profile minimal --component rust-src
+    - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
 
-    - name: Install cross-compilation toolchain
-      if: matrix.cross
-      run: |
-        sudo apt-get update -qy
-        if [[ $PLATFORM == linux-arm ]]; then
-          sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user
-        elif [[ $PLATFORM == linux-x86 ]]; then
-          sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu
-        elif [[ $PLATFORM == linux-powerpc64 ]]; then
-          sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user
-        fi
-      env:
-        PLATFORM: ${{ matrix.platform }}
+    - run: script/fetch-fixtures
 
-    - name: Install MinGW and Clang (Windows x64 MSYS2)
-      if: matrix.platform == 'windows-x64'
-      uses: msys2/setup-msys2@v2
-      with:
-        install: |
-          mingw-w64-x86_64-toolchain
-          mingw-w64-x86_64-clang
-          mingw-w64-x86_64-make
-          mingw-w64-x86_64-cmake
-
-    - name: Build C library (Windows x64 MSYS2 CMake)
-      if: matrix.platform == 'windows-x64'
-      shell: msys2 {0}
-      run: |
-        cmake -G Ninja -S . -B build/static \
-          -DBUILD_SHARED_LIBS=OFF \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-          -DTREE_SITTER_FEATURE_WASM=$WASM \
-          -DCMAKE_C_COMPILER=clang
-        cmake --build build/static
-
-        cmake -G Ninja -S . -B build/shared \
-          -DBUILD_SHARED_LIBS=ON \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-          -DTREE_SITTER_FEATURE_WASM=$WASM \
-          -DCMAKE_C_COMPILER=clang
-        cmake --build build/shared
-        rm -rf \
-          build/{static,shared} \
-          "${CMAKE_PREFIX_PATH}/artifacts" \
-          target/wasmtime-${WASMTIME_VERSION}
-      env:
-        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
-
-    - name: Download wasmtime C API
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      run: |
-        mkdir -p target
-        WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
-                           jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        case '${{ matrix.target }}' in
-          x86_64-unknown-linux-gnu)  WT_TARGET=x86_64-linux ;;
-          aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
-          x86_64-apple-darwin)       WT_TARGET=x86_64-macos ;;
-          aarch64-apple-darwin)      WT_TARGET=aarch64-macos ;;
-        esac
-        curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
-          | tar xJf - -C target
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
-      env:
-        WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
-
-    - name: Build C library (make)
-      if: runner.os != 'Windows' && !matrix.vm
-      run: |
-        if [[ $PLATFORM == linux-arm ]]; then
-          CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar
-        elif [[ $PLATFORM == linux-x86 ]]; then
-          CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar
-        elif [[ $PLATFORM == linux-powerpc64 ]]; then
-          CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar
-        else
-          CC=gcc; AR=ar
-        fi
-        make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
-      env:
-        PLATFORM: ${{ matrix.platform }}
-        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
-
-    - name: Build C library (CMake)
-      if: "!matrix.cross && !matrix.vm"
-      run: |
-        cmake -S . -B build/static \
-          -DBUILD_SHARED_LIBS=OFF \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-          -DTREE_SITTER_FEATURE_WASM=$WASM
-        cmake --build build/static --verbose
-
-        cmake -S . -B build/shared \
-          -DBUILD_SHARED_LIBS=ON \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-          -DTREE_SITTER_FEATURE_WASM=$WASM
-        cmake --build build/shared --verbose
-      env:
-        CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
-        WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
-
-    - name: Build C library and Rust crate (illumos gmake)
-      if: matrix.platform == 'illumos-x64'
-      uses: vmactions/omnios-vm@v1.3.6
-      with:
-        release: r151056-build
-        copyback: false
-        prepare: |
-          pkg install -q build-essential || [ $? -eq 4 ]
-          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
-        run: |
-          . "$HOME/.cargo/env"
-          gmake -j
-          cargo build -p tree-sitter
-
-    - name: Build Wasm library
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      shell: bash
-      run: |
-        cd lib/binding_web
-        npm ci
-        CJS=true npm run build
-        CJS=true npm run build:debug
-        npm run build
-        npm run build:debug
-
-    - name: Check no_std builds
-      if: inputs.run-test && !matrix.no-run
-      working-directory: lib
-      shell: bash
-      run: cargo check --no-default-features --target='${{ matrix.target }}'
-
-    - name: Build target
-      if: "!inputs.run-test && !matrix.vm"
-      run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
-      env:
-        PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
-
-    - name: Cache fixtures
+    - uses: ./.github/actions/cache
       id: cache
-      if: inputs.run-test && !matrix.no-run
-      uses: ./.github/actions/cache
-
-    - name: Fetch fixtures
-      if: inputs.run-test && !matrix.no-run
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures
 
     - name: Generate fixtures
-      if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true'
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
+      if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host
+      run: script/generate-fixtures
 
-    - name: Generate Wasm fixtures
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
+    - name: Generate WASM fixtures
+      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step
+      run: script/generate-fixtures-wasm
 
     - name: Run main tests
-      if: inputs.run-test && !matrix.no-run
-      run: cargo test --workspace --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
+      if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host
+      run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
 
-    - name: Run Wasm tests
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
+    - name: Run wasm tests
+      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step
+      run: script/test-wasm
 
-    - name: Run Rust Wasm web test
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-rust-wasm-web
+    - name: Run benchmarks
+      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense
+      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
 
     - name: Upload CLI artifact
-      if: "!inputs.run-test && !matrix.no-run"
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: tree-sitter.${{ matrix.platform }}
-        path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }}
         if-no-files-found: error
         retention-days: 7
 
-    - name: Upload Wasm artifacts
-      if: "!inputs.run-test && matrix.platform == 'linux-x64'"
-      uses: actions/upload-artifact@v7
+    - name: Upload WASM artifacts
+      if: ${{ matrix.platform == 'linux-x64' }}
+      uses: actions/upload-artifact@v4
       with:
         name: tree-sitter.wasm
         path: |
-          lib/binding_web/web-tree-sitter.js
-          lib/binding_web/web-tree-sitter.js.map
-          lib/binding_web/web-tree-sitter.cjs
-          lib/binding_web/web-tree-sitter.cjs.map
-          lib/binding_web/web-tree-sitter.wasm
-          lib/binding_web/web-tree-sitter.wasm.map
-          lib/binding_web/debug/web-tree-sitter.cjs
-          lib/binding_web/debug/web-tree-sitter.cjs.map
-          lib/binding_web/debug/web-tree-sitter.js
-          lib/binding_web/debug/web-tree-sitter.js.map
-          lib/binding_web/debug/web-tree-sitter.wasm
-          lib/binding_web/debug/web-tree-sitter.wasm.map
-          lib/binding_web/lib/*.c
-          lib/binding_web/lib/*.h
-          lib/binding_web/lib/*.ts
-          lib/binding_web/src/*.ts
+          lib/binding_web/tree-sitter.js
+          lib/binding_web/tree-sitter.wasm
         if-no-files-found: error
         retention-days: 7

.github/workflows/ci.yml

@@ -1,14 +1,9 @@
 name: CI
-
 on:
+  pull_request:
   push:
     branches:
       - 'master'
-      - 'release-[0-9]+.[0-9]+'
-  pull_request:
-    branches:
-      - 'master'
-      - 'release-[0-9]+.[0-9]+'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,34 +13,15 @@ jobs:
   checks:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Set up stable Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: stable
-          components: clippy, rustfmt
-
-      - name: Lint Rust files
-        run: make lint
-
-      - name: Install Taplo
-        uses: taiki-e/install-action@v2
-        with:
-          tool: taplo@0.10.0
-
-      - name: Lint TOML files
-        run: make lint-toml
-
-      - name: Lint web files
-        run: make lint-web
+    - uses: actions/checkout@v4
+    - run: rustup toolchain install stable --profile minimal
+    - run: rustup toolchain install nightly --profile minimal
+    - run: rustup component add --toolchain nightly rustfmt
+    - uses: Swatinem/rust-cache@v2
+    - run: make lint
 
   sanitize:
     uses: ./.github/workflows/sanitize.yml
 
   build:
     uses: ./.github/workflows/build.yml
-
-  check-wasm-stdlib:
-    uses: ./.github/workflows/wasm_stdlib.yml

.github/workflows/crate_versions.yml

@@ -1,22 +0,0 @@
-name: Crate Versions Check
-
-on:
-  pull_request:
-    types: [labeled, opened, synchronize, reopened]
-  workflow_dispatch:
-
-jobs:
-  check-crates:
-    runs-on: ubuntu-latest
-    if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Check crates against crates.io
-        uses: katyo/publish-crates@v2
-        with:
-          dry-run: true

.github/workflows/docs.yml

@@ -1,44 +0,0 @@
-name: Deploy Docs
-on:
-  push:
-    branches: [master]
-    paths: [docs/**]
-  workflow_dispatch:
-
-jobs:
-  deploy-docs:
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: write
-      pages: write
-      id-token: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Install mdbook
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
-          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.5.4 --jq "$jq_expr")
-          mkdir mdbook
-          curl -sSL "$url" | tar -xz -C mdbook
-          printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
-
-      - name: Build Book
-        run: mdbook build docs
-
-      - name: Setup Pages
-        uses: actions/configure-pages@v6
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v5
-        with:
-          path: docs/book
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v5

.github/workflows/nvim_ts.yml

@@ -1,96 +0,0 @@
-name: nvim-treesitter parser tests
-
-on:
-  pull_request:
-    paths:
-      - 'lib/**'
-      - 'crates/cli/**'
-      - 'crates/config/**'
-      - 'crates/generate/**'
-      - 'crates/loader/**'
-      - '.github/workflows/nvim_ts.yml'
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  check_compilation:
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        type: [generate, build]
-    name: ${{ matrix.os }} - ${{ matrix.type }}
-    runs-on: ${{ matrix.os }}
-    env:
-      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim.exe' || 'nvim' }}
-      NVIM_TAG: stable
-      NVIM_DIR: neovim
-      NVIM_TS_DIR: nvim-treesitter
-    steps:
-      - uses: actions/checkout@v7.0.1
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - run: cargo build --profile optimize
-
-      - name: Clone Neovim
-        uses: actions/checkout@v7.0.1
-        with:
-          repository: neovim/neovim
-          ref: ${{ env.NVIM_TAG }}
-          path: ${{ env.NVIM_DIR }}
-
-      - if: runner.os != 'Windows'
-        name: Setup environment (Posix)
-        run: |
-          echo ${{ github.workspace }}/target/optimize >> "$GITHUB_PATH"
-          echo ${{ github.workspace }}/neovim/build/bin >> "$GITHUB_PATH"
-          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$GITHUB_ENV"
-
-      - if: runner.os == 'Windows'
-        name: Setup environment (why can't you just be normal?!)
-        run: |
-          ${{ env.NVIM_DIR }}/.github/scripts/env.ps1
-          echo ${{ github.workspace }}/target/optimize >> "$env:GITHUB_PATH"
-          echo ${{ github.workspace }}/neovim/build/bin >> "$env:GITHUB_PATH"
-          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$env:GITHUB_ENV"
-
-      - name: Build Neovim
-        working-directory: ${{ env.NVIM_DIR }}
-        run: |
-          cmake -S cmake.deps -B .deps -G Ninja -D CMAKE_BUILD_TYPE=Release -D TREESITTER_URL=https://github.com/tree-sitter/tree-sitter/archive/${{ github.event.pull_request.head.sha }}.tar.gz -D DEPS_IGNORE_SHA=TRUE
-          cmake --build .deps --config Release
-          cmake -B build -G Ninja -D CMAKE_BUILD_TYPE=Release
-          cmake --build build --config Release
-
-      - name: Clone nvim-treesitter
-        uses: actions/checkout@v7.0.1
-        with:
-          repository: nvim-treesitter/nvim-treesitter
-          path: ${{ env.NVIM_TS_DIR }}
-
-      - if: matrix.type == 'generate'
-        name: Generate and compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=10
-        working-directory: ${{ env.NVIM_TS_DIR }}
-        shell: bash
-
-      - if: matrix.type == 'build'
-        name: Compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10
-        working-directory: ${{ env.NVIM_TS_DIR }}
-        shell: bash
-
-      - if: "!cancelled()"
-        name: Test parsers
-        run: $NVIM -l ./scripts/check-parsers.lua
-        working-directory: ${{ env.NVIM_TS_DIR }}
-        shell: bash
-
-      - if: "!cancelled()"
-        name: Test queries
-        run: $NVIM -l ./scripts/check-queries.lua
-        working-directory: ${{ env.NVIM_TS_DIR }}
-        shell: bash

.github/workflows/release.yml

@@ -1,14 +1,6 @@
 name: Release
-
 on:
-  schedule:
-    - cron: '5 5 * * *'
   workflow_dispatch:
-    inputs:
-      tag_name:
-        description: 'Tag name for release'
-        required: false
-        default: nightly
   push:
     tags:
       - v[0-9]+.[0-9]+.[0-9]+
@@ -17,33 +9,19 @@ jobs:
   build:
     uses: ./.github/workflows/build.yml
     with:
-      run-test: false
+      run_test: false
 
   release:
-    name: Release on GitHub
+    name: Release
     runs-on: ubuntu-latest
     needs: build
     permissions:
-      id-token: write
-      attestations: write
       contents: write
     steps:
-      - if: github.event_name == 'workflow_dispatch'
-        env:
-          TAG_NAME: ${{ github.event.inputs.tag_name }}
-        run: echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_ENV
-
-      - if: github.event_name == 'schedule'
-        run: echo 'TAG_NAME=nightly' >> $GITHUB_ENV
-
-      - if: github.event_name == 'push'
-        run: echo "TAG_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
-
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+      - uses: actions/checkout@v4
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 
@@ -53,109 +31,70 @@ jobs:
 
       - name: Prepare release artifacts
         run: |
-          mkdir -p target web
-          mv artifacts/tree-sitter.wasm/* web/
-
-          tar -czf target/web-tree-sitter.tar.gz -C web .
-
+          mkdir -p target
+          mv artifacts/tree-sitter.wasm/* target/
           rm -r artifacts/tree-sitter.wasm
-
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
-            chmod +x $exe
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
-            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
 
-      - name: Generate attestations
-        uses: actions/attest-build-provenance@v4
-        with:
-          subject-path: |
-            target/tree-sitter-*.gz
-            target/tree-sitter-cli-*.zip
-            target/web-tree-sitter.tar.gz
-
-      - if: env.TAG_NAME == 'nightly'
-        run: |
-          echo 'PRERELEASE=--prerelease' >> $GITHUB_ENV
-          gh release delete nightly --yes || true
-          git push https://${GITHUB_ACTOR}:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY} :nightly || true
-        env:
-          GH_TOKEN: ${{ github.token }}
-
       - name: Create release
-        run: |-
-          gh release create ${{ env.TAG_NAME }} $PRERELEASE \
-            target/tree-sitter-*.gz \
-            target/tree-sitter-cli-*.zip \
-            target/web-tree-sitter.tar.gz
-        env:
-          GH_TOKEN: ${{ github.token }}
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ github.ref_name }}
+          tag_name: ${{ github.ref_name }}
+          fail_on_unmatched_files: true
+          files: |
+            target/tree-sitter-*.gz
+            target/tree-sitter.wasm
+            target/tree-sitter.js
 
   crates_io:
-    name: Publish packages to Crates.io
-    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
+    name: Publish CLI to Crates.io
     runs-on: ubuntu-latest
-    environment: crates
-    permissions:
-      id-token: write
-      contents: read
     needs: release
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+      - uses: actions/checkout@v4
 
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Set up registry token
-        id: auth
-        uses: rust-lang/crates-io-auth-action@v1
+      - name: Setup Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
 
       - name: Publish crates to Crates.io
         uses: katyo/publish-crates@v2
         with:
-          registry-token: ${{ steps.auth.outputs.token }}
+          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
 
   npm:
-    name: Publish packages to npmjs.com
-    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
+    name: Publish lib to npmjs.com
     runs-on: ubuntu-latest
-    environment: npm
-    permissions:
-      id-token: write
-      contents: read
     needs: release
     strategy:
       fail-fast: false
       matrix:
-        directory: [crates/cli/npm, lib/binding_web]
+        directory: ["cli/npm", "lib/binding_web"]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Set up Node
-        uses: actions/setup-node@v7.0.0
-        with:
-          node-version: 24
-          registry-url: https://registry.npmjs.org
-
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses: actions/checkout@v4
 
       - name: Build wasm
         if: matrix.directory == 'lib/binding_web'
+        run: ./script/build-wasm
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Publish lib to npmjs.com
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
         run: |
           cd ${{ matrix.directory }}
-          npm ci
-          npm run build
-          npm run build:debug
-          CJS=true npm run build
-          CJS=true npm run build:debug
-          npm run build:dts
-
-      - name: Publish to npmjs.com
-        working-directory: ${{ matrix.directory }}
-        run: npm publish
+          npm publish

.github/workflows/response.yml

@@ -1,47 +1,34 @@
-name: No response
-
+name: no_response
 on:
   schedule:
-    - cron: "30 1 * * *" # Run every day at 01:30
+    - cron: '30 1 * * *' # Run every day at 01:30
   workflow_dispatch:
   issue_comment:
 
-permissions:
-  issues: write
-  pull-requests: write
-
 jobs:
   close:
-    name: Close issues with no response
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-      - name: Checkout script
-        uses: actions/checkout@v7.0.1
-        with:
-          sparse-checkout: .github/scripts/close_unresponsive.js
-          sparse-checkout-cone-mode: false
-
-      - name: Run script
-        uses: actions/github-script@v9
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
         with:
           script: |
             const script = require('./.github/scripts/close_unresponsive.js')
             await script({github, context})
 
   remove_label:
-    name: Remove response label
     if: github.event_name == 'issue_comment'
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-      - name: Checkout script
-        uses: actions/checkout@v7.0.1
-        with:
-          sparse-checkout: .github/scripts/remove_response_label.js
-          sparse-checkout-cone-mode: false
-
-      - name: Run script
-        uses: actions/github-script@v9
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
         with:
           script: |
             const script = require('./.github/scripts/remove_response_label.js')

.github/workflows/reviewers_remove.yml

@@ -1,31 +1,17 @@
-name: Remove Reviewers
-
+name: "reviewers: remove"
 on:
   pull_request_target:
     types: [converted_to_draft, closed]
-
-permissions:
-  pull-requests: write
-
 jobs:
   remove-reviewers:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
     steps:
-      - name: Remove reviewers
-        uses: actions/github-script@v9
+      - uses: actions/checkout@v4
+      - name: 'Remove reviewers'
+        uses: actions/github-script@v7
         with:
           script: |
-            const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.issue.number,
-            });
-
-            const reviewers = requestedReviewers.data.users.map((e) => e.login);
-
-            github.rest.pulls.removeRequestedReviewers({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.issue.number,
-              reviewers: reviewers,
-            });
+            const script = require('./.github/scripts/reviewers_remove.js')
+            await script({github, context})

.github/workflows/sanitize.yml

@@ -8,44 +8,39 @@ on:
   workflow_call:
 
 jobs:
-  check-undefined-behaviour:
+  check_undefined_behaviour:
+    name: Sanitizer checks
     runs-on: ubuntu-latest
     timeout-minutes: 20
     env:
       TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+    - name: Checkout source code
+      uses: actions/checkout@v4
 
-      - name: Install UBSAN library
-        run: sudo apt-get update -y && sudo apt-get install -y libubsan1
+    - name: Install UBSAN library
+      run: sudo apt-get update -y && sudo apt-get install -y libubsan1
 
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+    - run: rustup toolchain install stable --profile minimal
+    - uses: Swatinem/rust-cache@v2
+    - run: cargo build --release
+    - run: script/fetch-fixtures
 
-      - name: Build project
-        run: cargo build --release
+    - uses: ./.github/actions/cache
+      id: cache
 
-      - name: Cache fixtures
-        uses: ./.github/actions/cache
-        id: cache
+    - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      run: script/generate-fixtures
 
-      - name: Fetch fixtures
-        run: cargo xtask fetch-fixtures
+    - name: Run main tests with undefined behaviour sanitizer (UBSAN)
+      env:
+        CFLAGS: -fsanitize=undefined
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
+      run: cargo test -- --test-threads 1
 
-      - name: Generate fixtures
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: cargo xtask generate-fixtures
-
-      - name: Run main tests with undefined behaviour sanitizer (UBSAN)
-        run: cargo test -- --test-threads 1
-        env:
-          CFLAGS: -fsanitize=undefined
-          RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
-
-      - name: Run main tests with address sanitizer (ASAN)
-        run: cargo test -- --test-threads 1
-        env:
-          ASAN_OPTIONS: verify_asan_link_order=0
-          CFLAGS: -fsanitize=address
-          RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing
+    - name: Run main tests with address sanitizer (ASAN)
+      env:
+        ASAN_OPTIONS: verify_asan_link_order=0
+        CFLAGS: -fsanitize=address
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing
+      run: cargo test -- --test-threads 1

.github/workflows/spam.yml

@@ -1,29 +0,0 @@
-name: Close as spam
-
-on:
-  issues:
-    types: [labeled]
-  pull_request_target:
-    types: [labeled]
-
-permissions:
-  issues: write
-  pull-requests: write
-
-jobs:
-  spam:
-    runs-on: ubuntu-latest
-    if: github.event.label.name == 'spam'
-    steps:
-      - name: Checkout script
-        uses: actions/checkout@v7.0.1
-        with:
-          sparse-checkout: .github/scripts/close_spam.js
-          sparse-checkout-cone-mode: false
-
-      - name: Run script
-        uses: actions/github-script@v9
-        with:
-          script: |
-            const script = require('./.github/scripts/close_spam.js')
-            await script({github, context})

.github/workflows/wasm_exports.yml

@@ -1,41 +0,0 @@
-name: Check Wasm Exports
-
-on:
-  pull_request:
-    paths:
-      - lib/include/tree_sitter/api.h
-      - lib/binding_web/**
-      - xtask/src/**
-  push:
-    branches: [master]
-    paths:
-      - lib/include/tree_sitter/api.h
-      - lib/binding_rust/bindings.rs
-      - CMakeLists.txt
-
-jobs:
-  check-wasm-exports:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Set up stable Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Install wasm-objdump
-        run: sudo apt-get update -y && sudo apt-get install -y wabt
-
-      - name: Build C library (make)
-        run: make -j CFLAGS="$CFLAGS"
-        env:
-          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
-
-      - name: Build Wasm Library
-        working-directory: lib/binding_web
-        run: npm ci && npm run build:debug
-
-      - name: Check Wasm exports
-        run: cargo xtask check-wasm-exports

.github/workflows/wasm_stdlib.yml

@@ -1,19 +0,0 @@
-name: Check Wasm Stdlib build
-
-on:
-  workflow_call:
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Check directory changes
-        uses: actions/github-script@v9
-        with:
-          script: |
-            const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
-            const script = require(scriptPath);
-            return script({ github, context, core });

.gitignore

@@ -1,12 +1,10 @@
 log*.html
-.direnv
 
 .idea
 *.xcodeproj
 .vscode
 .cache
 .zig-cache
-.direnv
 
 profile*
 fuzz-results
@@ -14,6 +12,7 @@ test/fuzz/out
 test/fixtures/grammars/*
 !test/fixtures/grammars/.gitkeep
 
+package-lock.json
 node_modules
 
 docs/assets/js/tree-sitter.js
@@ -26,7 +25,6 @@ docs/assets/js/tree-sitter.js
 *.dylib
 *.so
 *.so.[0-9]*
-*.dll
 *.o
 *.obj
 *.exp
@@ -36,5 +34,3 @@ docs/assets/js/tree-sitter.js
 .build
 build
 zig-*
-
-/result

.taplo.toml

@@ -1,19 +0,0 @@
-[formatting]
-column_width = 100
-compact_arrays = false
-reorder_inline_tables = true
-reorder_keys = true
-
-[[rule]]
-include = [ "**/Cargo.toml" ]
-keys = [ "package" ]
-
-[rule.formatting]
-reorder_keys = false
-
-[[rule]]
-include = [ "**/Cargo.toml" ]
-keys = [ "profile" ]
-
-[rule.formatting]
-reorder_keys = false

CHANGELOG.md

@@ -0,0 +1,470 @@
+# Changelog
+
+## [0.23.0] - 2024-08-26
+
+### Breaking
+
+- Introduce tree-sitter-language crate for grammar crates to depend on (<https://github.com/tree-sitter/tree-sitter/pull/3069>)
+- Revert interning of a sequence or choice of a single rule (<https://github.com/tree-sitter/tree-sitter/pull/3548>)
+- **bindings**: Use capsules in python (<https://github.com/tree-sitter/tree-sitter/pull/3384>)
+- **dsl**: Support other JS runtimes (<https://github.com/tree-sitter/tree-sitter/pull/3355>)
+
+### Features
+
+- Add `fuzz` subcommand (<https://github.com/tree-sitter/tree-sitter/pull/3385>)
+- Allow external scanners to use the logger (<https://github.com/tree-sitter/tree-sitter/pull/3204>)
+- **bindings**: Add query constants to python
+- **bindings**: Add node, python, swift tests (<https://github.com/tree-sitter/tree-sitter/pull/3178>)
+- **bindings**: Update npm scripts (<https://github.com/tree-sitter/tree-sitter/pull/3210>)
+- **cli**: Bump unicode data to v15.1.0
+- **cli**: Add debug build flag (<https://github.com/tree-sitter/tree-sitter/pull/3279>)
+- **cli**: Attach helpful context when `grammar.json` cannot be found (<https://github.com/tree-sitter/tree-sitter/pull/3405>)
+- **cli**: Add `--show-fields` flag to `test` command (<https://github.com/tree-sitter/tree-sitter/pull/3502>)
+- **lib**: Add `ts_query_end_byte_for_pattern` (<https://github.com/tree-sitter/tree-sitter/pull/3451>)
+- **lib**: Support no_std
+- **zig**: Update outdated path syntax (<https://github.com/tree-sitter/tree-sitter/pull/3383>)
+
+### Bug Fixes
+
+- Always reset to the first language when iterating over language attributes (<https://github.com/tree-sitter/tree-sitter/pull/3375>)
+- Better error when a supertype rule is invalid (<https://github.com/tree-sitter/tree-sitter/pull/3400>)
+- Intern a sequence or choice of a single element the same as the element itself
+- Do not "absorb" rules that consist of a single terminal if the rule is hidden (<https://github.com/tree-sitter/tree-sitter/pull/2577>)
+- **bindings**: Update go bindings (<https://github.com/tree-sitter/tree-sitter/pull/3544>)
+- **cli**: Installation via authenticated proxy (<https://github.com/tree-sitter/tree-sitter/pull/3414>)
+- **cli**: Dedup `preceding_auxiliary_symbols` (<https://github.com/tree-sitter/tree-sitter/pull/3550>)
+- **dsl**: Improve error message when a rule function returns undefined (<https://github.com/tree-sitter/tree-sitter/pull/3452>)
+- **generate**: Rename `cargo.toml` template (<https://github.com/tree-sitter/tree-sitter/pull/3532>)
+- **go**: Update parser name in binding files, add to docs (<https://github.com/tree-sitter/tree-sitter/pull/3547>)
+- **lib**: A null clock must have `tv_nsec` be 0 as well (<https://github.com/tree-sitter/tree-sitter/pull/3372>)
+- **lib**: Restrict pattern_map optimization when a wildcard step has an immediate first child (<https://github.com/tree-sitter/tree-sitter/pull/3440>)
+- **lib**: An empty root node should not precede an empty range (<https://github.com/tree-sitter/tree-sitter/pull/3450>)
+- **lib**: Fix api header C++ interop (<https://github.com/tree-sitter/tree-sitter/pull/3534>)
+- **make**: Fail properly on Windows (<https://github.com/tree-sitter/tree-sitter/pull/3418>)
+- **rust**: Fetch `CARGO_MANIFEST_DIR` at runtime in build script (<https://github.com/tree-sitter/tree-sitter/pull/3352>)
+- **rust**: Fix new clippy warnings (<https://github.com/tree-sitter/tree-sitter/pull/3491>)
+- **test**: Multi-grammar corpus tests are now in the repo root (<https://github.com/tree-sitter/tree-sitter/pull/3342>)
+- **wasm**: Update test
+
+### Performance
+
+- Hoist out common subexpressions in satisfies_text_predicates (<https://github.com/tree-sitter/tree-sitter/pull/3397>)
+
+### Documentation
+
+- Update changelog
+- Remove duplicate pr # in changelog
+- Add note for bullet
+- Fix syntax highlighting unit testing example (<https://github.com/tree-sitter/tree-sitter/pull/3434>)
+- Add tsserver annotation to example (<https://github.com/tree-sitter/tree-sitter/pull/3460>)
+- Fix tree cursor documentation (<https://github.com/tree-sitter/tree-sitter/pull/3324>)
+- Document rust library features (<https://github.com/tree-sitter/tree-sitter/pull/3395>)
+- Clean up binding & parser lists (<https://github.com/tree-sitter/tree-sitter/pull/3443>)
+
+### Refactor
+
+- Remove ansi_term dependency (<https://github.com/tree-sitter/tree-sitter/pull/3387>)
+- Remove difference dependency (<https://github.com/tree-sitter/tree-sitter/pull/3388>)
+- **scripts**: Clean up bash scripts (<https://github.com/tree-sitter/tree-sitter/pull/3231>)
+
+### Testing
+
+- Modernize scanner files (<https://github.com/tree-sitter/tree-sitter/pull/3340>)
+
+### Build System and CI
+
+- **deps**: bump wasmtime, cc, and wasmparser (<https://github.com/tree-sitter/tree-sitter/pull/3529>
+- **bindings**: Use language version in soname (<https://github.com/tree-sitter/tree-sitter/pull/3308>)
+- **lib**: Include the minor in the soname
+- **loader**: Make dependencies optional (<https://github.com/tree-sitter/tree-sitter/pull/1638>)
+- **swift**: Declare header search path (<https://github.com/tree-sitter/tree-sitter/pull/3474>)
+- **wasm**: Don't minify JS (<https://github.com/tree-sitter/tree-sitter/pull/3380>)
+- **wasm**: Bump emscripten to 3.1.64 (<https://github.com/tree-sitter/tree-sitter/pull/3497>)
+- **wasm**: Support big endian machines (<https://github.com/tree-sitter/tree-sitter/pull/3492>)
+- **zig**: Git ignore updated Zig cache directory (<https://github.com/tree-sitter/tree-sitter/pull/3408>)
+
+### Other
+
+- Swap `sprintf()` for `snprintf()` (<https://github.com/tree-sitter/tree-sitter/pull/3430>)
+- Add `.build` to gitignore (<https://github.com/tree-sitter/tree-sitter/pull/3498>)
+- Reset language when resetting wasm store (<https://github.com/tree-sitter/tree-sitter/pull/3495>)
+- Clone wasm store engine (<https://github.com/tree-sitter/tree-sitter/pull/3542>)
+- **bindings**: Fix indent & line endings (<https://github.com/tree-sitter/tree-sitter/pull/3284>)
+
+## [0.22.6] — 2024-05-05
+
+### Features
+
+- Improve handling of serialization buffer overflows (<https://github.com/tree-sitter/tree-sitter/pull/3318>)
+- Reverse iteration through node parents (<https://github.com/tree-sitter/tree-sitter/pull/3214>)
+- **cli**: Support `NO_COLOR` (<https://github.com/tree-sitter/tree-sitter/pull/3299>)
+- **cli**: Add test listing and allow users to parse a specific test number (<https://github.com/tree-sitter/tree-sitter/pull/3067>)
+- **grammar**: Add "inherits" field if available (<https://github.com/tree-sitter/tree-sitter/pull/3295>)
+
+### Bug Fixes
+
+- Correctly load field data from wasm languages
+- Improve error message when the `tree-sitter` field is malformed
+- Don't error out on package.json lookup errors if `--no-bindings` is passed (<https://github.com/tree-sitter/tree-sitter/pull/3323>)
+- **cli**: Keep default cc flags in build
+- **cli**: Properly account for multi-grammar repos when using docker to build a wasm parser (<https://github.com/tree-sitter/tree-sitter/pull/3337>)
+- **generate**: Don't check arbitrarily named dirs
+- **generate**: Take `AsRef<Path>` for the path parameter to avoid clones (<https://github.com/tree-sitter/tree-sitter/pull/3322>)
+- **highlight**: Correct signature of `ts_highlighter_add_language`
+- **lib**: Do not return field names for extras (<https://github.com/tree-sitter/tree-sitter/pull/3330>)
+- **lib**: Advance the lookahead end byte by 4 when there's an invalid code point (<https://github.com/tree-sitter/tree-sitter/pull/3305>)
+- **rust**: Update README example (<https://github.com/tree-sitter/tree-sitter/pull/3307>)
+- **rust**: Use unix + wasi cfg instead of not windows for fd (<https://github.com/tree-sitter/tree-sitter/pull/3304>)
+- **test**: Allow newlines in between test name and attribute
+- **wasm**: Correct `childrenFromFieldXXX` method signatures (<https://github.com/tree-sitter/tree-sitter/pull/3301>)
+- **xtask**: Always bump every crate in tandem
+- **zig**: Make usable as a zig dependency (<https://github.com/tree-sitter/tree-sitter/pull/3315>)
+
+### Documentation
+
+- Mention build command variables
+- Swap `\s` for `\\s` in query example
+- **highlight**: Typo (<https://github.com/tree-sitter/tree-sitter/pull/3290>)
+
+### Refactor
+
+- **tests**: Migrate remaining `grammar.json` tests to `grammar.js` (<https://github.com/tree-sitter/tree-sitter/pull/3325>)
+
+### Build System and CI
+
+- Add nightly rustfmt to workflow for linting (<https://github.com/tree-sitter/tree-sitter/pull/3333>)
+- Fix address sanitizer step (<https://github.com/tree-sitter/tree-sitter/pull/3188>)
+- **deps**: Bump cc from 1.0.92 to 1.0.94 in the cargo group (<https://github.com/tree-sitter/tree-sitter/pull/3298>)
+- **deps**: Bump the cargo group with 6 updates (<https://github.com/tree-sitter/tree-sitter/pull/3313>)
+- **xtask**: Bump `build.zig.zon` version when bumping versions
+
+## [0.22.5] — 2024-04-14
+
+### Bug Fixes
+
+- Avoid generating unused character set constants
+- **cli**: Test parsing on windows (<https://github.com/tree-sitter/tree-sitter/pull/3289>)
+- **rust**: Compilation on wasm32-wasi (<https://github.com/tree-sitter/tree-sitter/pull/3293>)
+
+## [0.22.4] — 2024-04-12
+
+### Bug Fixes
+
+- Fix sorting of transitions within a lex state
+- Include 2-character ranges in array-based state transitions
+
+### Build System and CI
+
+- Always bump at least the patch version in bump xtask
+
+## [0.22.3] — 2024-04-12
+
+### Features
+
+- Add strncat to wasm stdlib
+- Generate simpler code for matching large character sets (<https://github.com/tree-sitter/tree-sitter/pull/3234>)
+- When loading languages via WASM, gracefully handle memory errors and leaks in external scanners (<https://github.com/tree-sitter/tree-sitter/pull/3181>)
+
+### Bug Fixes
+
+- **bindings**: Add utf-8 flag to python & node (<https://github.com/tree-sitter/tree-sitter/pull/3278>)
+- **bindings**: Generate parser.c if missing (<https://github.com/tree-sitter/tree-sitter/pull/3277>)
+- **bindings**: Remove required platforms for swift (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
+- **cli**: Fix mismatched parenthesis when accounting for `&&` (<https://github.com/tree-sitter/tree-sitter/pull/3274>)
+- **lib**: Do not consider childless nodes for ts_node_parent (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
+- **lib**: Properly account for aliased root nodes and root nodes with
+  children in `ts_subtree_string` (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
+- **lib**: Account for the root node of a tree cursor being an alias (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
+- **lib**: Use correct format specifier in log message (<https://github.com/tree-sitter/tree-sitter/pull/3255>)
+- **parser**: Fix variadic macro (<https://github.com/tree-sitter/tree-sitter/pull/3229>)
+- render: Proper function prototypes (<https://github.com/tree-sitter/tree-sitter/pull/3277>)
+- **windows**: Add `/utf-8` flag for parsers using unicode symbols (<https://github.com/tree-sitter/tree-sitter/pull/3223>)
+- Add a semicolon after SKIP macros (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
+- Add back `build-wasm` temporarily (<https://github.com/tree-sitter/tree-sitter/pull/3203>)
+- Add lifetime to matches function (<https://github.com/tree-sitter/tree-sitter/pull/3254>)
+- Default output directory for `build --wasm` should use current_dir (<https://github.com/tree-sitter/tree-sitter/pull/3203>)
+- Fix sorting of wasm stdlib symbols
+- Insert "tree-sitter" section in current directory's package.json if it exists (<https://github.com/tree-sitter/tree-sitter/pull/3224>)
+- Tie the lifetime of the cursor to the query in `QueryCursor::captures()` (<https://github.com/tree-sitter/tree-sitter/pull/3266>)
+- Wrong flag check in `build.rs`
+
+### Performance
+
+- **cli**: Reduced the compile time of generated parsers by generating C code with fewer conditionals (<https://github.com/tree-sitter/tree-sitter/pull/3234>)
+
+### Documentation
+
+- Add NGINX grammar
+
+### Refactor
+
+- **parser**: Make REDUCE macro non-variadic (<https://github.com/tree-sitter/tree-sitter/pull/3280>)
+- **js**: Misc fixes & tidying
+- **rust**: Misc fixes & tidying
+
+### Testing
+
+- Add regression test for node parent + string bug (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
+- **test**: Allow colons in test names (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
+
+### Build System and CI
+
+- Upgrade wasmtime
+- Update emscripten version (<https://github.com/tree-sitter/tree-sitter/pull/3272>)
+- **dependabot**: Improve PR labels (<https://github.com/tree-sitter/tree-sitter/pull/3282>)
+
+## [0.22.2] — 2024-03-17
+
+### Breaking
+
+- **cli**: Add a separate build command to compile parsers
+
+### Features
+
+- **bindings/rust**: Expose `Parser::included_ranges`
+- Lower the lib's MSRV (<https://github.com/tree-sitter/tree-sitter/pull/3169>)
+- **lib**: Implement Display for Node (<https://github.com/tree-sitter/tree-sitter/pull/3177>)
+
+### Bug Fixes
+
+- **bindings/wasm**: Fix `Parser.getIncludedRanges()` (<https://github.com/tree-sitter/tree-sitter/pull/3164>)
+- **lib**: Makefile installation on macOS (<https://github.com/tree-sitter/tree-sitter/pull/3167>)
+- **lib**: Makefile installation (<https://github.com/tree-sitter/tree-sitter/pull/3173>)
+- **lib**: Avoid possible UB of calling memset on a null ptr when 0 is passed into `array_grow_by` (<https://github.com/tree-sitter/tree-sitter/pull/3176>)
+- **lib**: Allow hiding symbols (<https://github.com/tree-sitter/tree-sitter/pull/3180>)
+
+### Documentation
+
+- Fix typo (<https://github.com/tree-sitter/tree-sitter/pull/3158>)
+- **licensfe**: Update year (<https://github.com/tree-sitter/tree-sitter/pull/3183>)
+
+### Refactor
+
+- Remove dependency on which crate (<https://github.com/tree-sitter/tree-sitter/pull/3172>)
+- Turbofish styling
+
+### Testing
+
+- Fix header writes (<https://github.com/tree-sitter/tree-sitter/pull/3174>)
+
+### Build System and CI
+
+- Simplify workflows (<https://github.com/tree-sitter/tree-sitter/pull/3002>)
+- **lib**: Allow overriding CFLAGS on the commandline (<https://github.com/tree-sitter/tree-sitter/pull/3159>)
+
+## [0.22.1] — 2024-03-10
+
+### Bug Fixes
+
+- Cli build script behavior on release
+
+## [0.22.0] — 2024-03-10
+
+### Breaking
+
+- Remove top-level `corpus` dir for tests
+  The cli will now only look in `test/corpus` for tests
+- Remove redundant escape regex & curly brace regex preprocessing (<https://github.com/tree-sitter/tree-sitter/pull/2838>)
+- **bindings**: Convert node bindings to NAPI (<https://github.com/tree-sitter/tree-sitter/pull/3077>)
+- **wasm**: Make `current*`, `is*`, and `has*` methods properties (<https://github.com/tree-sitter/tree-sitter/pull/3103>)
+- **wasm**: Keep API in-line with upstream and start aligning with node (<https://github.com/tree-sitter/tree-sitter/pull/3149>)
+
+### Features
+
+- Add xtasks to assist with bumping crates (<https://github.com/tree-sitter/tree-sitter/pull/3065>)
+- Improve language bindings (<https://github.com/tree-sitter/tree-sitter/pull/2438>)
+- Expose the allocator and array header files for external scanners (<https://github.com/tree-sitter/tree-sitter/pull/3063>)
+- Add typings for the node bindings
+- Replace `nan` with `node-addon-api` and conditionally print logs
+- **bindings**: Add more make targets
+- **bindings**: Add peerDependencies for npm
+- **bindings**: Add prebuildify to node
+- **bindings**: Remove dsl types file (<https://github.com/tree-sitter/tree-sitter/pull/3126>)
+- **node**: Type tag the language (<https://github.com/tree-sitter/tree-sitter/pull/3109>)
+- **test**: Add attributes for corpus tests
+
+### Bug Fixes
+
+- Apply some `scan-build` suggestions (unused assignment/garbage access) (<https://github.com/tree-sitter/tree-sitter/pull/3056>)
+- Wrap `||` comparison in parentheses when `&&` is used (<https://github.com/tree-sitter/tree-sitter/pull/3070>)
+- Ignore unused variables in the array macros (<https://github.com/tree-sitter/tree-sitter/pull/3083>)
+- `binding.cc` overwrite should replace `PARSER_NAME` (<https://github.com/tree-sitter/tree-sitter/pull/3116>)
+- Don't use `__declspec(dllexport)` on windows (<https://github.com/tree-sitter/tree-sitter/pull/3128>)
+- Parsers should export the language function on windows
+- Allow the regex `v` flag (<https://github.com/tree-sitter/tree-sitter/pull/3154>)
+- **assertions**: Case shouldn't matter for comment node detection
+- **bindings**: Editorconfig and setup.py fixes (<https://github.com/tree-sitter/tree-sitter/pull/3082>)
+- **bindings**: Insert `types` after `main` if it exists (<https://github.com/tree-sitter/tree-sitter/pull/3122>)
+- **bindings**: Fix template oversights (<https://github.com/tree-sitter/tree-sitter/pull/3155>)
+- **cli**: Only output the sources with `--no-bindings` (<https://github.com/tree-sitter/tree-sitter/pull/3123>)
+- **generate**: Add `.npmignore`, populate Swift's exclude list (<https://github.com/tree-sitter/tree-sitter/pull/3085>)
+- **generate**: Extern allocator functions for the template don't need to be "exported" (<https://github.com/tree-sitter/tree-sitter/pull/3132>)
+- **generate**: Camel case name in `Cargo.toml` description (<https://github.com/tree-sitter/tree-sitter/pull/3140>)
+- **lib**: Include `api.h` so `ts_set_allocator` is visible (<https://github.com/tree-sitter/tree-sitter/pull/3092>)
+
+### Documentation
+
+- Add GitHub user and PR info to the changelog
+- Add css for inline code (<https://github.com/tree-sitter/tree-sitter/pull/2844>)
+- Document test attributes
+- Add `Ohm` language parser
+- Remove duplicate `the`'s (<https://github.com/tree-sitter/tree-sitter/pull/3120>)
+- Add discord and matrix badges (<https://github.com/tree-sitter/tree-sitter/pull/3148>)
+
+### Refactor
+
+- Rename TS_REUSE_ALLOCATOR flag (<https://github.com/tree-sitter/tree-sitter/pull/3088>)
+- Remove extern/const where possible
+- **array**: Use pragma GCC in clang too
+- **bindings**: Remove npmignore (<https://github.com/tree-sitter/tree-sitter/pull/3089>)
+
+### Testing
+
+- Don't use TS_REUSE_ALLOCATOR on Darwin systems (<https://github.com/tree-sitter/tree-sitter/pull/3087>)
+- Add test case for parse stack merging with incorrect error cost bug (<https://github.com/tree-sitter/tree-sitter/pull/3098>)
+
+### Build System and CI
+
+- Improve changelog settings (<https://github.com/tree-sitter/tree-sitter/pull/3064>)
+- Unify crate versions via workspace (<https://github.com/tree-sitter/tree-sitter/pull/3074>)
+- Update `cc` to remove annoying debug output (<https://github.com/tree-sitter/tree-sitter/pull/3075>)
+- Adjust dependabot settings (<https://github.com/tree-sitter/tree-sitter/pull/3079>)
+- Use c11 everywhere
+- Add uninstall command
+- Don't skip tests on failing lint (<https://github.com/tree-sitter/tree-sitter/pull/3102>)
+- Remove unused deps, bump deps, and bump MSRV to 1.74.1 (<https://github.com/tree-sitter/tree-sitter/pull/3153>)
+- **bindings**: Metadata improvements
+- **bindings**: Make everything c11 (<https://github.com/tree-sitter/tree-sitter/pull/3099>)
+- **dependabot**: Update weekly instead of daily (<https://github.com/tree-sitter/tree-sitter/pull/3112>)
+- **deps**: Bump the cargo group with 1 update (<https://github.com/tree-sitter/tree-sitter/pull/3081>)
+- **deps**: Bump the cargo group with 1 update (<https://github.com/tree-sitter/tree-sitter/pull/3097>)
+- **deps**: Bump deps & lockfile (<https://github.com/tree-sitter/tree-sitter/pull/3060>)
+- **deps**: Bump the cargo group with 4 updates (<https://github.com/tree-sitter/tree-sitter/pull/3134>)
+- **lint**: Detect if `Cargo.lock` needs to be updated (<https://github.com/tree-sitter/tree-sitter/pull/3066>)
+- **lint**: Make lockfile check quiet (<https://github.com/tree-sitter/tree-sitter/pull/3078>)
+- **swift**: Move 'cLanguageStandard' behind 'targets' (<https://github.com/tree-sitter/tree-sitter/pull/3101>)
+
+### Other
+
+- Make Node.js language bindings context aware (<https://github.com/tree-sitter/tree-sitter/pull/2841>)
+  They don't have any dynamic global data, so all it takes is just declaring them as such
+- Fix crash when attempting to load ancient languages via wasm (<https://github.com/tree-sitter/tree-sitter/pull/3068>)
+- Use workspace dependencies for internal crates like Tree-sitter (<https://github.com/tree-sitter/tree-sitter/pull/3076>)
+- Remove vendored wasmtime headers (<https://github.com/tree-sitter/tree-sitter/pull/3084>)
+  When building rust binding, use wasmtime headers provided via cargo
+  by the wasmtime-c-api crate.
+- Fix invalid parse stack recursive merging with mismatched error cost (<https://github.com/tree-sitter/tree-sitter/pull/3086>)
+  Allowing this invalid merge caused an invariant to be violated
+  later on during parsing, when handling a later error.
+- Fix regression in `subtree_compare` (<https://github.com/tree-sitter/tree-sitter/pull/3111>)
+- docs: Add `Ohm` language parser (<https://github.com/tree-sitter/tree-sitter/pull/3114>)
+- Delete `binding_files.rs` (<https://github.com/tree-sitter/tree-sitter/pull/3106>)
+- **bindings**: Consistent wording (<https://github.com/tree-sitter/tree-sitter/pull/3096>)
+- **bindings**: Ignore more artifacts (<https://github.com/tree-sitter/tree-sitter/pull/3119>)
+
+## [0.21.0] — 2024-02-21
+
+### Breaking
+
+- Remove the apply-all-captures flag, make last-wins precedence the default
+
+  **NOTE**: This change might cause breakage in your grammar's highlight tests.
+  Just flip the order around of the relevant queries, and keep in mind that the
+  last query that matches will win.
+
+### Features
+
+- Use lockfiles to dedup recompilation
+- Improve error message for files with an unknown grammar path (<https://github.com/tree-sitter/tree-sitter/pull/2475>)
+- Implement first-line-regex (<https://github.com/tree-sitter/tree-sitter/pull/2479>)
+- Error out if an empty string is in the `extras` array
+- Allow specifying an external scanner's files (<https://github.com/tree-sitter/tree-sitter/pull/3031>)
+- Better error info when a scanner is missing required symbols
+- **cli**: Add an optional `grammar-path` argument for the playground (<https://github.com/tree-sitter/tree-sitter/pull/3014>)
+- **cli**: Add optional `config-path` argument (<https://github.com/tree-sitter/tree-sitter/pull/3050>)
+- **loader**: Add more commonly used default parser directories
+
+### Bug Fixes
+
+- Prettify xml output and add node position info (<https://github.com/tree-sitter/tree-sitter/pull/2970>)
+- Inherited grammar generation
+- Properly error out when the word property is an invalid rule
+- Update schema for regex flags (<https://github.com/tree-sitter/tree-sitter/pull/3006>)
+- Properly handle `Query.matches` when filtering out results (<https://github.com/tree-sitter/tree-sitter/pull/3013>)
+- Sexp format edge case with quoted closed parenthesis (<https://github.com/tree-sitter/tree-sitter/pull/3016>)
+- Always push the default files if there's no `externals`
+- Don't log NUL characters (<https://github.com/tree-sitter/tree-sitter/pull/3037>)
+- Don't throw an error if the user uses `map` in the grammar (<https://github.com/tree-sitter/tree-sitter/pull/3041>)
+- Remove redundant imports (<https://github.com/tree-sitter/tree-sitter/pull/3047>)
+- **cli**: Installation via a HTTP tunnel proxy (<https://github.com/tree-sitter/tree-sitter/pull/2824>)
+- **cli**: Don't update tests automatically if parse errors are detected (<https://github.com/tree-sitter/tree-sitter/pull/3033>)
+- **cli**: Don't use `long` for `grammar_path`
+- **test**: Allow writing updates to tests without erroneous nodes instead of denying all of them if a single error is found
+- **test**: Edge case when parsing `UNEXPECTED`/`MISSING` nodes with an indentation level greater than 0
+- **wasm**: Remove C++ mangled symbols (<https://github.com/tree-sitter/tree-sitter/pull/2971>)
+
+### Documentation
+
+- Create issue template (<https://github.com/tree-sitter/tree-sitter/pull/2978>)
+- Document regex limitations
+- Mention that `token($.foo)` is illegal
+- Explicitly mention behavior of walking outside the given "root" node for a `TSTreeCursor` (<https://github.com/tree-sitter/tree-sitter/pull/3021>)
+- Small fixes (<https://github.com/tree-sitter/tree-sitter/pull/2987>)
+- Add `Tact` language parser (<https://github.com/tree-sitter/tree-sitter/pull/3030>)
+- **web**: Provide deno usage information (<https://github.com/tree-sitter/tree-sitter/pull/2498>)
+
+### Refactor
+
+- Extract regex check into a function and lower its precedence
+- `&PathBuf` -> `&Path` (<https://github.com/tree-sitter/tree-sitter/pull/3035>)
+- Name anonymous types in api.h (<https://github.com/tree-sitter/tree-sitter/pull/1659>)
+
+### Testing
+
+- Add quotes around bash variables (<https://github.com/tree-sitter/tree-sitter/pull/3023>)
+- Update html tests
+
+### Build System and CI
+
+- Only create release for normal semver tags (<https://github.com/tree-sitter/tree-sitter/pull/2973>)
+- Add useful development targets to makefile (<https://github.com/tree-sitter/tree-sitter/pull/2979>)
+- Remove minimum glibc information in summary page (<https://github.com/tree-sitter/tree-sitter/pull/2988>)
+- Use the native m1 mac runner (<https://github.com/tree-sitter/tree-sitter/pull/2995>)
+- Add editorconfig (<https://github.com/tree-sitter/tree-sitter/pull/2998>)
+- Remove symbolic links from repository (<https://github.com/tree-sitter/tree-sitter/pull/2997>)
+- Move common Cargo.toml keys into the workspace and inherit them (<https://github.com/tree-sitter/tree-sitter/pull/3019>)
+- Remove reviewers when drafting or closing a PR (<https://github.com/tree-sitter/tree-sitter/pull/2963>)
+- Enable creating changelogs with git-cliff (<https://github.com/tree-sitter/tree-sitter/pull/3040>)
+- Cache fixtures (<https://github.com/tree-sitter/tree-sitter/pull/3038>)
+- Don't cancel jobs on master (<https://github.com/tree-sitter/tree-sitter/pull/3052>)
+- Relax caching requirements (<https://github.com/tree-sitter/tree-sitter/pull/3051>)
+- **deps**: Bump clap from 4.4.18 to 4.5.0 (<https://github.com/tree-sitter/tree-sitter/pull/3007>)
+- **deps**: Bump wasmtime from v16.0.0 to v17.0.1 (<https://github.com/tree-sitter/tree-sitter/pull/3008>)
+- **deps**: Bump wasmtime to v18.0.1 (<https://github.com/tree-sitter/tree-sitter/pull/3057>)
+- **sanitize**: Add a timeout of 60 minutes (<https://github.com/tree-sitter/tree-sitter/pull/3017>)
+- **sanitize**: Reduce timeout to 20 minutes (<https://github.com/tree-sitter/tree-sitter/pull/3054>)
+
+### Other
+
+- Document preferred language for scanner (<https://github.com/tree-sitter/tree-sitter/pull/2972>)
+- Add java and tsx to corpus tests (<https://github.com/tree-sitter/tree-sitter/pull/2992>)
+- Provide a CLI flag to open `log.html` (<https://github.com/tree-sitter/tree-sitter/pull/2996>)
+- Some more clippy lints (<https://github.com/tree-sitter/tree-sitter/pull/3010>)
+- Remove deprecated query parsing mechanism (<https://github.com/tree-sitter/tree-sitter/pull/3011>)
+- Print out full compiler arguments ran when it fails (<https://github.com/tree-sitter/tree-sitter/pull/3018>)
+- Deprecate C++ scanners (<https://github.com/tree-sitter/tree-sitter/pull/3020>)
+- Add some documentation to the playground page (<https://github.com/tree-sitter/tree-sitter/pull/1495>)
+- Update relevant rust tests (<https://github.com/tree-sitter/tree-sitter/pull/2947>)
+- Clippy lints (<https://github.com/tree-sitter/tree-sitter/pull/3032>)
+- Error out when multiple arguments are passed to `token`/`token.immediate` (<https://github.com/tree-sitter/tree-sitter/pull/3036>)
+- Tidying
+- Prefer turbofish syntax where possible (<https://github.com/tree-sitter/tree-sitter/pull/3048>)
+- Use published wasmtime crates
+- Cleaner cast
+- Update `Cargo.lock`
+- Get rid of `github_issue_test` file (<https://github.com/tree-sitter/tree-sitter/pull/3055>)
+- **cli**: Use spawn to display `emcc`'s stdout and stderr (<https://github.com/tree-sitter/tree-sitter/pull/2494>)
+- **cli**: Warn users when a query path needed for a subcommand isn't specified in a grammar's package.json
+- **generate**: Dedup and warn about duplicate or invalid rules (<https://github.com/tree-sitter/tree-sitter/pull/2994>)
+- **test**: Use different languages for async tests (<https://github.com/tree-sitter/tree-sitter/pull/2953>)
+- **wasm**: Use `SIDE_MODULE=2` to silence warning (<https://github.com/tree-sitter/tree-sitter/pull/3003>)

CONTRIBUTING.md

@@ -1 +1 @@
-See [docs/src/6-contributing.md](./docs/src/6-contributing.md)
+See [section-6-contributing.md](./docs/section-6-contributing.md)

Cargo.toml

@@ -1,68 +1,34 @@
 [workspace]
-default-members = [ "crates/cli" ]
+default-members = ["cli"]
 members = [
-  "crates/cli",
-  "crates/config",
-  "crates/generate",
-  "crates/highlight",
-  "crates/loader",
-  "crates/tags",
-  "crates/xtask",
-  "crates/language",
+  "cli",
+  "cli/config",
+  "cli/loader",
   "lib",
+  "lib/language",
+  "tags",
+  "highlight",
+  "xtask",
 ]
 resolver = "2"
 
 [workspace.package]
-authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>" ]
-categories = [ "command-line-utilities", "parsing" ]
-edition = "2024"
+version = "0.24.4"
+authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
+edition = "2021"
+rust-version = "1.74.1"
 homepage = "https://tree-sitter.github.io/tree-sitter"
-keywords = [ "incremental", "parsing" ]
-license = "MIT"
 repository = "https://github.com/tree-sitter/tree-sitter"
-rust-version = "1.90"
-version = "0.28.0"
-
-[workspace.lints.clippy]
-cargo = { level = "warn", priority = -1 }
-dbg_macro = "deny"
-nursery = { level = "warn", priority = -1 }
-pedantic = { level = "warn", priority = -1 }
-todo = "deny"
-
-# The lints below are a specific subset of the pedantic+nursery lints
-# that we explicitly allow in the tree-sitter codebase because they either:
-#
-# 1. Contain false positives,
-# 2. Are unnecessary, or
-# 3. Worsen the code
-
-cast_possible_truncation = "allow"
-cast_possible_wrap = "allow"
-cast_precision_loss = "allow"
-cast_sign_loss = "allow"
-if_not_else = "allow"
-items_after_statements = "allow"
-missing_errors_doc = "allow"
-missing_panics_doc = "allow"
-multiple_crate_versions = "allow"
-option_if_let_else = "allow"
-similar_names = "allow"
-string_lit_as_bytes = "allow"
-struct_excessive_bools = "allow"
-too_many_lines = "allow"
-tuple_array_conversions = "allow"
-
-[workspace.lints.rust]
-mismatched_lifetime_syntaxes = "allow"
+license = "MIT"
+keywords = ["incremental", "parsing"]
+categories = ["command-line-utilities", "parsing"]
 
 [profile.optimize]
 inherits = "release"
-codegen-units = 1    # Maximum size reduction optimizations.
+strip = true         # Automatically strip symbols from the binary.
 lto = true           # Link-time optimization.
 opt-level = 3        # Optimization level 3.
-strip = true         # Automatically strip symbols from the binary.
+codegen-units = 1    # Maximum size reduction optimizations.
 
 [profile.size]
 inherits = "optimize"
@@ -70,69 +36,69 @@ opt-level = "s"       # Optimize for size.
 
 [profile.release-dev]
 inherits = "release"
-codegen-units = 256
+lto = false
 debug = true
 debug-assertions = true
-incremental = true
-lto = false
 overflow-checks = true
+incremental = true
+codegen-units = 256
 
 [workspace.dependencies]
-ansi_colours = "1.2.3"
-anstyle = "1.0.14"
-anyhow = "1.0.102"
-bstr = "1.12.1"
-cc = "1.2.63"
-clap = { features = [
+anstyle = "1.0.8"
+anyhow = "1.0.89"
+bstr = "1.10.0"
+cc = "1.1.22"
+clap = { version = "4.5.18", features = [
   "cargo",
   "derive",
   "env",
   "help",
-  "string",
   "unstable-styles",
-], version = "4.5.58" }
-clap_complete = "4.6.3"
-clap_complete_nushell = "4.5.10"
-crc32fast = "1.5.0"
-ctor = "0.6.3"
-ctrlc = { features = [ "termination" ], version = "3.5.2" }
-dialoguer = { features = [ "fuzzy-select" ], version = "0.12.0" }
-etcetera = "0.11.0"
-glob = "0.3.3"
-hashbrown = { default-features = false, version = "0.17.1" }
+] }
+clap_complete = "4.5.29"
+ctor = "0.2.8"
+ctrlc = { version = "3.4.5", features = ["termination"] }
+dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
+dirs = "5.0.1"
+filetime = "0.2.25"
+fs4 = "0.9.1"
+git2 = "0.19.0"
+glob = "0.3.1"
 heck = "0.5.0"
 html-escape = "0.2.13"
-indexmap = "2.13.0"
-indoc = "2.0.7"
-libloading = "0.9.0"
-log = { features = [ "std" ], version = "0.4.30" }
-memchr = "2.8.1"
-once_cell = "1.21.3"
+indexmap = "2.5.0"
+indoc = "2.0.5"
+lazy_static = "1.5.0"
+libloading = "0.8.5"
+log = { version = "0.4.22", features = ["std"] }
+memchr = "2.7.4"
+once_cell = "1.19.0"
+path-slash = "0.2.1"
 pretty_assertions = "1.4.1"
-rand = "0.10.1"
-regex = "1.12.3"
-regex-syntax = "0.8.9"
-rustc-hash = "2.1.1"
-schemars = "1.2.1"
-semver = { features = [ "serde" ], version = "1.0.27" }
-serde = { features = [ "derive" ], version = "1.0.228" }
-serde_json = { features = [ "preserve_order" ], version = "1.0.150" }
-similar = "2.7.0"
+rand = "0.8.5"
+regex = "1.10.6"
+regex-syntax = "0.8.4"
+rustc-hash = "2.0.0"
+semver = { version = "1.0.23", features = ["serde"] }
+serde = { version = "1.0.210", features = ["derive"] }
+serde_derive = "1.0.210"
+serde_json = { version = "1.0.128", features = ["preserve_order"] }
+similar = "2.6.0"
+smallbitvec = "2.5.3"
 streaming-iterator = "0.1.9"
-tempfile = "3.25.0"
-thiserror = "2.0.18"
+tempfile = "3.12.0"
+thiserror = "1.0.64"
 tiny_http = "0.12.0"
-topological-sort = "0.2.2"
-unindent = "0.2.4"
+toml = "0.8.19"
+unindent = "0.2.3"
+url = { version = "2.5.2", features = ["serde"] }
 walkdir = "2.5.0"
-wasmparser = "0.244.0"
-webbrowser = "1.2.1"
+wasmparser = "0.217.0"
+webbrowser = "1.0.2"
 
-tree-sitter = { path = "./lib", version = "0.28.0" }
-tree-sitter-config = { path = "./crates/config", version = "0.28.0" }
-tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.28.0" }
-tree-sitter-highlight = { path = "./crates/highlight", version = "0.28.0" }
-tree-sitter-loader = { path = "./crates/loader", version = "0.28.0" }
-tree-sitter-tags = { path = "./crates/tags", version = "0.28.0" }
-
-tree-sitter-language = { path = "./crates/language", version = "0.1.8" }
+tree-sitter = { version = "0.24.4", path = "./lib" }
+tree-sitter-generate = { version = "0.24.4", path = "./cli/generate" }
+tree-sitter-loader = { version = "0.24.4", path = "./cli/loader" }
+tree-sitter-config = { version = "0.24.4", path = "./cli/config" }
+tree-sitter-highlight = { version = "0.24.4", path = "./highlight" }
+tree-sitter-tags = { version = "0.24.4", path = "./tags" }

Dockerfile

@@ -0,0 +1,10 @@
+FROM rust:1.76-buster
+
+WORKDIR /app
+
+RUN apt-get update
+RUN apt-get install -y nodejs
+
+COPY . .
+
+CMD cargo test --all-features

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Max Brunsfeld
+Copyright (c) 2018-2024 Max Brunsfeld
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,4 +1,8 @@
-VERSION := 0.28.0
+ifeq ($(OS),Windows_NT)
+$(error Windows is not supported)
+endif
+
+VERSION := 0.24.4
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -6,7 +10,6 @@ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 PREFIX ?= /usr/local
 INCLUDEDIR ?= $(PREFIX)/include
 LIBDIR ?= $(PREFIX)/lib
-BINDIR ?= $(PREFIX)/bin
 PCLIBDIR ?= $(LIBDIR)/pkgconfig
 
 # collect sources
@@ -22,9 +25,8 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -pedantic
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -32,25 +34,20 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
 SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
 
 # OS-specific bits
-MACHINE := $(shell $(CC) -dumpmachine)
-
-ifneq ($(findstring darwin,$(MACHINE)),)
+ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),)
 	SOEXT = dylib
 	SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
 	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
 	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER)
-else ifneq ($(findstring mingw32,$(MACHINE)),)
-	SOEXT = dll
-	LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a
 else
 	SOEXT = so
 	SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
 	SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
 	LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER)
+endif
 ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
 	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
 endif
-endif
 
 all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc
 
@@ -63,10 +60,6 @@ ifneq ($(STRIP),)
 	$(STRIP) $@
 endif
 
-ifneq ($(findstring mingw32,$(MACHINE)),)
-libtree-sitter.dll.a: libtree-sitter.$(SOEXT)
-endif
-
 tree-sitter.pc: lib/tree-sitter.pc.in
 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
 		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
@@ -75,27 +68,17 @@ tree-sitter.pc: lib/tree-sitter.pc.in
 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
 
-shared: libtree-sitter.$(SOEXT)
-
-static: libtree-sitter.a
-
 clean:
-	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a
+	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT)
 
 install: all
 	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
 	install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h
 	install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
 	install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a
-ifneq ($(findstring mingw32,$(MACHINE)),)
-	install -d '$(DESTDIR)$(BINDIR)'
-	install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll
-	install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a
-else
 	install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER)
-	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR)
-	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT)
-endif
+	ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR)
+	ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT)
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \
@@ -104,40 +87,31 @@ uninstall:
 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \
 		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \
 		'$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
-	rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
 
-.PHONY: all shared static install uninstall clean
+.PHONY: all install uninstall clean
 
 
 ##### Dev targets #####
 
 test:
-	cargo xtask fetch-fixtures
-	cargo xtask generate-fixtures
-	cargo xtask test
+	script/fetch-fixtures
+	script/generate-fixtures
+	script/test
 
-test-wasm:
-	cargo xtask generate-fixtures --wasm
-	cargo xtask test-wasm
+test_wasm:
+	script/generate-fixtures-wasm
+	script/test-wasm
 
 lint:
 	cargo update --workspace --locked --quiet
-	cargo fmt --all --check
+	cargo check --workspace --all-targets
+	cargo +nightly fmt --all --check
 	cargo clippy --workspace --all-targets -- -D warnings
 
-lint-web:
-	npm --prefix lib/binding_web ci
-	npm --prefix lib/binding_web run lint
-
-lint-toml:
-	taplo check
-	taplo format --check --diff
-
 format:
-	cargo fmt --all
-	taplo format
+	cargo +nightly fmt --all
 
 changelog:
-	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)
+	@git-cliff --config script/cliff.toml --output CHANGELOG.md --latest --github-token $(shell gh auth token)
 
-.PHONY: test test-wasm lint format changelog
+.PHONY: test test_wasm lint format changelog

Package.swift

@@ -0,0 +1,21 @@
+// swift-tools-version: 5.8
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "TreeSitter",
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "TreeSitter",
+            targets: ["TreeSitter"]),
+    ],
+    targets: [
+        .target(name: "TreeSitter",
+                path: "lib",
+                sources: ["src/lib.c"],
+                cSettings: [.headerSearchPath("src")]),
+    ],
+    cLanguageStandard: .c11
+)

README.md

@@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca
 ## Links
 - [Documentation](https://tree-sitter.github.io)
 - [Rust binding](lib/binding_rust/README.md)
-- [Wasm binding](lib/binding_web/README.md)
-- [Command-line interface](crates/cli/README.md)
+- [WASM binding](lib/binding_web/README.md)
+- [Command-line interface](cli/README.md)
 
 [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord
 [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix

build.zig

@@ -1,147 +1,18 @@
 const std = @import("std");
 
-pub fn build(b: *std.Build) !void {
-    const target = b.standardTargetOptions(.{});
-    const optimize = b.standardOptimizeOption(.{});
-
-    var threaded: std.Io.Threaded = .init(b.allocator, .{});
-    defer threaded.deinit();
-    const io = threaded.io();
-
-    const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
-    const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
-    const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
-
-    var tree_sitter = b.createModule(.{
-        .target = target,
-        .optimize = optimize,
-        .link_libc = true,
-        .pic = if (shared) true else null,
-    });
-    const lib: *std.Build.Step.Compile = b.addLibrary(.{
+pub fn build(b: *std.Build) void {
+    var lib = b.addStaticLibrary(.{
         .name = "tree-sitter",
-        .linkage = if (shared) .dynamic else .static,
-        .root_module = tree_sitter,
+        .target = b.standardTargetOptions(.{}),
+        .optimize = b.standardOptimizeOption(.{}),
     });
 
-    if (amalgamated) {
-        tree_sitter.addCSourceFile(.{
-            .file = b.path("lib/src/lib.c"),
-            .flags = &.{"-std=c11"},
-        });
-    } else {
-        const files = try findSourceFiles(b, io);
-        defer b.allocator.free(files);
-        tree_sitter.addCSourceFiles(.{
-            .root = b.path("lib/src"),
-            .files = files,
-            .flags = &.{"-std=c11"},
-        });
-    }
-
-    tree_sitter.addIncludePath(b.path("lib/include"));
-    tree_sitter.addIncludePath(b.path("lib/src"));
-    tree_sitter.addIncludePath(b.path("lib/src/wasm"));
-
-    tree_sitter.addCMacro("_POSIX_C_SOURCE", "200112L");
-    tree_sitter.addCMacro("_DEFAULT_SOURCE", "");
-    tree_sitter.addCMacro("_BSD_SOURCE", "");
-    tree_sitter.addCMacro("_DARWIN_C_SOURCE", "");
-
-    if (wasm) {
-        if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
-            tree_sitter.addCMacro("TREE_SITTER_FEATURE_WASM", "");
-            tree_sitter.addSystemIncludePath(wasmtime.path("include"));
-            tree_sitter.addLibraryPath(wasmtime.path("lib"));
-            if (shared) tree_sitter.linkSystemLibrary("wasmtime", .{});
-        }
-    }
+    lib.linkLibC();
+    lib.addCSourceFile(.{ .file = b.path("lib/src/lib.c"), .flags = &.{"-std=c11"} });
+    lib.addIncludePath(b.path("lib/include"));
+    lib.addIncludePath(b.path("lib/src"));
 
     lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
 
     b.installArtifact(lib);
 }
-
-/// Get the name of the wasmtime dependency for this target.
-pub fn wasmtimeDep(target: std.Target) []const u8 {
-    const arch = target.cpu.arch;
-    const os = target.os.tag;
-    const abi = target.abi;
-    return @as(?[]const u8, switch (os) {
-        .linux => switch (arch) {
-            .x86_64 => switch (abi) {
-                .gnu => "wasmtime_c_api_x86_64_linux",
-                .musl => "wasmtime_c_api_x86_64_musl",
-                .android => "wasmtime_c_api_x86_64_android",
-                else => null,
-            },
-            .aarch64 => switch (abi) {
-                .gnu => "wasmtime_c_api_aarch64_linux",
-                .musl => "wasmtime_c_api_aarch64_musl",
-                .android => "wasmtime_c_api_aarch64_android",
-                else => null,
-            },
-            .x86 => switch (abi) {
-                .gnu => "wasmtime_c_api_i686_linux",
-                else => null,
-            },
-            .arm => switch (abi) {
-                .gnueabi => "wasmtime_c_api_armv7_linux",
-                else => null,
-            },
-            .s390x => switch (abi) {
-                .gnu => "wasmtime_c_api_s390x_linux",
-                else => null,
-            },
-            .riscv64 => switch (abi) {
-                .gnu => "wasmtime_c_api_riscv64gc_linux",
-                else => null,
-            },
-            else => null,
-        },
-        .windows => switch (arch) {
-            .x86_64 => switch (abi) {
-                .gnu => "wasmtime_c_api_x86_64_mingw",
-                .msvc => "wasmtime_c_api_x86_64_windows",
-                else => null,
-            },
-            .aarch64 => switch (abi) {
-                .msvc => "wasmtime_c_api_aarch64_windows",
-                else => null,
-            },
-            .x86 => switch (abi) {
-                .msvc => "wasmtime_c_api_i686_windows",
-                else => null,
-            },
-            else => null,
-        },
-        .macos => switch (arch) {
-            .x86_64 => "wasmtime_c_api_x86_64_macos",
-            .aarch64 => "wasmtime_c_api_aarch64_macos",
-            else => null,
-        },
-        else => null,
-    }) orelse std.debug.panic(
-        "Unsupported target for wasmtime: {s}-{s}-{s}",
-        .{ @tagName(arch), @tagName(os), @tagName(abi) },
-    );
-}
-
-fn findSourceFiles(b: *std.Build, io: std.Io) ![]const []const u8 {
-    var sources: std.ArrayListUnmanaged([]const u8) = .empty;
-
-    var dir = try b.build_root.handle.openDir(io, "lib/src", .{ .iterate = true });
-    var iter = dir.iterate();
-    defer dir.close(io);
-
-    while (try iter.next(io)) |entry| {
-        if (entry.kind != .file) continue;
-        const file = entry.name;
-        const ext = std.fs.path.extension(file);
-        if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
-            try sources.append(b.allocator, b.dupe(file));
-        }
-    }
-
-    return sources.toOwnedSlice(b.allocator);
-}

build.zig.zon

@@ -1,96 +1,10 @@
 .{
-    .name = .tree_sitter,
-    .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.28.0",
-    .minimum_zig_version = "0.16.0",
+    .name = "tree-sitter",
+    .version = "0.24.4",
     .paths = .{
         "build.zig",
         "build.zig.zon",
         "lib/src",
         "lib/include",
-        "README.md",
-        "LICENSE",
-    },
-    .dependencies = .{
-        .wasmtime_c_api_aarch64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIp_mQVzQOITXcYcWxYLJkvB1W1SvLlrdiU2G7fj",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAMztsgU5Aj4oI3MRHXJVe5rW72op-kT_78I3kZVM",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AANZxOwT27sdrKxDDGGKsiwtcZlHy204xAWNIgDBH",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAJL1zQW9yxC98uc60lSuVUHhH77QHTto0zwIQnBj",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-windows-c-api.zip",
-            .hash = "N-V-__8AAHRCtQU93hJcRFOgVcof3IQpRV9stT2Pp54wpJc2",
-            .lazy = true,
-        },
-        .wasmtime_c_api_armv7_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-armv7-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAJaW6gT8QdULOU0jxX4a_DOCA5YD6cxWBC8IqhQF",
-            .lazy = true,
-        },
-        .wasmtime_c_api_i686_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-linux-c-api.tar.xz",
-            .hash = "N-V-__8AANguMgVX4XMhdOVkdj4yfFKXrG8RTgZDs3nQB8J8",
-            .lazy = true,
-        },
-        .wasmtime_c_api_i686_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-windows-c-api.zip",
-            .hash = "N-V-__8AANY9ggXg4rK2_1o3EIlrCq124l5RfykPv-DPforq",
-            .lazy = true,
-        },
-        .wasmtime_c_api_riscv64gc_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-riscv64gc-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAPDtCAdQ0dD9Rs-qWl-kPr2c7L3PVsUVjwy12Iz1",
-            .lazy = true,
-        },
-        .wasmtime_c_api_s390x_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-s390x-linux-c-api.tar.xz",
-            .hash = "N-V-__8AANA3BwY1ZOoGCWCR_tTY9G1vfIX128RGVxufN3ov",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAF4AIgY0ltjevj1ybGfvMU1ErPRnNve5X1TmvCru",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAIR0cAbjf3DkrTbu81Oq_zociz-0lCpb5DR0lIC9",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAFJ4lgRgCBnYdz8-Yfc4hLve45Hv-0RICAEuCp4s",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_mingw = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-mingw-c-api.zip",
-            .hash = "N-V-__8AAMxZxQZUpp1cU8J5zgLiMNq4e4dy0hcchlQFy03J",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAN5pWgZrZBt8VYWkN82WjyFe3DkGcE7uLn3jpt38",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-windows-c-api.zip",
-            .hash = "N-V-__8AAEIJkgaVHFETgakbognNUpFELuV17vpjw6NjsWhQ",
-            .lazy = true,
-        },
     },
 }

cli/Cargo.toml

@@ -0,0 +1,75 @@
+[package]
+name = "tree-sitter-cli"
+version.workspace = true
+description = "CLI tool for developing, testing, and using Tree-sitter parsers"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+readme = "README.md"
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+[[bin]]
+name = "tree-sitter"
+path = "src/main.rs"
+doc = false
+
+[[bench]]
+name = "benchmark"
+harness = false
+
+[features]
+wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
+
+[dependencies]
+anstyle.workspace = true
+anyhow.workspace = true
+bstr.workspace = true
+clap.workspace = true
+clap_complete.workspace = true
+ctor.workspace = true
+ctrlc.workspace = true
+dialoguer.workspace = true
+dirs.workspace = true
+filetime.workspace = true
+glob.workspace = true
+heck.workspace = true
+html-escape.workspace = true
+indexmap.workspace = true
+indoc.workspace = true
+lazy_static.workspace = true
+log.workspace = true
+memchr.workspace = true
+rand.workspace = true
+regex.workspace = true
+regex-syntax.workspace = true
+rustc-hash.workspace = true
+semver.workspace = true
+serde.workspace = true
+serde_derive.workspace = true
+serde_json.workspace = true
+similar.workspace = true
+smallbitvec.workspace = true
+streaming-iterator.workspace = true
+tiny_http.workspace = true
+url.workspace = true
+walkdir.workspace = true
+wasmparser.workspace = true
+webbrowser.workspace = true
+
+tree-sitter.workspace = true
+tree-sitter-generate.workspace = true
+tree-sitter-config.workspace = true
+tree-sitter-highlight.workspace = true
+tree-sitter-loader.workspace = true
+tree-sitter-tags.workspace = true
+
+[dev-dependencies]
+tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
+
+tempfile.workspace = true
+pretty_assertions.workspace = true
+unindent.workspace = true

cli/README.md

@@ -7,20 +7,20 @@
 [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
 [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
 
-The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`,
-`Linux`, and `Windows`.
+The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on MacOS, Linux, and Windows.
 
 ### Installation
 
-You can install the `tree-sitter-cli` with [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall):
+You can install the `tree-sitter-cli` with `cargo`:
 
 ```sh
-cargo binstall tree-sitter-cli
+cargo install tree-sitter-cli
 ```
 
-or you can build it from source:
+or with `npm`:
+
 ```sh
-cargo install --locked tree-sitter-cli
+npm install tree-sitter-cli
 ```
 
 You can also download a pre-built binary for your platform from [the releases page].
@@ -34,11 +34,9 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
 
 ### Commands
 
-* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current
-  working directory. See [the documentation] for more information.
+* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information.
 
-* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory.
-  See [the documentation] for more information.
+* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information.
 
 * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.
 

cli/benches/benchmark.rs

@@ -0,0 +1,222 @@
+use std::{
+    collections::BTreeMap,
+    env, fs,
+    path::{Path, PathBuf},
+    str,
+    time::Instant,
+};
+
+use anyhow::Context;
+use lazy_static::lazy_static;
+use tree_sitter::{Language, Parser, Query};
+use tree_sitter_loader::{CompileConfig, Loader};
+
+include!("../src/tests/helpers/dirs.rs");
+
+lazy_static! {
+    static ref LANGUAGE_FILTER: Option<String> =
+        env::var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER").ok();
+    static ref EXAMPLE_FILTER: Option<String> =
+        env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok();
+    static ref REPETITION_COUNT: usize = env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
+        .map(|s| s.parse::<usize>().unwrap())
+        .unwrap_or(5);
+    static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
+    static ref EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)> = {
+        fn process_dir(result: &mut BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>, dir: &Path) {
+            if dir.join("grammar.js").exists() {
+                let relative_path = dir.strip_prefix(GRAMMARS_DIR.as_path()).unwrap();
+                let (example_paths, query_paths) =
+                    result.entry(relative_path.to_owned()).or_default();
+
+                if let Ok(example_files) = fs::read_dir(dir.join("examples")) {
+                    example_paths.extend(example_files.filter_map(|p| {
+                        let p = p.unwrap().path();
+                        if p.is_file() {
+                            Some(p)
+                        } else {
+                            None
+                        }
+                    }));
+                }
+
+                if let Ok(query_files) = fs::read_dir(dir.join("queries")) {
+                    query_paths.extend(query_files.filter_map(|p| {
+                        let p = p.unwrap().path();
+                        if p.is_file() {
+                            Some(p)
+                        } else {
+                            None
+                        }
+                    }));
+                }
+            } else {
+                for entry in fs::read_dir(dir).unwrap() {
+                    let entry = entry.unwrap().path();
+                    if entry.is_dir() {
+                        process_dir(result, &entry);
+                    }
+                }
+            }
+        }
+
+        let mut result = BTreeMap::new();
+        process_dir(&mut result, &GRAMMARS_DIR);
+        result
+    };
+}
+
+fn main() {
+    let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR
+        .values()
+        .flat_map(|(e, q)| {
+            e.iter()
+                .chain(q.iter())
+                .map(|s| s.file_name().unwrap().to_str().unwrap().len())
+        })
+        .max()
+        .unwrap_or(0);
+
+    eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT);
+
+    let mut parser = Parser::new();
+    let mut all_normal_speeds = Vec::new();
+    let mut all_error_speeds = Vec::new();
+
+    for (language_path, (example_paths, query_paths)) in
+        EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
+    {
+        let language_name = language_path.file_name().unwrap().to_str().unwrap();
+
+        if let Some(filter) = LANGUAGE_FILTER.as_ref() {
+            if language_name != filter.as_str() {
+                continue;
+            }
+        }
+
+        eprintln!("\nLanguage: {language_name}");
+        let language = get_language(language_path);
+        parser.set_language(&language).unwrap();
+
+        eprintln!("  Constructing Queries");
+        for path in query_paths {
+            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
+                if !path.to_str().unwrap().contains(filter.as_str()) {
+                    continue;
+                }
+            }
+
+            parse(path, max_path_length, |source| {
+                Query::new(&language, str::from_utf8(source).unwrap())
+                    .with_context(|| format!("Query file path: {path:?}"))
+                    .expect("Failed to parse query");
+            });
+        }
+
+        eprintln!("  Parsing Valid Code:");
+        let mut normal_speeds = Vec::new();
+        for example_path in example_paths {
+            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
+                if !example_path.to_str().unwrap().contains(filter.as_str()) {
+                    continue;
+                }
+            }
+
+            normal_speeds.push(parse(example_path, max_path_length, |code| {
+                parser.parse(code, None).expect("Failed to parse");
+            }));
+        }
+
+        eprintln!("  Parsing Invalid Code (mismatched languages):");
+        let mut error_speeds = Vec::new();
+        for (other_language_path, (example_paths, _)) in
+            EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
+        {
+            if other_language_path != language_path {
+                for example_path in example_paths {
+                    if let Some(filter) = EXAMPLE_FILTER.as_ref() {
+                        if !example_path.to_str().unwrap().contains(filter.as_str()) {
+                            continue;
+                        }
+                    }
+
+                    error_speeds.push(parse(example_path, max_path_length, |code| {
+                        parser.parse(code, None).expect("Failed to parse");
+                    }));
+                }
+            }
+        }
+
+        if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) {
+            eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
+            eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+        }
+
+        if let Some((average_error, worst_error)) = aggregate(&error_speeds) {
+            eprintln!("  Average Speed (errors): {average_error} bytes/ms");
+            eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
+        }
+
+        all_normal_speeds.extend(normal_speeds);
+        all_error_speeds.extend(error_speeds);
+    }
+
+    eprintln!("\n  Overall");
+    if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) {
+        eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
+        eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+    }
+
+    if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) {
+        eprintln!("  Average Speed (errors): {average_error} bytes/ms");
+        eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
+    }
+    eprintln!();
+}
+
+fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
+    if speeds.is_empty() {
+        return None;
+    }
+    let mut total = 0;
+    let mut max = usize::MAX;
+    for speed in speeds.iter().copied() {
+        total += speed;
+        if speed < max {
+            max = speed;
+        }
+    }
+    Some((total / speeds.len(), max))
+}
+
+fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize {
+    eprint!(
+        "    {:width$}\t",
+        path.file_name().unwrap().to_str().unwrap(),
+        width = max_path_length
+    );
+
+    let source_code = fs::read(path)
+        .with_context(|| format!("Failed to read {path:?}"))
+        .unwrap();
+    let time = Instant::now();
+    for _ in 0..*REPETITION_COUNT {
+        action(&source_code);
+    }
+    let duration = time.elapsed() / (*REPETITION_COUNT as u32);
+    let duration_ns = duration.as_nanos();
+    let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns;
+    eprintln!(
+        "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
+        (duration_ns as f64) / 1e6,
+    );
+    speed as usize
+}
+
+fn get_language(path: &Path) -> Language {
+    let src_path = GRAMMARS_DIR.join(path).join("src");
+    TEST_LOADER
+        .load_language_at_path(CompileConfig::new(&src_path, None, None))
+        .with_context(|| format!("Failed to load language at path {src_path:?}"))
+        .unwrap()
+}

cli/build.rs

@@ -0,0 +1,142 @@
+use std::{
+    env,
+    ffi::OsStr,
+    fs,
+    path::{Path, PathBuf},
+    time::SystemTime,
+};
+
+fn main() {
+    if let Some(git_sha) = read_git_sha() {
+        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
+    }
+
+    println!("cargo:rustc-check-cfg=cfg(sanitizing)");
+    println!("cargo:rustc-check-cfg=cfg(TREE_SITTER_EMBED_WASM_BINDING)");
+
+    if web_playground_files_present() {
+        println!("cargo:rustc-cfg=TREE_SITTER_EMBED_WASM_BINDING");
+    }
+
+    let build_time = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap()
+        .as_secs_f64();
+    println!("cargo:rustc-env=BUILD_TIME={build_time}");
+
+    #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "netbsd",
+        target_os = "dragonfly",
+    ))]
+    {
+        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()).join("dynamic-symbols.txt");
+        std::fs::write(
+            &out_dir,
+            "{
+                ts_current_malloc;
+                ts_current_calloc;
+                ts_current_realloc;
+                ts_current_free;
+            };",
+        )
+        .unwrap();
+        println!(
+            "cargo:rustc-link-arg=-Wl,--dynamic-list={}",
+            out_dir.display()
+        );
+    }
+}
+
+fn web_playground_files_present() -> bool {
+    let paths = [
+        "../docs/assets/js/playground.js",
+        "../lib/binding_web/tree-sitter.js",
+        "../lib/binding_web/tree-sitter.wasm",
+    ];
+
+    paths.iter().all(|p| Path::new(p).exists())
+}
+
+fn read_git_sha() -> Option<String> {
+    let mut repo_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+
+    let mut git_path;
+    loop {
+        git_path = repo_path.join(".git");
+        if git_path.exists() {
+            break;
+        }
+        if !repo_path.pop() {
+            return None;
+        }
+    }
+
+    let git_dir_path;
+    if git_path.is_dir() {
+        git_dir_path = git_path;
+    } else if let Ok(git_path_content) = fs::read_to_string(&git_path) {
+        git_dir_path = repo_path.join(git_path_content.get("gitdir: ".len()..).unwrap().trim_end());
+    } else {
+        return None;
+    }
+    let git_head_path = git_dir_path.join("HEAD");
+    if let Some(path) = git_head_path.to_str() {
+        println!("cargo:rerun-if-changed={path}");
+    }
+    if let Ok(mut head_content) = fs::read_to_string(&git_head_path) {
+        if head_content.ends_with('\n') {
+            head_content.pop();
+        }
+
+        // If we're on a branch, read the SHA from the ref file.
+        if head_content.starts_with("ref: ") {
+            head_content.replace_range(0.."ref: ".len(), "");
+            let ref_filename = {
+                // Go to real non-worktree gitdir
+                let git_dir_path = git_dir_path
+                    .parent()
+                    .and_then(|p| {
+                        p.file_name()
+                            .map(|n| n == OsStr::new("worktrees"))
+                            .and_then(|x| x.then(|| p.parent()))
+                    })
+                    .flatten()
+                    .unwrap_or(&git_dir_path);
+
+                let file = git_dir_path.join(&head_content);
+                if file.is_file() {
+                    file
+                } else {
+                    let packed_refs = git_dir_path.join("packed-refs");
+                    if let Ok(packed_refs_content) = fs::read_to_string(&packed_refs) {
+                        for line in packed_refs_content.lines() {
+                            if let Some((hash, r#ref)) = line.split_once(' ') {
+                                if r#ref == head_content {
+                                    if let Some(path) = packed_refs.to_str() {
+                                        println!("cargo:rerun-if-changed={path}");
+                                    }
+                                    return Some(hash.to_string());
+                                }
+                            }
+                        }
+                    }
+                    return None;
+                }
+            };
+            if let Some(path) = ref_filename.to_str() {
+                println!("cargo:rerun-if-changed={path}");
+            }
+            return fs::read_to_string(&ref_filename).ok();
+        }
+        // If we're on a detached commit, then the `HEAD` file itself contains the sha.
+        if head_content.len() == 40 {
+            return Some(head_content);
+        }
+    }
+
+    None
+}

cli/config/Cargo.toml

@@ -5,22 +5,15 @@ description = "User configuration of tree-sitter's command line programs"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
+readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
-documentation = "https://docs.rs/tree-sitter-config"
 license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 
-[lib]
-path = "src/tree_sitter_config.rs"
-
-[lints]
-workspace = true
-
 [dependencies]
-etcetera.workspace = true
-log.workspace = true
+anyhow.workspace = true
+dirs.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-thiserror.workspace = true

cli/config/src/lib.rs

@@ -1,64 +1,10 @@
-#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
+#![doc = include_str!("../README.md")]
 
-use std::{
-    env, fs,
-    path::{Path, PathBuf},
-};
+use std::{env, fs, path::PathBuf};
 
-use etcetera::BaseStrategy as _;
-use log::warn;
+use anyhow::{anyhow, Context, Result};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use thiserror::Error;
-
-pub type ConfigResult<T> = Result<T, ConfigError>;
-
-#[derive(Debug, Error)]
-pub enum ConfigError {
-    #[error("Bad JSON config {0} -- {1}")]
-    ConfigRead(String, serde_json::Error),
-    #[error(transparent)]
-    HomeDir(#[from] etcetera::HomeDirError),
-    #[error(transparent)]
-    IO(IoError),
-    #[error(transparent)]
-    Serialization(#[from] serde_json::Error),
-}
-
-#[derive(Debug, Error)]
-pub struct IoError {
-    pub error: std::io::Error,
-    pub path: Option<PathBuf>,
-}
-
-impl PartialEq for IoError {
-    fn eq(&self, other: &Self) -> bool {
-        self.path == other.path
-            && self.error.kind() == other.error.kind()
-            && self.error.raw_os_error() == other.error.raw_os_error()
-    }
-}
-
-impl Eq for IoError {}
-
-impl IoError {
-    fn new(error: std::io::Error, path: Option<&Path>) -> Self {
-        Self {
-            error,
-            path: path.map(Path::to_path_buf),
-        }
-    }
-}
-
-impl std::fmt::Display for IoError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.error)?;
-        if let Some(ref path) = self.path {
-            write!(f, " ({})", path.display())?;
-        }
-        Ok(())
-    }
-}
 
 /// Holds the contents of tree-sitter's configuration file.
 ///
@@ -75,7 +21,7 @@ pub struct Config {
 }
 
 impl Config {
-    pub fn find_config_file() -> ConfigResult<Option<PathBuf>> {
+    pub fn find_config_file() -> Result<Option<PathBuf>> {
         if let Ok(path) = env::var("TREE_SITTER_DIR") {
             let mut path = PathBuf::from(path);
             path.push("config.json");
@@ -92,28 +38,8 @@ impl Config {
             return Ok(Some(xdg_path));
         }
 
-        if cfg!(target_os = "macos") {
-            let legacy_apple_path = etcetera::base_strategy::Apple::new()?
-                .data_dir() // `$HOME/Library/Application Support/`
-                .join("tree-sitter")
-                .join("config.json");
-            if legacy_apple_path.is_file() {
-                let xdg_dir = xdg_path.parent().unwrap();
-                fs::create_dir_all(xdg_dir)
-                    .map_err(|e| ConfigError::IO(IoError::new(e, Some(xdg_dir))))?;
-                fs::rename(&legacy_apple_path, &xdg_path).map_err(|e| {
-                    ConfigError::IO(IoError::new(e, Some(legacy_apple_path.as_path())))
-                })?;
-                warn!(
-                    "Your config.json file has been automatically migrated from \"{}\" to \"{}\"",
-                    legacy_apple_path.display(),
-                    xdg_path.display()
-                );
-                return Ok(Some(xdg_path));
-            }
-        }
-
-        let legacy_path = etcetera::home_dir()?
+        let legacy_path = dirs::home_dir()
+            .ok_or_else(|| anyhow!("Cannot determine home directory"))?
             .join(".tree-sitter")
             .join("config.json");
         if legacy_path.is_file() {
@@ -123,9 +49,9 @@ impl Config {
         Ok(None)
     }
 
-    fn xdg_config_file() -> ConfigResult<PathBuf> {
-        let xdg_path = etcetera::choose_base_strategy()?
-            .config_dir()
+    fn xdg_config_file() -> Result<PathBuf> {
+        let xdg_path = dirs::config_dir()
+            .ok_or_else(|| anyhow!("Cannot determine config directory"))?
             .join("tree-sitter")
             .join("config.json");
         Ok(xdg_path)
@@ -137,10 +63,10 @@ impl Config {
     ///   - Location specified by the path parameter if provided
     ///   - `$TREE_SITTER_DIR/config.json`, if the `TREE_SITTER_DIR` environment variable is set
     ///   - `tree-sitter/config.json` in your default user configuration directory, as determined by
-    ///     [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy)
+    ///     [`dirs::config_dir`](https://docs.rs/dirs/*/dirs/fn.config_dir.html)
     ///   - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
     ///     its configuration
-    pub fn load(path: Option<PathBuf>) -> ConfigResult<Self> {
+    pub fn load(path: Option<PathBuf>) -> Result<Self> {
         let location = if let Some(path) = path {
             path
         } else if let Some(path) = Self::find_config_file()? {
@@ -150,9 +76,9 @@ impl Config {
         };
 
         let content = fs::read_to_string(&location)
-            .map_err(|e| ConfigError::IO(IoError::new(e, Some(location.as_path()))))?;
+            .with_context(|| format!("Failed to read {}", &location.to_string_lossy()))?;
         let config = serde_json::from_str(&content)
-            .map_err(|e| ConfigError::ConfigRead(location.to_string_lossy().to_string(), e))?;
+            .with_context(|| format!("Bad JSON config {}", &location.to_string_lossy()))?;
         Ok(Self { location, config })
     }
 
@@ -162,7 +88,7 @@ impl Config {
     /// disk.
     ///
     /// (Note that this is typically only done by the `tree-sitter init-config` command.)
-    pub fn initial() -> ConfigResult<Self> {
+    pub fn initial() -> Result<Self> {
         let location = if let Ok(path) = env::var("TREE_SITTER_DIR") {
             let mut path = PathBuf::from(path);
             path.push("config.json");
@@ -175,20 +101,17 @@ impl Config {
     }
 
     /// Saves this configuration to the file that it was originally loaded from.
-    pub fn save(&self) -> ConfigResult<()> {
+    pub fn save(&self) -> Result<()> {
         let json = serde_json::to_string_pretty(&self.config)?;
-        let config_dir = self.location.parent().unwrap();
-        fs::create_dir_all(config_dir)
-            .map_err(|e| ConfigError::IO(IoError::new(e, Some(config_dir))))?;
-        fs::write(&self.location, json)
-            .map_err(|e| ConfigError::IO(IoError::new(e, Some(self.location.as_path()))))?;
+        fs::create_dir_all(self.location.parent().unwrap())?;
+        fs::write(&self.location, json)?;
         Ok(())
     }
 
     /// Parses a component-specific configuration from the configuration file.  The type `C` must
     /// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON
     /// object, and must only include the fields relevant to that component.
-    pub fn get<C>(&self) -> ConfigResult<C>
+    pub fn get<C>(&self) -> Result<C>
     where
         C: for<'de> Deserialize<'de>,
     {
@@ -199,7 +122,7 @@ impl Config {
     /// Adds a component-specific configuration to the configuration file.  The type `C` must be
     /// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and
     /// must only include the fields relevant to that component.
-    pub fn add<C>(&mut self, config: C) -> ConfigResult<()>
+    pub fn add<C>(&mut self, config: C) -> Result<()>
     where
         C: Serialize,
     {

cli/eslint/package.json

@@ -4,8 +4,7 @@
   "description": "Eslint configuration for Tree-sitter grammar files",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
-    "directory": "crates/cli/eslint"
+    "url": "git+https://github.com/tree-sitter/tree-sitter.git"
   },
   "license": "MIT",
   "author": "Amaan Qureshi <amaanq12@gmail.com>",
@@ -17,13 +16,9 @@
     "tree-sitter"
   ],
   "dependencies": {
-    "eslint-plugin-jsdoc": "^62.7.0"
+    "eslint-plugin-jsdoc": "^50.2.4"
   },
   "peerDependencies": {
     "eslint": ">= 9"
-  },
-  "scripts": {
-    "prepack": "cp ../../../LICENSE .",
-    "postpack": "rm LICENSE"
   }
 }

cli/generate/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "tree-sitter-generate"
+version.workspace = true
+description = "Library for generating C source code from a tree-sitter grammar"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+readme = "README.md"
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+heck.workspace = true
+indexmap.workspace = true
+indoc.workspace = true
+lazy_static.workspace = true
+log.workspace = true
+regex.workspace = true
+regex-syntax.workspace = true
+rustc-hash.workspace = true
+semver.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+smallbitvec.workspace = true
+url.workspace = true
+
+tree-sitter.workspace = true

cli/generate/src/build_tables/build_lex_table.rs

@@ -1,11 +1,9 @@
 use std::{
-    collections::{VecDeque, hash_map::Entry},
+    collections::{hash_map::Entry, HashMap, VecDeque},
     mem,
 };
 
-use rustc_hash::FxHashMap;
-
-use log::debug;
+use log::info;
 
 use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap};
 use crate::{
@@ -13,7 +11,7 @@ use crate::{
     grammars::{LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor},
     rules::{Symbol, TokenSet},
-    tables::{AdvanceAction, LexState, LexStateId, LexTable, ParseStateId, ParseTable},
+    tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable},
 };
 
 pub const LARGE_CHARACTER_RANGE_COUNT: usize = 8;
@@ -45,17 +43,15 @@ pub fn build_lex_table(
         let tokens = state
             .terminal_entries
             .keys()
-            .copied()
-            .chain(state.reserved_words.iter())
             .filter_map(|token| {
                 if token.is_terminal() {
                     if keywords.contains(token) {
                         syntax_grammar.word_token
                     } else {
-                        Some(token)
+                        Some(*token)
                     }
                 } else if token.is_eof() {
-                    Some(token)
+                    Some(*token)
                 } else {
                     None
                 }
@@ -67,17 +63,18 @@ pub fn build_lex_table(
             if merge_token_set(
                 &mut entry.0,
                 &tokens,
+                lexical_grammar,
                 token_conflict_map,
                 coincident_token_index,
             ) {
                 did_merge = true;
-                entry.1.push(i as u32);
+                entry.1.push(i);
                 break;
             }
         }
 
         if !did_merge {
-            parse_state_ids_by_token_set.push((tokens, vec![i as u32]));
+            parse_state_ids_by_token_set.push((tokens, vec![i]));
         }
     }
 
@@ -85,7 +82,7 @@ pub fn build_lex_table(
     for (tokens, parse_state_ids) in parse_state_ids_by_token_set {
         let lex_state_id = builder.add_state_for_tokens(&tokens);
         for id in parse_state_ids {
-            parse_table.states[id as usize].lex_state_id = lex_state_id;
+            parse_table.states[id].lex_state_id = lex_state_id;
         }
     }
 
@@ -131,7 +128,7 @@ pub fn build_lex_table(
 }
 
 struct QueueEntry {
-    state_id: LexStateId,
+    state_id: usize,
     nfa_states: Vec<u32>,
     eof_valid: bool,
 }
@@ -141,7 +138,7 @@ struct LexTableBuilder<'a> {
     cursor: NfaCursor<'a>,
     table: LexTable,
     state_queue: VecDeque<QueueEntry>,
-    state_ids_by_nfa_state_set: FxHashMap<(Vec<u32>, bool), LexStateId>,
+    state_ids_by_nfa_state_set: HashMap<(Vec<u32>, bool), usize>,
 }
 
 impl<'a> LexTableBuilder<'a> {
@@ -151,7 +148,7 @@ impl<'a> LexTableBuilder<'a> {
             cursor: NfaCursor::new(&lexical_grammar.nfa, vec![]),
             table: LexTable::default(),
             state_queue: VecDeque::new(),
-            state_ids_by_nfa_state_set: FxHashMap::default(),
+            state_ids_by_nfa_state_set: HashMap::new(),
         }
     }
 
@@ -161,13 +158,13 @@ impl<'a> LexTableBuilder<'a> {
         self.state_ids_by_nfa_state_set.clear();
     }
 
-    fn add_state_for_tokens(&mut self, tokens: &TokenSet) -> LexStateId {
+    fn add_state_for_tokens(&mut self, tokens: &TokenSet) -> usize {
         let mut eof_valid = false;
         let nfa_states = tokens
             .iter()
             .filter_map(|token| {
                 if token.is_terminal() {
-                    Some(self.lexical_grammar.variables[token.index as usize].start_state)
+                    Some(self.lexical_grammar.variables[token.index].start_state)
                 } else {
                     eof_valid = true;
                     None
@@ -177,11 +174,12 @@ impl<'a> LexTableBuilder<'a> {
         let (state_id, is_new) = self.add_state(nfa_states, eof_valid);
 
         if is_new {
-            debug!(
-                "entry point state: {state_id}, tokens: {:?}",
+            info!(
+                "entry point state: {}, tokens: {:?}",
+                state_id,
                 tokens
                     .iter()
-                    .map(|t| &self.lexical_grammar.variables[t.index as usize].name)
+                    .map(|t| &self.lexical_grammar.variables[t.index].name)
                     .collect::<Vec<_>>()
             );
         }
@@ -197,7 +195,7 @@ impl<'a> LexTableBuilder<'a> {
         state_id
     }
 
-    fn add_state(&mut self, nfa_states: Vec<u32>, eof_valid: bool) -> (LexStateId, bool) {
+    fn add_state(&mut self, nfa_states: Vec<u32>, eof_valid: bool) -> (usize, bool) {
         self.cursor.reset(nfa_states);
         match self
             .state_ids_by_nfa_state_set
@@ -205,7 +203,7 @@ impl<'a> LexTableBuilder<'a> {
         {
             Entry::Occupied(o) => (*o.get(), false),
             Entry::Vacant(v) => {
-                let state_id = self.table.states.len() as u32;
+                let state_id = self.table.states.len();
                 self.table.states.push(LexState::default());
                 self.state_queue.push_back(QueueEntry {
                     state_id,
@@ -218,52 +216,53 @@ impl<'a> LexTableBuilder<'a> {
         }
     }
 
-    fn populate_state(&mut self, state_id: LexStateId, nfa_states: Vec<u32>, eof_valid: bool) {
+    fn populate_state(&mut self, state_id: usize, nfa_states: Vec<u32>, eof_valid: bool) {
         self.cursor.force_reset(nfa_states);
 
         // The EOF state is represented as an empty list of NFA states.
         let mut completion = None;
         for (id, prec) in self.cursor.completions() {
-            if let Some((prev_id, prev_precedence)) = completion
-                && TokenConflictMap::prefer_token(
+            if let Some((prev_id, prev_precedence)) = completion {
+                if TokenConflictMap::prefer_token(
                     self.lexical_grammar,
                     (prev_precedence, prev_id),
                     (prec, id),
-                )
-            {
-                continue;
+                ) {
+                    continue;
+                }
             }
             completion = Some((id, prec));
         }
 
-        let (transitions, has_sep) = self.cursor.transitions_and_any_sep();
+        let transitions = self.cursor.transitions();
+        let has_sep = self.cursor.transition_chars().any(|(_, sep)| sep);
 
         // If EOF is a valid lookahead token, add a transition predicated on the null
         // character that leads to the empty set of NFA states.
         if eof_valid {
             let (next_state_id, _) = self.add_state(Vec::new(), false);
-            self.table.states[state_id as usize].eof_action = Some(AdvanceAction {
+            self.table.states[state_id].eof_action = Some(AdvanceAction {
                 state: next_state_id,
                 in_main_token: true,
             });
         }
 
         for transition in transitions {
-            if let Some((completed_id, completed_precedence)) = completion
-                && !TokenConflictMap::prefer_transition(
+            if let Some((completed_id, completed_precedence)) = completion {
+                if !TokenConflictMap::prefer_transition(
                     self.lexical_grammar,
                     &transition,
                     completed_id,
                     completed_precedence,
                     has_sep,
-                )
-            {
-                continue;
+                ) {
+                    continue;
+                }
             }
 
             let (next_state_id, _) =
                 self.add_state(transition.states, eof_valid && transition.is_separator);
-            self.table.states[state_id as usize].advance_actions.push((
+            self.table.states[state_id].advance_actions.push((
                 transition.characters,
                 AdvanceAction {
                     state: next_state_id,
@@ -273,78 +272,41 @@ impl<'a> LexTableBuilder<'a> {
         }
 
         if let Some((complete_id, _)) = completion {
-            self.table.states[state_id as usize].accept_action =
-                Some(Symbol::terminal(complete_id));
+            self.table.states[state_id].accept_action = Some(Symbol::terminal(complete_id));
         } else if self.cursor.state_ids.is_empty() {
-            self.table.states[state_id as usize].accept_action = Some(Symbol::end());
+            self.table.states[state_id].accept_action = Some(Symbol::end());
         }
     }
 }
 
-fn check_token_conflicts(
-    i: usize,
-    set_without_terminal: &TokenSet,
-    token_conflict_map: &TokenConflictMap,
-    coincident_token_index: &CoincidentTokenIndex,
-) -> bool {
-    let wpr = token_conflict_map.row_words;
-    let row_start = i * wpr;
-    let set_bits = set_without_terminal.terminal_bits_words();
-
-    // Does terminal i conflict with or match-prefix any terminal in the set?
-    let conflict_row = &token_conflict_map.conflict_or_prefix_bits[row_start..row_start + wpr];
-    for (&c, &s) in conflict_row.iter().zip(set_bits) {
-        if c & s != 0 {
-            return true;
-        }
-    }
-
-    // Does terminal i overlap (in either direction) with any non-coincident terminal in the set?
-    let overlap_row = &token_conflict_map.overlap_either_bits[row_start..row_start + wpr];
-    let coincident_row = &coincident_token_index.row_bits[row_start..row_start + wpr];
-    for ((&o, &s), &c) in overlap_row.iter().zip(set_bits).zip(coincident_row) {
-        if o & s & !c != 0 {
-            return true;
-        }
-    }
-
-    false
-}
-
 fn merge_token_set(
     tokens: &mut TokenSet,
     other: &TokenSet,
+    lexical_grammar: &LexicalGrammar,
     token_conflict_map: &TokenConflictMap,
     coincident_token_index: &CoincidentTokenIndex,
 ) -> bool {
-    if tokens
-        .terminals()
-        .filter(|terminal| !other.contains_terminal(terminal.index as usize))
-        .any(|terminal| {
-            check_token_conflicts(
-                terminal.index as usize,
-                other,
-                token_conflict_map,
-                coincident_token_index,
-            )
-        })
-    {
-        return false;
-    }
+    for i in 0..lexical_grammar.variables.len() {
+        let symbol = Symbol::terminal(i);
+        let set_without_terminal = match (tokens.contains_terminal(i), other.contains_terminal(i)) {
+            (true, false) => other,
+            (false, true) => tokens,
+            _ => continue,
+        };
 
-    if other
-        .terminals()
-        .filter(|terminal| !tokens.contains_terminal(terminal.index as usize))
-        .any(|terminal| {
-            check_token_conflicts(
-                terminal.index as usize,
-                tokens,
-                token_conflict_map,
-                coincident_token_index,
-            )
-        })
-    {
-        return false;
+        for existing_token in set_without_terminal.terminals() {
+            if token_conflict_map.does_conflict(i, existing_token.index)
+                || token_conflict_map.does_match_prefix(i, existing_token.index)
+            {
+                return false;
+            }
+            if !coincident_token_index.contains(symbol, existing_token)
+                && (token_conflict_map.does_overlap(existing_token.index, i)
+                    || token_conflict_map.does_overlap(i, existing_token.index))
+            {
+                return false;
+            }
+        }
     }
 
     tokens.insert_all(other);
@@ -354,7 +316,7 @@ fn merge_token_set(
 fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
     // Initially group the states by their accept action and their
     // valid lookahead characters.
-    let mut state_ids_by_signature = FxHashMap::default();
+    let mut state_ids_by_signature = HashMap::new();
     for (i, state) in table.states.iter().enumerate() {
         let signature = (
             i == 0,
@@ -369,7 +331,7 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         state_ids_by_signature
             .entry(signature)
             .or_insert(Vec::new())
-            .push(i as u32);
+            .push(i);
     }
     let mut state_ids_by_group_id = state_ids_by_signature
         .into_iter()
@@ -382,10 +344,10 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         .unwrap();
     state_ids_by_group_id.swap(error_group_index, 0);
 
-    let mut group_ids_by_state_id = vec![0u32; table.states.len()];
+    let mut group_ids_by_state_id = vec![0; table.states.len()];
     for (group_id, state_ids) in state_ids_by_group_id.iter().enumerate() {
         for state_id in state_ids {
-            group_ids_by_state_id[*state_id as usize] = group_id as u32;
+            group_ids_by_state_id[*state_id] = group_id;
         }
     }
 
@@ -395,40 +357,37 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         &mut group_ids_by_state_id,
         1,
         lex_states_differ,
-    ) {}
+    ) {
+        continue;
+    }
 
     let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
     for state_ids in &state_ids_by_group_id {
         let mut new_state = LexState::default();
-        mem::swap(&mut new_state, &mut table.states[state_ids[0] as usize]);
+        mem::swap(&mut new_state, &mut table.states[state_ids[0]]);
 
         for (_, advance_action) in &mut new_state.advance_actions {
-            advance_action.state = group_ids_by_state_id[advance_action.state as usize];
+            advance_action.state = group_ids_by_state_id[advance_action.state];
         }
         if let Some(eof_action) = &mut new_state.eof_action {
-            eof_action.state = group_ids_by_state_id[eof_action.state as usize];
+            eof_action.state = group_ids_by_state_id[eof_action.state];
         }
         new_states.push(new_state);
     }
 
     for state in &mut parse_table.states {
-        state.lex_state_id = group_ids_by_state_id[state.lex_state_id as usize];
+        state.lex_state_id = group_ids_by_state_id[state.lex_state_id];
     }
 
     table.states = new_states;
 }
 
-fn lex_states_differ(
-    left: &LexState,
-    right: &LexState,
-    group_ids_by_state_id: &[LexStateId],
-) -> bool {
+fn lex_states_differ(left: &LexState, right: &LexState, group_ids_by_state_id: &[usize]) -> bool {
     left.advance_actions
         .iter()
         .zip(right.advance_actions.iter())
         .any(|(left, right)| {
-            group_ids_by_state_id[left.1.state as usize]
-                != group_ids_by_state_id[right.1.state as usize]
+            group_ids_by_state_id[left.1.state] != group_ids_by_state_id[right.1.state]
         })
 }
 
@@ -438,9 +397,9 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
     old_ids_by_new_id[1..].sort_by_key(|id| &table.states[*id]);
 
     // Get the inverse mapping
-    let mut new_ids_by_old_id = vec![0u32; old_ids_by_new_id.len()];
+    let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
     for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
-        new_ids_by_old_id[*old_id] = id as u32;
+        new_ids_by_old_id[*old_id] = id;
     }
 
     // Reorder the parse states and update their references to reflect
@@ -451,10 +410,10 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
             let mut state = LexState::default();
             mem::swap(&mut state, &mut table.states[*old_id]);
             for (_, advance_action) in &mut state.advance_actions {
-                advance_action.state = new_ids_by_old_id[advance_action.state as usize];
+                advance_action.state = new_ids_by_old_id[advance_action.state];
             }
             if let Some(eof_action) = &mut state.eof_action {
-                eof_action.state = new_ids_by_old_id[eof_action.state as usize];
+                eof_action.state = new_ids_by_old_id[eof_action.state];
             }
             state
         })
@@ -462,6 +421,6 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
 
     // Update the parse table's lex state references
     for state in &mut parse_table.states {
-        state.lex_state_id = new_ids_by_old_id[state.lex_state_id as usize];
+        state.lex_state_id = new_ids_by_old_id[state.lex_state_id];
     }
 }

cli/generate/src/build_tables/coincident_tokens.rs

@@ -0,0 +1,79 @@
+use std::fmt;
+
+use crate::{
+    grammars::LexicalGrammar,
+    rules::Symbol,
+    tables::{ParseStateId, ParseTable},
+};
+
+pub struct CoincidentTokenIndex<'a> {
+    entries: Vec<Vec<ParseStateId>>,
+    grammar: &'a LexicalGrammar,
+    n: usize,
+}
+
+impl<'a> CoincidentTokenIndex<'a> {
+    pub fn new(table: &ParseTable, lexical_grammar: &'a LexicalGrammar) -> Self {
+        let n = lexical_grammar.variables.len();
+        let mut result = Self {
+            n,
+            grammar: lexical_grammar,
+            entries: vec![Vec::new(); n * n],
+        };
+        for (i, state) in table.states.iter().enumerate() {
+            for symbol in state.terminal_entries.keys() {
+                if symbol.is_terminal() {
+                    for other_symbol in state.terminal_entries.keys() {
+                        if other_symbol.is_terminal() {
+                            let index = result.index(symbol.index, other_symbol.index);
+                            if result.entries[index].last().copied() != Some(i) {
+                                result.entries[index].push(i);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        result
+    }
+
+    pub fn states_with(&self, a: Symbol, b: Symbol) -> &[ParseStateId] {
+        &self.entries[self.index(a.index, b.index)]
+    }
+
+    pub fn contains(&self, a: Symbol, b: Symbol) -> bool {
+        !self.entries[self.index(a.index, b.index)].is_empty()
+    }
+
+    #[must_use]
+    const fn index(&self, a: usize, b: usize) -> usize {
+        if a < b {
+            a * self.n + b
+        } else {
+            b * self.n + a
+        }
+    }
+}
+
+impl<'a> fmt::Debug for CoincidentTokenIndex<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "CoincidentTokenIndex {{")?;
+
+        writeln!(f, "  entries: {{")?;
+        for i in 0..self.n {
+            writeln!(f, "    {}: {{", self.grammar.variables[i].name)?;
+            for j in 0..self.n {
+                writeln!(
+                    f,
+                    "      {}: {:?},",
+                    self.grammar.variables[j].name,
+                    self.entries[self.index(i, j)].len()
+                )?;
+            }
+            writeln!(f, "    }},")?;
+        }
+        write!(f, "  }},")?;
+        write!(f, "}}")?;
+        Ok(())
+    }
+}

cli/generate/src/build_tables/item.rs

@@ -0,0 +1,416 @@
+use std::{
+    cmp::Ordering,
+    fmt,
+    hash::{Hash, Hasher},
+};
+
+use lazy_static::lazy_static;
+
+use crate::{
+    grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
+    rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
+};
+
+lazy_static! {
+    static ref START_PRODUCTION: Production = Production {
+        dynamic_precedence: 0,
+        steps: vec![ProductionStep {
+            symbol: Symbol {
+                index: 0,
+                kind: SymbolType::NonTerminal,
+            },
+            precedence: Precedence::None,
+            associativity: None,
+            alias: None,
+            field_name: None,
+        }],
+    };
+}
+
+/// A [`ParseItem`] represents an in-progress match of a single production in a grammar.
+#[derive(Clone, Copy, Debug)]
+pub struct ParseItem<'a> {
+    /// The index of the parent rule within the grammar.
+    pub variable_index: u32,
+    /// The number of symbols that have already been matched.
+    pub step_index: u32,
+    /// The production being matched.
+    pub production: &'a Production,
+    /// A boolean indicating whether any of the already-matched children were
+    /// hidden nodes and had fields. Ordinarily, a parse item's behavior is not
+    /// affected by the symbols of its preceding children; it only needs to
+    /// keep track of their fields and aliases.
+    ///
+    /// Take for example these two items:
+    ///   X -> a b • c
+    ///   X -> a g • c
+    ///
+    /// They can be considered equivalent, for the purposes of parse table
+    /// generation, because they entail the same actions. But if this flag is
+    /// true, then the item's set of inherited fields may depend on the specific
+    /// symbols of its preceding children.
+    pub has_preceding_inherited_fields: bool,
+}
+
+/// A [`ParseItemSet`] represents a set of in-progress matches of productions in a
+/// grammar, and for each in-progress match, a set of "lookaheads" - tokens that
+/// are allowed to *follow* the in-progress rule. This object corresponds directly
+/// to a state in the final parse table.
+#[derive(Clone, Debug, PartialEq, Eq, Default)]
+pub struct ParseItemSet<'a> {
+    pub entries: Vec<(ParseItem<'a>, TokenSet)>,
+}
+
+/// A [`ParseItemSetCore`] is like a [`ParseItemSet`], but without the lookahead
+/// information. Parse states with the same core are candidates for merging.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ParseItemSetCore<'a> {
+    pub entries: Vec<ParseItem<'a>>,
+}
+
+pub struct ParseItemDisplay<'a>(
+    pub &'a ParseItem<'a>,
+    pub &'a SyntaxGrammar,
+    pub &'a LexicalGrammar,
+);
+
+pub struct TokenSetDisplay<'a>(
+    pub &'a TokenSet,
+    pub &'a SyntaxGrammar,
+    pub &'a LexicalGrammar,
+);
+
+pub struct ParseItemSetDisplay<'a>(
+    pub &'a ParseItemSet<'a>,
+    pub &'a SyntaxGrammar,
+    pub &'a LexicalGrammar,
+);
+
+impl<'a> ParseItem<'a> {
+    pub fn start() -> Self {
+        ParseItem {
+            variable_index: u32::MAX,
+            production: &START_PRODUCTION,
+            step_index: 0,
+            has_preceding_inherited_fields: false,
+        }
+    }
+
+    pub fn step(&self) -> Option<&'a ProductionStep> {
+        self.production.steps.get(self.step_index as usize)
+    }
+
+    pub fn symbol(&self) -> Option<Symbol> {
+        self.step().map(|step| step.symbol)
+    }
+
+    pub fn associativity(&self) -> Option<Associativity> {
+        self.prev_step().and_then(|step| step.associativity)
+    }
+
+    pub fn precedence(&self) -> &Precedence {
+        self.prev_step()
+            .map_or(&Precedence::None, |step| &step.precedence)
+    }
+
+    pub fn prev_step(&self) -> Option<&'a ProductionStep> {
+        if self.step_index > 0 {
+            Some(&self.production.steps[self.step_index as usize - 1])
+        } else {
+            None
+        }
+    }
+
+    #[must_use]
+    pub fn is_done(&self) -> bool {
+        self.step_index as usize == self.production.steps.len()
+    }
+
+    #[must_use]
+    pub const fn is_augmented(&self) -> bool {
+        self.variable_index == u32::MAX
+    }
+
+    /// Create an item like this one, but advanced by one step.
+    #[must_use]
+    pub const fn successor(&self) -> Self {
+        ParseItem {
+            variable_index: self.variable_index,
+            production: self.production,
+            step_index: self.step_index + 1,
+            has_preceding_inherited_fields: self.has_preceding_inherited_fields,
+        }
+    }
+
+    /// Create an item identical to this one, but with a different production.
+    /// This is used when dynamically "inlining" certain symbols in a production.
+    pub const fn substitute_production(&self, production: &'a Production) -> Self {
+        let mut result = *self;
+        result.production = production;
+        result
+    }
+}
+
+impl<'a> ParseItemSet<'a> {
+    pub fn with(elements: impl IntoIterator<Item = (ParseItem<'a>, TokenSet)>) -> Self {
+        let mut result = Self::default();
+        for (item, lookaheads) in elements {
+            result.insert(item, &lookaheads);
+        }
+        result
+    }
+
+    pub fn insert(&mut self, item: ParseItem<'a>, lookaheads: &TokenSet) -> &mut TokenSet {
+        match self.entries.binary_search_by(|(i, _)| i.cmp(&item)) {
+            Err(i) => {
+                self.entries.insert(i, (item, lookaheads.clone()));
+                &mut self.entries[i].1
+            }
+            Ok(i) => {
+                self.entries[i].1.insert_all(lookaheads);
+                &mut self.entries[i].1
+            }
+        }
+    }
+
+    pub fn core(&self) -> ParseItemSetCore<'a> {
+        ParseItemSetCore {
+            entries: self.entries.iter().map(|e| e.0).collect(),
+        }
+    }
+}
+
+impl<'a> fmt::Display for ParseItemDisplay<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        if self.0.is_augmented() {
+            write!(f, "START →")?;
+        } else {
+            write!(
+                f,
+                "{} →",
+                &self.1.variables[self.0.variable_index as usize].name
+            )?;
+        }
+
+        for (i, step) in self.0.production.steps.iter().enumerate() {
+            if i == self.0.step_index as usize {
+                write!(f, " •")?;
+                if let Some(associativity) = step.associativity {
+                    if step.precedence.is_none() {
+                        write!(f, " ({associativity:?})")?;
+                    } else {
+                        write!(f, " ({} {associativity:?})", step.precedence)?;
+                    }
+                } else if !step.precedence.is_none() {
+                    write!(f, " ({})", step.precedence)?;
+                }
+            }
+
+            write!(f, " ")?;
+            if step.symbol.is_terminal() {
+                if let Some(variable) = self.2.variables.get(step.symbol.index) {
+                    write!(f, "{}", &variable.name)?;
+                } else {
+                    write!(f, "terminal-{}", step.symbol.index)?;
+                }
+            } else if step.symbol.is_external() {
+                write!(f, "{}", &self.1.external_tokens[step.symbol.index].name)?;
+            } else {
+                write!(f, "{}", &self.1.variables[step.symbol.index].name)?;
+            }
+
+            if let Some(alias) = &step.alias {
+                write!(f, "@{}", alias.value)?;
+            }
+        }
+
+        if self.0.is_done() {
+            write!(f, " •")?;
+            if let Some(step) = self.0.production.steps.last() {
+                if let Some(associativity) = step.associativity {
+                    if step.precedence.is_none() {
+                        write!(f, " ({associativity:?})")?;
+                    } else {
+                        write!(f, " ({} {associativity:?})", step.precedence)?;
+                    }
+                } else if !step.precedence.is_none() {
+                    write!(f, " ({})", step.precedence)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a> fmt::Display for TokenSetDisplay<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "[")?;
+        for (i, symbol) in self.0.iter().enumerate() {
+            if i > 0 {
+                write!(f, ", ")?;
+            }
+
+            if symbol.is_terminal() {
+                if let Some(variable) = self.2.variables.get(symbol.index) {
+                    write!(f, "{}", &variable.name)?;
+                } else {
+                    write!(f, "terminal-{}", symbol.index)?;
+                }
+            } else if symbol.is_external() {
+                write!(f, "{}", &self.1.external_tokens[symbol.index].name)?;
+            } else {
+                write!(f, "{}", &self.1.variables[symbol.index].name)?;
+            }
+        }
+        write!(f, "]")?;
+        Ok(())
+    }
+}
+
+impl<'a> fmt::Display for ParseItemSetDisplay<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        for (item, lookaheads) in &self.0.entries {
+            writeln!(
+                f,
+                "{}\t{}",
+                ParseItemDisplay(item, self.1, self.2),
+                TokenSetDisplay(lookaheads, self.1, self.2)
+            )?;
+        }
+        Ok(())
+    }
+}
+
+impl<'a> Hash for ParseItem<'a> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_u32(self.variable_index);
+        hasher.write_u32(self.step_index);
+        hasher.write_i32(self.production.dynamic_precedence);
+        hasher.write_usize(self.production.steps.len());
+        hasher.write_i32(i32::from(self.has_preceding_inherited_fields));
+        self.precedence().hash(hasher);
+        self.associativity().hash(hasher);
+
+        // The already-matched children don't play any role in the parse state for
+        // this item, unless any of the following are true:
+        //   * the children have fields
+        //   * the children have aliases
+        //   * the children are hidden and
+        // See the docs for `has_preceding_inherited_fields`.
+        for step in &self.production.steps[0..self.step_index as usize] {
+            step.alias.hash(hasher);
+            step.field_name.hash(hasher);
+            if self.has_preceding_inherited_fields {
+                step.symbol.hash(hasher);
+            }
+        }
+        for step in &self.production.steps[self.step_index as usize..] {
+            step.hash(hasher);
+        }
+    }
+}
+
+impl<'a> PartialEq for ParseItem<'a> {
+    fn eq(&self, other: &Self) -> bool {
+        if self.variable_index != other.variable_index
+            || self.step_index != other.step_index
+            || self.production.dynamic_precedence != other.production.dynamic_precedence
+            || self.production.steps.len() != other.production.steps.len()
+            || self.precedence() != other.precedence()
+            || self.associativity() != other.associativity()
+            || self.has_preceding_inherited_fields != other.has_preceding_inherited_fields
+        {
+            return false;
+        }
+
+        for (i, step) in self.production.steps.iter().enumerate() {
+            // See the previous comment (in the `Hash::hash` impl) regarding comparisons
+            // of parse items' already-completed steps.
+            if i < self.step_index as usize {
+                if step.alias != other.production.steps[i].alias {
+                    return false;
+                }
+                if step.field_name != other.production.steps[i].field_name {
+                    return false;
+                }
+                if self.has_preceding_inherited_fields
+                    && step.symbol != other.production.steps[i].symbol
+                {
+                    return false;
+                }
+            } else if *step != other.production.steps[i] {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+impl<'a> Ord for ParseItem<'a> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.step_index
+            .cmp(&other.step_index)
+            .then_with(|| self.variable_index.cmp(&other.variable_index))
+            .then_with(|| {
+                self.production
+                    .dynamic_precedence
+                    .cmp(&other.production.dynamic_precedence)
+            })
+            .then_with(|| {
+                self.production
+                    .steps
+                    .len()
+                    .cmp(&other.production.steps.len())
+            })
+            .then_with(|| self.precedence().cmp(other.precedence()))
+            .then_with(|| self.associativity().cmp(&other.associativity()))
+            .then_with(|| {
+                for (i, step) in self.production.steps.iter().enumerate() {
+                    // See the previous comment (in the `Hash::hash` impl) regarding comparisons
+                    // of parse items' already-completed steps.
+                    let o = if i < self.step_index as usize {
+                        step.alias
+                            .cmp(&other.production.steps[i].alias)
+                            .then_with(|| {
+                                step.field_name.cmp(&other.production.steps[i].field_name)
+                            })
+                    } else {
+                        step.cmp(&other.production.steps[i])
+                    };
+                    if o != Ordering::Equal {
+                        return o;
+                    }
+                }
+                Ordering::Equal
+            })
+    }
+}
+
+impl<'a> PartialOrd for ParseItem<'a> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<'a> Eq for ParseItem<'a> {}
+
+impl<'a> Hash for ParseItemSet<'a> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_usize(self.entries.len());
+        for (item, lookaheads) in &self.entries {
+            item.hash(hasher);
+            lookaheads.hash(hasher);
+        }
+    }
+}
+
+impl<'a> Hash for ParseItemSetCore<'a> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_usize(self.entries.len());
+        for item in &self.entries {
+            item.hash(hasher);
+        }
+    }
+}

cli/generate/src/build_tables/item_set_builder.rs

@@ -0,0 +1,345 @@
+use std::{
+    collections::{HashMap, HashSet},
+    fmt,
+};
+
+use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, TokenSetDisplay};
+use crate::{
+    grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
+    rules::{Symbol, SymbolType, TokenSet},
+};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct TransitiveClosureAddition<'a> {
+    item: ParseItem<'a>,
+    info: FollowSetInfo,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct FollowSetInfo {
+    lookaheads: TokenSet,
+    propagates_lookaheads: bool,
+}
+
+pub struct ParseItemSetBuilder<'a> {
+    syntax_grammar: &'a SyntaxGrammar,
+    lexical_grammar: &'a LexicalGrammar,
+    first_sets: HashMap<Symbol, TokenSet>,
+    last_sets: HashMap<Symbol, TokenSet>,
+    inlines: &'a InlinedProductionMap,
+    transitive_closure_additions: Vec<Vec<TransitiveClosureAddition<'a>>>,
+}
+
+fn find_or_push<T: Eq>(vector: &mut Vec<T>, value: T) {
+    if !vector.contains(&value) {
+        vector.push(value);
+    }
+}
+
+impl<'a> ParseItemSetBuilder<'a> {
+    pub fn new(
+        syntax_grammar: &'a SyntaxGrammar,
+        lexical_grammar: &'a LexicalGrammar,
+        inlines: &'a InlinedProductionMap,
+    ) -> Self {
+        let mut result = Self {
+            syntax_grammar,
+            lexical_grammar,
+            first_sets: HashMap::new(),
+            last_sets: HashMap::new(),
+            inlines,
+            transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()],
+        };
+
+        // For each grammar symbol, populate the FIRST and LAST sets: the set of
+        // terminals that appear at the beginning and end that symbol's productions,
+        // respectively.
+        //
+        // For a terminal symbol, the FIRST and LAST set just consists of the
+        // terminal itself.
+        for i in 0..lexical_grammar.variables.len() {
+            let symbol = Symbol::terminal(i);
+            let mut set = TokenSet::new();
+            set.insert(symbol);
+            result.first_sets.insert(symbol, set.clone());
+            result.last_sets.insert(symbol, set);
+        }
+
+        for i in 0..syntax_grammar.external_tokens.len() {
+            let symbol = Symbol::external(i);
+            let mut set = TokenSet::new();
+            set.insert(symbol);
+            result.first_sets.insert(symbol, set.clone());
+            result.last_sets.insert(symbol, set);
+        }
+
+        // The FIRST set of a non-terminal `i` is the union of the following sets:
+        // * the set of all terminals that appear at the beginnings of i's productions
+        // * the FIRST sets of all the non-terminals that appear at the beginnings of i's
+        //   productions
+        //
+        // Rather than computing these sets using recursion, we use an explicit stack
+        // called `symbols_to_process`.
+        let mut symbols_to_process = Vec::new();
+        let mut processed_non_terminals = HashSet::new();
+        for i in 0..syntax_grammar.variables.len() {
+            let symbol = Symbol::non_terminal(i);
+
+            let first_set = result
+                .first_sets
+                .entry(symbol)
+                .or_insert_with(TokenSet::new);
+            processed_non_terminals.clear();
+            symbols_to_process.clear();
+            symbols_to_process.push(symbol);
+            while let Some(current_symbol) = symbols_to_process.pop() {
+                if current_symbol.is_terminal() || current_symbol.is_external() {
+                    first_set.insert(current_symbol);
+                } else if processed_non_terminals.insert(current_symbol) {
+                    for production in &syntax_grammar.variables[current_symbol.index].productions {
+                        if let Some(step) = production.steps.first() {
+                            symbols_to_process.push(step.symbol);
+                        }
+                    }
+                }
+            }
+
+            // The LAST set is defined in a similar way to the FIRST set.
+            let last_set = result.last_sets.entry(symbol).or_insert_with(TokenSet::new);
+            processed_non_terminals.clear();
+            symbols_to_process.clear();
+            symbols_to_process.push(symbol);
+            while let Some(current_symbol) = symbols_to_process.pop() {
+                if current_symbol.is_terminal() || current_symbol.is_external() {
+                    last_set.insert(current_symbol);
+                } else if processed_non_terminals.insert(current_symbol) {
+                    for production in &syntax_grammar.variables[current_symbol.index].productions {
+                        if let Some(step) = production.steps.last() {
+                            symbols_to_process.push(step.symbol);
+                        }
+                    }
+                }
+            }
+        }
+
+        // To compute an item set's transitive closure, we find each item in the set
+        // whose next symbol is a non-terminal, and we add new items to the set for
+        // each of that symbols' productions. These productions might themselves begin
+        // with non-terminals, so the process continues recursively. In this process,
+        // the total set of entries that get added depends only on two things:
+        //   * the set of non-terminal symbols that occur at each item's current position
+        //   * the set of terminals that occurs after each of these non-terminal symbols
+        //
+        // So we can avoid a lot of duplicated recursive work by precomputing, for each
+        // non-terminal symbol `i`, a final list of *additions* that must be made to an
+        // item set when `i` occurs as the next symbol in one if its core items. The
+        // structure of an *addition* is as follows:
+        //   * `item` - the new item that must be added as part of the expansion of `i`
+        //   * `lookaheads` - lookahead tokens that can always come after that item in the expansion
+        //     of `i`
+        //   * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the
+        //     *end* of the expansion of `i`, so that i's own current lookahead tokens can occur
+        //     after `item`.
+        //
+        // Again, rather than computing these additions recursively, we use an explicit
+        // stack called `entries_to_process`.
+        for i in 0..syntax_grammar.variables.len() {
+            let empty_lookaheads = TokenSet::new();
+            let mut entries_to_process = vec![(i, &empty_lookaheads, true)];
+
+            // First, build up a map whose keys are all of the non-terminals that can
+            // appear at the beginning of non-terminal `i`, and whose values store
+            // information about the tokens that can follow each non-terminal.
+            let mut follow_set_info_by_non_terminal = HashMap::new();
+            while let Some(entry) = entries_to_process.pop() {
+                let (variable_index, lookaheads, propagates_lookaheads) = entry;
+                let existing_info = follow_set_info_by_non_terminal
+                    .entry(variable_index)
+                    .or_insert_with(|| FollowSetInfo {
+                        lookaheads: TokenSet::new(),
+                        propagates_lookaheads: false,
+                    });
+
+                let did_add_follow_set_info;
+                if propagates_lookaheads {
+                    did_add_follow_set_info = !existing_info.propagates_lookaheads;
+                    existing_info.propagates_lookaheads = true;
+                } else {
+                    did_add_follow_set_info = existing_info.lookaheads.insert_all(lookaheads);
+                }
+
+                if did_add_follow_set_info {
+                    for production in &syntax_grammar.variables[variable_index].productions {
+                        if let Some(symbol) = production.first_symbol() {
+                            if symbol.is_non_terminal() {
+                                if production.steps.len() == 1 {
+                                    entries_to_process.push((
+                                        symbol.index,
+                                        lookaheads,
+                                        propagates_lookaheads,
+                                    ));
+                                } else {
+                                    entries_to_process.push((
+                                        symbol.index,
+                                        &result.first_sets[&production.steps[1].symbol],
+                                        false,
+                                    ));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            // Store all of those non-terminals' productions, along with their associated
+            // lookahead info, as *additions* associated with non-terminal `i`.
+            let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
+            for (variable_index, follow_set_info) in follow_set_info_by_non_terminal {
+                let variable = &syntax_grammar.variables[variable_index];
+                let non_terminal = Symbol::non_terminal(variable_index);
+                let variable_index = variable_index as u32;
+                if syntax_grammar.variables_to_inline.contains(&non_terminal) {
+                    continue;
+                }
+                for production in &variable.productions {
+                    let item = ParseItem {
+                        variable_index,
+                        production,
+                        step_index: 0,
+                        has_preceding_inherited_fields: false,
+                    };
+
+                    if let Some(inlined_productions) =
+                        inlines.inlined_productions(item.production, item.step_index)
+                    {
+                        for production in inlined_productions {
+                            find_or_push(
+                                additions_for_non_terminal,
+                                TransitiveClosureAddition {
+                                    item: item.substitute_production(production),
+                                    info: follow_set_info.clone(),
+                                },
+                            );
+                        }
+                    } else {
+                        find_or_push(
+                            additions_for_non_terminal,
+                            TransitiveClosureAddition {
+                                item,
+                                info: follow_set_info.clone(),
+                            },
+                        );
+                    }
+                }
+            }
+        }
+
+        result
+    }
+
+    pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
+        let mut result = ParseItemSet::default();
+        for (item, lookaheads) in &item_set.entries {
+            if let Some(productions) = self
+                .inlines
+                .inlined_productions(item.production, item.step_index)
+            {
+                for production in productions {
+                    self.add_item(
+                        &mut result,
+                        item.substitute_production(production),
+                        lookaheads,
+                    );
+                }
+            } else {
+                self.add_item(&mut result, *item, lookaheads);
+            }
+        }
+        result
+    }
+
+    pub fn first_set(&self, symbol: &Symbol) -> &TokenSet {
+        &self.first_sets[symbol]
+    }
+
+    pub fn last_set(&self, symbol: &Symbol) -> &TokenSet {
+        &self.last_sets[symbol]
+    }
+
+    fn add_item(&self, set: &mut ParseItemSet<'a>, item: ParseItem<'a>, lookaheads: &TokenSet) {
+        if let Some(step) = item.step() {
+            if step.symbol.is_non_terminal() {
+                let next_step = item.successor().step();
+
+                // Determine which tokens can follow this non-terminal.
+                let following_tokens = next_step.map_or(lookaheads, |next_step| {
+                    self.first_sets.get(&next_step.symbol).unwrap()
+                });
+
+                // Use the pre-computed *additions* to expand the non-terminal.
+                for addition in &self.transitive_closure_additions[step.symbol.index] {
+                    let lookaheads = set.insert(addition.item, &addition.info.lookaheads);
+                    if addition.info.propagates_lookaheads {
+                        lookaheads.insert_all(following_tokens);
+                    }
+                }
+            }
+        }
+        set.insert(item, lookaheads);
+    }
+}
+
+impl<'a> fmt::Debug for ParseItemSetBuilder<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "ParseItemSetBuilder {{")?;
+
+        writeln!(f, "  first_sets: {{")?;
+        for (symbol, first_set) in &self.first_sets {
+            let name = match symbol.kind {
+                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
+                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
+                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
+                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
+            };
+            writeln!(
+                f,
+                "    first({name:?}): {}",
+                TokenSetDisplay(first_set, self.syntax_grammar, self.lexical_grammar)
+            )?;
+        }
+        writeln!(f, "  }}")?;
+
+        writeln!(f, "  last_sets: {{")?;
+        for (symbol, last_set) in &self.last_sets {
+            let name = match symbol.kind {
+                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
+                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
+                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
+                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
+            };
+            writeln!(
+                f,
+                "    last({name:?}): {}",
+                TokenSetDisplay(last_set, self.syntax_grammar, self.lexical_grammar)
+            )?;
+        }
+        writeln!(f, "  }}")?;
+
+        writeln!(f, "  additions: {{")?;
+        for (i, variable) in self.syntax_grammar.variables.iter().enumerate() {
+            writeln!(f, "    {}: {{", variable.name)?;
+            for addition in &self.transitive_closure_additions[i] {
+                writeln!(
+                    f,
+                    "      {}",
+                    ParseItemDisplay(&addition.item, self.syntax_grammar, self.lexical_grammar)
+                )?;
+            }
+            writeln!(f, "    }},")?;
+        }
+        write!(f, "  }},")?;
+
+        write!(f, "}}")?;
+        Ok(())
+    }
+}

cli/generate/src/build_tables/minimize_parse_table.rs

@@ -0,0 +1,492 @@
+use std::{
+    collections::{HashMap, HashSet},
+    mem,
+};
+
+use log::info;
+
+use super::token_conflicts::TokenConflictMap;
+use crate::{
+    dedup::split_state_id_groups,
+    grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
+    rules::{AliasMap, Symbol, TokenSet},
+    tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
+};
+
+pub fn minimize_parse_table(
+    parse_table: &mut ParseTable,
+    syntax_grammar: &SyntaxGrammar,
+    lexical_grammar: &LexicalGrammar,
+    simple_aliases: &AliasMap,
+    token_conflict_map: &TokenConflictMap,
+    keywords: &TokenSet,
+) {
+    let mut minimizer = Minimizer {
+        parse_table,
+        syntax_grammar,
+        lexical_grammar,
+        token_conflict_map,
+        keywords,
+        simple_aliases,
+    };
+    minimizer.merge_compatible_states();
+    minimizer.remove_unit_reductions();
+    minimizer.remove_unused_states();
+    minimizer.reorder_states_by_descending_size();
+}
+
+struct Minimizer<'a> {
+    parse_table: &'a mut ParseTable,
+    syntax_grammar: &'a SyntaxGrammar,
+    lexical_grammar: &'a LexicalGrammar,
+    token_conflict_map: &'a TokenConflictMap<'a>,
+    keywords: &'a TokenSet,
+    simple_aliases: &'a AliasMap,
+}
+
+impl<'a> Minimizer<'a> {
+    fn remove_unit_reductions(&mut self) {
+        let mut aliased_symbols = HashSet::new();
+        for variable in &self.syntax_grammar.variables {
+            for production in &variable.productions {
+                for step in &production.steps {
+                    if step.alias.is_some() {
+                        aliased_symbols.insert(step.symbol);
+                    }
+                }
+            }
+        }
+
+        let mut unit_reduction_symbols_by_state = HashMap::new();
+        for (i, state) in self.parse_table.states.iter().enumerate() {
+            let mut only_unit_reductions = true;
+            let mut unit_reduction_symbol = None;
+            for (_, entry) in &state.terminal_entries {
+                for action in &entry.actions {
+                    match action {
+                        ParseAction::ShiftExtra => continue,
+                        ParseAction::Reduce {
+                            child_count: 1,
+                            production_id: 0,
+                            symbol,
+                            ..
+                        } if !self.simple_aliases.contains_key(symbol)
+                            && !self.syntax_grammar.supertype_symbols.contains(symbol)
+                            && !self.syntax_grammar.extra_symbols.contains(symbol)
+                            && !aliased_symbols.contains(symbol)
+                            && self.syntax_grammar.variables[symbol.index].kind
+                                != VariableType::Named
+                            && (unit_reduction_symbol.is_none()
+                                || unit_reduction_symbol == Some(symbol)) =>
+                        {
+                            unit_reduction_symbol = Some(symbol);
+                            continue;
+                        }
+                        _ => {}
+                    }
+                    only_unit_reductions = false;
+                    break;
+                }
+
+                if !only_unit_reductions {
+                    break;
+                }
+            }
+
+            if let Some(symbol) = unit_reduction_symbol {
+                if only_unit_reductions {
+                    unit_reduction_symbols_by_state.insert(i, *symbol);
+                }
+            }
+        }
+
+        for state in &mut self.parse_table.states {
+            let mut done = false;
+            while !done {
+                done = true;
+                state.update_referenced_states(|other_state_id, state| {
+                    unit_reduction_symbols_by_state.get(&other_state_id).map_or(
+                        other_state_id,
+                        |symbol| {
+                            done = false;
+                            match state.nonterminal_entries.get(symbol) {
+                                Some(GotoAction::Goto(state_id)) => *state_id,
+                                _ => other_state_id,
+                            }
+                        },
+                    )
+                });
+            }
+        }
+    }
+
+    fn merge_compatible_states(&mut self) {
+        let core_count = 1 + self
+            .parse_table
+            .states
+            .iter()
+            .map(|state| state.core_id)
+            .max()
+            .unwrap();
+
+        // Initially group the states by their parse item set core.
+        let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
+        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
+        for (i, state) in self.parse_table.states.iter().enumerate() {
+            state_ids_by_group_id[state.core_id].push(i);
+            group_ids_by_state_id.push(state.core_id);
+        }
+
+        split_state_id_groups(
+            &self.parse_table.states,
+            &mut state_ids_by_group_id,
+            &mut group_ids_by_state_id,
+            0,
+            |left, right, groups| self.states_conflict(left, right, groups),
+        );
+
+        while split_state_id_groups(
+            &self.parse_table.states,
+            &mut state_ids_by_group_id,
+            &mut group_ids_by_state_id,
+            0,
+            |left, right, groups| self.state_successors_differ(left, right, groups),
+        ) {
+            continue;
+        }
+
+        let error_group_index = state_ids_by_group_id
+            .iter()
+            .position(|g| g.contains(&0))
+            .unwrap();
+        let start_group_index = state_ids_by_group_id
+            .iter()
+            .position(|g| g.contains(&1))
+            .unwrap();
+        state_ids_by_group_id.swap(error_group_index, 0);
+        state_ids_by_group_id.swap(start_group_index, 1);
+
+        // Create a list of new parse states: one state for each group of old states.
+        let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
+        for state_ids in &state_ids_by_group_id {
+            // Initialize the new state based on the first old state in the group.
+            let mut parse_state = ParseState::default();
+            mem::swap(&mut parse_state, &mut self.parse_table.states[state_ids[0]]);
+
+            // Extend the new state with all of the actions from the other old states
+            // in the group.
+            for state_id in &state_ids[1..] {
+                let mut other_parse_state = ParseState::default();
+                mem::swap(
+                    &mut other_parse_state,
+                    &mut self.parse_table.states[*state_id],
+                );
+
+                parse_state
+                    .terminal_entries
+                    .extend(other_parse_state.terminal_entries);
+                parse_state
+                    .nonterminal_entries
+                    .extend(other_parse_state.nonterminal_entries);
+            }
+
+            // Update the new state's outgoing references using the new grouping.
+            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
+            new_states.push(parse_state);
+        }
+
+        self.parse_table.states = new_states;
+    }
+
+    fn states_conflict(
+        &self,
+        left_state: &ParseState,
+        right_state: &ParseState,
+        group_ids_by_state_id: &[ParseStateId],
+    ) -> bool {
+        for (token, left_entry) in &left_state.terminal_entries {
+            if let Some(right_entry) = right_state.terminal_entries.get(token) {
+                if self.entries_conflict(
+                    left_state.id,
+                    right_state.id,
+                    token,
+                    left_entry,
+                    right_entry,
+                    group_ids_by_state_id,
+                ) {
+                    return true;
+                }
+            } else if self.token_conflicts(
+                left_state.id,
+                right_state.id,
+                right_state.terminal_entries.keys(),
+                *token,
+            ) {
+                return true;
+            }
+        }
+
+        for token in right_state.terminal_entries.keys() {
+            if !left_state.terminal_entries.contains_key(token)
+                && self.token_conflicts(
+                    left_state.id,
+                    right_state.id,
+                    left_state.terminal_entries.keys(),
+                    *token,
+                )
+            {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn state_successors_differ(
+        &self,
+        state1: &ParseState,
+        state2: &ParseState,
+        group_ids_by_state_id: &[ParseStateId],
+    ) -> bool {
+        for (token, entry1) in &state1.terminal_entries {
+            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap() {
+                if let Some(entry2) = state2.terminal_entries.get(token) {
+                    if let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap() {
+                        let group1 = group_ids_by_state_id[*s1];
+                        let group2 = group_ids_by_state_id[*s2];
+                        if group1 != group2 {
+                            info!(
+                                "split states {} {} - successors for {} are split: {s1} {s2}",
+                                state1.id,
+                                state2.id,
+                                self.symbol_name(token),
+                            );
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+
+        for (symbol, s1) in &state1.nonterminal_entries {
+            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
+                match (s1, s2) {
+                    (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => continue,
+                    (GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
+                        let group1 = group_ids_by_state_id[*s1];
+                        let group2 = group_ids_by_state_id[*s2];
+                        if group1 != group2 {
+                            info!(
+                                "split states {} {} - successors for {} are split: {s1} {s2}",
+                                state1.id,
+                                state2.id,
+                                self.symbol_name(symbol),
+                            );
+                            return true;
+                        }
+                    }
+                    _ => return true,
+                }
+            }
+        }
+
+        false
+    }
+
+    fn entries_conflict(
+        &self,
+        state_id1: ParseStateId,
+        state_id2: ParseStateId,
+        token: &Symbol,
+        entry1: &ParseTableEntry,
+        entry2: &ParseTableEntry,
+        group_ids_by_state_id: &[ParseStateId],
+    ) -> bool {
+        // To be compatible, entries need to have the same actions.
+        let actions1 = &entry1.actions;
+        let actions2 = &entry2.actions;
+        if actions1.len() != actions2.len() {
+            info!(
+                "split states {state_id1} {state_id2} - differing action counts for token {}",
+                self.symbol_name(token)
+            );
+            return true;
+        }
+
+        for (i, action1) in actions1.iter().enumerate() {
+            let action2 = &actions2[i];
+
+            // Two shift actions are equivalent if their destinations are in the same group.
+            if let (
+                ParseAction::Shift {
+                    state: s1,
+                    is_repetition: is_repetition1,
+                },
+                ParseAction::Shift {
+                    state: s2,
+                    is_repetition: is_repetition2,
+                },
+            ) = (action1, action2)
+            {
+                let group1 = group_ids_by_state_id[*s1];
+                let group2 = group_ids_by_state_id[*s2];
+                if group1 == group2 && is_repetition1 == is_repetition2 {
+                    continue;
+                }
+                info!(
+                    "split states {state_id1} {state_id2} - successors for {} are split: {s1} {s2}",
+                    self.symbol_name(token),
+                );
+                return true;
+            } else if action1 != action2 {
+                info!(
+                    "split states {state_id1} {state_id2} - unequal actions for {}",
+                    self.symbol_name(token),
+                );
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn token_conflicts<'b>(
+        &self,
+        left_id: ParseStateId,
+        right_id: ParseStateId,
+        existing_tokens: impl Iterator<Item = &'b Symbol>,
+        new_token: Symbol,
+    ) -> bool {
+        if new_token == Symbol::end_of_nonterminal_extra() {
+            info!("split states {left_id} {right_id} - end of non-terminal extra",);
+            return true;
+        }
+
+        // Do not add external tokens; they could conflict lexically with any of the state's
+        // existing lookahead tokens.
+        if new_token.is_external() {
+            info!(
+                "split states {left_id} {right_id} - external token {}",
+                self.symbol_name(&new_token),
+            );
+            return true;
+        }
+
+        // Do not add tokens which are both internal and external. Their validity could
+        // influence the behavior of the external scanner.
+        if self
+            .syntax_grammar
+            .external_tokens
+            .iter()
+            .any(|external| external.corresponding_internal_token == Some(new_token))
+        {
+            info!(
+                "split states {left_id} {right_id} - internal/external token {}",
+                self.symbol_name(&new_token),
+            );
+            return true;
+        }
+
+        // Do not add a token if it conflicts with an existing token.
+        for token in existing_tokens {
+            if token.is_terminal()
+                && !(self.syntax_grammar.word_token == Some(*token)
+                    && self.keywords.contains(&new_token))
+                && !(self.syntax_grammar.word_token == Some(new_token)
+                    && self.keywords.contains(token))
+                && (self
+                    .token_conflict_map
+                    .does_conflict(new_token.index, token.index)
+                    || self
+                        .token_conflict_map
+                        .does_match_same_string(new_token.index, token.index))
+            {
+                info!(
+                    "split states {left_id} {right_id} - token {} conflicts with {}",
+                    self.symbol_name(&new_token),
+                    self.symbol_name(token),
+                );
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn symbol_name(&self, symbol: &Symbol) -> &String {
+        if symbol.is_non_terminal() {
+            &self.syntax_grammar.variables[symbol.index].name
+        } else if symbol.is_external() {
+            &self.syntax_grammar.external_tokens[symbol.index].name
+        } else {
+            &self.lexical_grammar.variables[symbol.index].name
+        }
+    }
+
+    fn remove_unused_states(&mut self) {
+        let mut state_usage_map = vec![false; self.parse_table.states.len()];
+
+        state_usage_map[0] = true;
+        state_usage_map[1] = true;
+
+        for state in &self.parse_table.states {
+            for referenced_state in state.referenced_states() {
+                state_usage_map[referenced_state] = true;
+            }
+        }
+        let mut removed_predecessor_count = 0;
+        let mut state_replacement_map = vec![0; self.parse_table.states.len()];
+        for state_id in 0..self.parse_table.states.len() {
+            state_replacement_map[state_id] = state_id - removed_predecessor_count;
+            if !state_usage_map[state_id] {
+                removed_predecessor_count += 1;
+            }
+        }
+        let mut state_id = 0;
+        let mut original_state_id = 0;
+        while state_id < self.parse_table.states.len() {
+            if state_usage_map[original_state_id] {
+                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
+                    state_replacement_map[other_state_id]
+                });
+                state_id += 1;
+            } else {
+                self.parse_table.states.remove(state_id);
+            }
+            original_state_id += 1;
+        }
+    }
+
+    fn reorder_states_by_descending_size(&mut self) {
+        // Get a mapping of old state index -> new_state_index
+        let mut old_ids_by_new_id = (0..self.parse_table.states.len()).collect::<Vec<_>>();
+        old_ids_by_new_id.sort_unstable_by_key(|i| {
+            // Don't changes states 0 (the error state) or 1 (the start state).
+            if *i <= 1 {
+                return *i as i64 - 1_000_000;
+            }
+
+            // Reorder all the other states by descending symbol count.
+            let state = &self.parse_table.states[*i];
+            -((state.terminal_entries.len() + state.nonterminal_entries.len()) as i64)
+        });
+
+        // Get the inverse mapping
+        let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
+        for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
+            new_ids_by_old_id[*old_id] = id;
+        }
+
+        // Reorder the parse states and update their references to reflect
+        // the new ordering.
+        self.parse_table.states = old_ids_by_new_id
+            .iter()
+            .map(|old_id| {
+                let mut state = ParseState::default();
+                mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
+                state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
+                state
+            })
+            .collect();
+    }
+}

cli/generate/src/build_tables/mod.rs

@@ -6,76 +6,53 @@ mod item_set_builder;
 mod minimize_parse_table;
 mod token_conflicts;
 
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashMap};
 
+use anyhow::Result;
 pub use build_lex_table::LARGE_CHARACTER_RANGE_COUNT;
-use build_parse_table::BuildTableResult;
-pub use build_parse_table::ParseTableBuilderError;
-use log::{debug, info};
-use rustc_hash::FxHashMap;
+use log::info;
 
 use self::{
     build_lex_table::build_lex_table,
-    build_parse_table::{ParseStateInfo, build_parse_table},
+    build_parse_table::{build_parse_table, ParseStateInfo},
     coincident_tokens::CoincidentTokenIndex,
-    item::ItemKeyMap,
-    item_set_builder::ParseItemSetBuilder,
     minimize_parse_table::minimize_parse_table,
     token_conflicts::TokenConflictMap,
 };
 use crate::{
-    Diagnostic, OptLevel,
     grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor},
     node_types::VariableInfo,
     rules::{AliasMap, Symbol, SymbolType, TokenSet},
-    strpool::StrPool,
-    tables::{ActionList, ActionListPool, LexTable, ParseAction, ParseTable, ParseTableEntry},
+    tables::{LexTable, ParseAction, ParseTable, ParseTableEntry},
 };
 
 pub struct Tables {
     pub parse_table: ParseTable,
     pub main_lex_table: LexTable,
     pub keyword_lex_table: LexTable,
+    pub word_token: Option<Symbol>,
     pub large_character_sets: Vec<(Option<Symbol>, CharacterSet)>,
 }
 
-#[expect(
-    clippy::too_many_arguments,
-    reason = "all parameters are required for table building"
-)]
 pub fn build_tables(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     simple_aliases: &AliasMap,
     variable_info: &[VariableInfo],
     inlines: &InlinedProductionMap,
-    str_pool: &StrPool,
     report_symbol_name: Option<&str>,
-    optimizations: OptLevel,
-    diagnostics: &mut Vec<Diagnostic>,
-) -> BuildTableResult<Tables> {
-    let item_key_map = ItemKeyMap::new(syntax_grammar, str_pool);
-    let item_set_builder =
-        ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines, &item_key_map);
-    let following_tokens = get_following_tokens(syntax_grammar, lexical_grammar, &item_set_builder);
-    let (mut parse_table, parse_state_info) = build_parse_table(
-        syntax_grammar,
-        lexical_grammar,
-        item_set_builder,
-        variable_info,
-        str_pool,
-        diagnostics,
-    )?;
+) -> Result<Tables> {
+    let (mut parse_table, following_tokens, parse_state_info) =
+        build_parse_table(syntax_grammar, lexical_grammar, inlines, variable_info)?;
     let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
-    let coincident_token_index =
-        CoincidentTokenIndex::new(&parse_table, lexical_grammar, syntax_grammar.word_token);
+    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
     let keywords = identify_keywords(
         lexical_grammar,
+        &parse_table,
         syntax_grammar.word_token,
         &token_conflict_map,
         &coincident_token_index,
-        str_pool,
     );
     populate_error_state(
         &mut parse_table,
@@ -84,10 +61,8 @@ pub fn build_tables(
         &coincident_token_index,
         &token_conflict_map,
         &keywords,
-        str_pool,
     );
     populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
-    let mut parse_table = ActionListPool::intern_table(parse_table);
     minimize_parse_table(
         &mut parse_table,
         syntax_grammar,
@@ -95,8 +70,6 @@ pub fn build_tables(
         simple_aliases,
         &token_conflict_map,
         &keywords,
-        str_pool,
-        optimizations,
     );
     let lex_tables = build_lex_table(
         &mut parse_table,
@@ -107,10 +80,7 @@ pub fn build_tables(
         &token_conflict_map,
     );
     populate_external_lex_states(&mut parse_table, syntax_grammar);
-    mark_fragile_tokens(&mut parse_table, &token_conflict_map);
-    parse_table
-        .action_lists
-        .canonicalize(&mut parse_table.states);
+    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
 
     if let Some(report_symbol_name) = report_symbol_name {
         report_state_info(
@@ -118,69 +88,26 @@ pub fn build_tables(
             lexical_grammar,
             &parse_table,
             &parse_state_info,
-            str_pool,
             report_symbol_name,
         );
     }
 
-    if parse_table.states.len() > u16::MAX as usize {
-        Err(ParseTableBuilderError::StateCount(parse_table.states.len()))?;
-    }
-
     Ok(Tables {
         parse_table,
         main_lex_table: lex_tables.main_lex_table,
         keyword_lex_table: lex_tables.keyword_lex_table,
         large_character_sets: lex_tables.large_character_sets,
+        word_token: syntax_grammar.word_token,
     })
 }
 
-fn get_following_tokens(
-    syntax_grammar: &SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-    builder: &ParseItemSetBuilder,
-) -> Vec<TokenSet> {
-    let n_terminals = lexical_grammar.variables.len();
-    let n_externals = syntax_grammar.external_tokens.len();
-    let mut result = vec![TokenSet::with_capacity(n_terminals, n_externals); n_terminals];
-    let all_tokens = (0..result.len())
-        .map(Symbol::terminal)
-        .collect::<TokenSet>();
-    for production in &syntax_grammar.productions {
-        let steps = &syntax_grammar.steps[production.step_range()];
-        for i in 1..steps.len() {
-            let left_tokens = builder.last_set(steps[i - 1].symbol());
-            let right_tokens = builder.first_set(steps[i].symbol());
-            let right_reserved_tokens = builder.reserved_first_set(steps[i].symbol());
-            for left_token in left_tokens.iter() {
-                if left_token.is_terminal() {
-                    result[left_token.index as usize].insert_all_terminals(right_tokens);
-                    if let Some(reserved_tokens) = right_reserved_tokens {
-                        result[left_token.index as usize].insert_all_terminals(reserved_tokens);
-                    }
-                }
-            }
-        }
-    }
-    for extra in &syntax_grammar.extra_symbols {
-        if extra.is_terminal() {
-            for entry in &mut result {
-                entry.insert(*extra);
-            }
-            result[extra.index as usize] = all_tokens.clone();
-        }
-    }
-    result
-}
-
 fn populate_error_state(
-    parse_table: &mut ParseTable<ParseTableEntry>,
+    parse_table: &mut ParseTable,
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     coincident_token_index: &CoincidentTokenIndex,
     token_conflict_map: &TokenConflictMap,
     keywords: &TokenSet,
-    str_pool: &StrPool,
 ) {
     let state = &mut parse_table.states[0];
     let n = lexical_grammar.variables.len();
@@ -197,9 +124,9 @@ fn populate_error_state(
             if conflicts_with_other_tokens {
                 None
             } else {
-                debug!(
+                info!(
                     "error recovery - token {} has no conflicts",
-                    str_pool.resolve(lexical_grammar.variables[i].name)
+                    lexical_grammar.variables[i].name
                 );
                 Some(Symbol::terminal(i))
             }
@@ -208,31 +135,31 @@ fn populate_error_state(
 
     let recover_entry = ParseTableEntry {
         reusable: false,
-        actions: ActionList::One(ParseAction::Recover),
+        actions: vec![ParseAction::Recover],
     };
 
     // Exclude from the error-recovery state any token that conflicts with one of
     // the *conflict-free tokens* identified above.
     for i in 0..n {
         let symbol = Symbol::terminal(i);
-        if !conflict_free_tokens.contains(symbol)
-            && !keywords.contains(symbol)
+        if !conflict_free_tokens.contains(&symbol)
+            && !keywords.contains(&symbol)
             && syntax_grammar.word_token != Some(symbol)
-            && let Some(t) = conflict_free_tokens.iter().find(|t| {
-                !coincident_token_index.contains(symbol, *t)
-                    && token_conflict_map.does_conflict(symbol.index as usize, t.index as usize)
-            })
         {
-            debug!(
-                "error recovery - exclude token {} because of conflict with {}",
-                str_pool.resolve(lexical_grammar.variables[i].name),
-                str_pool.resolve(lexical_grammar.variables[t.index as usize].name)
-            );
-            continue;
+            if let Some(t) = conflict_free_tokens.iter().find(|t| {
+                !coincident_token_index.contains(symbol, *t)
+                    && token_conflict_map.does_conflict(symbol.index, t.index)
+            }) {
+                info!(
+                    "error recovery - exclude token {} because of conflict with {}",
+                    lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
+                );
+                continue;
+            }
         }
-        debug!(
+        info!(
             "error recovery - include token {}",
-            str_pool.resolve(lexical_grammar.variables[i].name)
+            lexical_grammar.variables[i].name
         );
         state
             .terminal_entries
@@ -253,7 +180,7 @@ fn populate_error_state(
 }
 
 fn populate_used_symbols(
-    parse_table: &mut ParseTable<ParseTableEntry>,
+    parse_table: &mut ParseTable,
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
 ) {
@@ -263,13 +190,13 @@ fn populate_used_symbols(
     for state in &parse_table.states {
         for symbol in state.terminal_entries.keys() {
             match symbol.kind {
-                SymbolType::Terminal => terminal_usages[symbol.index as usize] = true,
-                SymbolType::External => external_usages[symbol.index as usize] = true,
+                SymbolType::Terminal => terminal_usages[symbol.index] = true,
+                SymbolType::External => external_usages[symbol.index] = true,
                 _ => {}
             }
         }
         for symbol in state.nonterminal_entries.keys() {
-            non_terminal_usages[symbol.index as usize] = true;
+            non_terminal_usages[symbol.index] = true;
         }
     }
     parse_table.symbols.push(Symbol::end());
@@ -281,10 +208,7 @@ fn populate_used_symbols(
             // ensure that a subtree's symbol can be successfully reassigned to the word token
             // without having to move the subtree to the heap.
             // See https://github.com/tree-sitter/tree-sitter/issues/258
-            if syntax_grammar
-                .word_token
-                .is_some_and(|t| t.index as usize == i)
-            {
+            if syntax_grammar.word_token.map_or(false, |t| t.index == i) {
                 parse_table.symbols.insert(1, Symbol::terminal(i));
             } else {
                 parse_table.symbols.push(Symbol::terminal(i));
@@ -304,7 +228,7 @@ fn populate_used_symbols(
 }
 
 fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &SyntaxGrammar) {
-    let mut external_tokens_by_corresponding_internal_token = FxHashMap::default();
+    let mut external_tokens_by_corresponding_internal_token = HashMap::new();
     for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
         if let Some(symbol) = external_token.corresponding_internal_token {
             external_tokens_by_corresponding_internal_token.insert(symbol.index, i);
@@ -320,11 +244,12 @@ fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &S
         for token in parse_table.states[i].terminal_entries.keys() {
             if token.is_external() {
                 external_tokens.insert(*token);
-            } else if token.is_terminal()
-                && let Some(index) =
+            } else if token.is_terminal() {
+                if let Some(index) =
                     external_tokens_by_corresponding_internal_token.get(&token.index)
-            {
-                external_tokens.insert(Symbol::external(*index));
+                {
+                    external_tokens.insert(Symbol::external(*index));
+                }
             }
         }
 
@@ -335,16 +260,16 @@ fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &S
             .unwrap_or_else(|| {
                 parse_table.external_lex_states.push(external_tokens);
                 parse_table.external_lex_states.len() - 1
-            }) as u32;
+            });
     }
 }
 
 fn identify_keywords(
     lexical_grammar: &LexicalGrammar,
+    parse_table: &ParseTable,
     word_token: Option<Symbol>,
     token_conflict_map: &TokenConflictMap,
     coincident_token_index: &CoincidentTokenIndex,
-    str_pool: &StrPool,
 ) -> TokenSet {
     if word_token.is_none() {
         return TokenSet::new();
@@ -362,12 +287,12 @@ fn identify_keywords(
         .filter_map(|(i, variable)| {
             cursor.reset(vec![variable.start_state]);
             if all_chars_are_alphabetical(&cursor)
-                && token_conflict_map.does_match_same_string(i, word_token.index as usize)
-                && !token_conflict_map.does_match_different_string(i, word_token.index as usize)
+                && token_conflict_map.does_match_same_string(i, word_token.index)
+                && !token_conflict_map.does_match_different_string(i, word_token.index)
             {
-                debug!(
+                info!(
                     "Keywords - add candidate {}",
-                    str_pool.resolve(lexical_grammar.variables[i].name)
+                    lexical_grammar.variables[i].name
                 );
                 Some(Symbol::terminal(i))
             } else {
@@ -382,14 +307,12 @@ fn identify_keywords(
         .filter(|token| {
             for other_token in keyword_candidates.iter() {
                 if other_token != *token
-                    && token_conflict_map
-                        .does_match_same_string(other_token.index as usize, token.index as usize)
+                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
                 {
-                    debug!(
+                    info!(
                         "Keywords - exclude {} because it matches the same string as {}",
-                        str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
-                        str_pool
-                            .resolve(lexical_grammar.variables[other_token.index as usize].name)
+                        lexical_grammar.variables[token.index].name,
+                        lexical_grammar.variables[other_token.index].name
                     );
                     return false;
                 }
@@ -400,12 +323,11 @@ fn identify_keywords(
 
     // Exclude keyword candidates for which substituting the keyword capture
     // token would introduce new lexical conflicts with other tokens.
-
-    keywords
+    let keywords = keywords
         .iter()
         .filter(|token| {
             for other_index in 0..lexical_grammar.variables.len() {
-                if keyword_candidates.contains(Symbol::terminal(other_index)) {
+                if keyword_candidates.contains(&Symbol::terminal(other_index)) {
                     continue;
                 }
 
@@ -413,48 +335,62 @@ fn identify_keywords(
                 // this keyword candidate, then substituting the word token won't
                 // introduce any new lexical conflicts.
                 if coincident_token_index
-                    .all_coincident_states_have_word(*token, Symbol::terminal(other_index))
+                    .states_with(*token, Symbol::terminal(other_index))
+                    .iter()
+                    .all(|state_id| {
+                        parse_table.states[*state_id]
+                            .terminal_entries
+                            .contains_key(&word_token)
+                    })
                 {
                     continue;
                 }
 
                 if !token_conflict_map.has_same_conflict_status(
-                    token.index as usize,
-                    word_token.index as usize,
+                    token.index,
+                    word_token.index,
                     other_index,
                 ) {
-                    debug!(
+                    info!(
                         "Keywords - exclude {} because of conflict with {}",
-                        str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
-                        str_pool.resolve(lexical_grammar.variables[other_index].name)
+                        lexical_grammar.variables[token.index].name,
+                        lexical_grammar.variables[other_index].name
                     );
                     return false;
                 }
             }
 
-            debug!(
+            info!(
                 "Keywords - include {}",
-                str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
+                lexical_grammar.variables[token.index].name,
             );
             true
         })
-        .collect()
+        .collect();
+
+    keywords
 }
 
-fn mark_fragile_tokens(parse_table: &mut ParseTable, token_conflict_map: &TokenConflictMap) {
-    let mut valid_terminal_indices = Vec::new();
+fn mark_fragile_tokens(
+    parse_table: &mut ParseTable,
+    lexical_grammar: &LexicalGrammar,
+    token_conflict_map: &TokenConflictMap,
+) {
+    let n = lexical_grammar.variables.len();
+    let mut valid_tokens_mask = Vec::with_capacity(n);
     for state in &mut parse_table.states {
-        valid_terminal_indices.clear();
+        valid_tokens_mask.clear();
+        valid_tokens_mask.resize(n, false);
         for token in state.terminal_entries.keys() {
             if token.is_terminal() {
-                valid_terminal_indices.push(token.index);
+                valid_tokens_mask[token.index] = true;
             }
         }
-        for (token, id) in &mut state.terminal_entries {
+        for (token, entry) in &mut state.terminal_entries {
             if token.is_terminal() {
-                for &i in &valid_terminal_indices {
-                    if token_conflict_map.does_overlap(i as usize, token.index as usize) {
-                        id.set_reusable(false);
+                for (i, is_valid) in valid_tokens_mask.iter().enumerate() {
+                    if *is_valid && token_conflict_map.does_overlap(i, token.index) {
+                        entry.reusable = false;
                         break;
                     }
                 }
@@ -467,8 +403,7 @@ fn report_state_info<'a>(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     parse_table: &ParseTable,
-    parse_state_info: &ParseStateInfo<'a>,
-    str_pool: &StrPool,
+    parse_state_info: &[ParseStateInfo<'a>],
     report_symbol_name: &'a str,
 ) {
     let mut all_state_indices = BTreeSet::new();
@@ -478,10 +413,10 @@ fn report_state_info<'a>(
 
     for (i, state) in parse_table.states.iter().enumerate() {
         all_state_indices.insert(i);
-        let item_set = parse_state_info.item_set(state.id);
-        for entry in &item_set.entries {
-            if !entry.item.is_augmented() {
-                symbols_with_state_indices[entry.item.variable_index as usize]
+        let item_set = &parse_state_info[state.id];
+        for (item, _) in &item_set.1.entries {
+            if !item.is_augmented() {
+                symbols_with_state_indices[item.variable_index as usize]
                     .1
                     .insert(i);
             }
@@ -493,18 +428,18 @@ fn report_state_info<'a>(
     let max_symbol_name_length = syntax_grammar
         .variables
         .iter()
-        .map(|v| str_pool.resolve(v.name).len())
+        .map(|v| v.name.len())
         .max()
         .unwrap();
     for (symbol, states) in &symbols_with_state_indices {
-        info!(
+        eprintln!(
             "{:width$}\t{}",
-            str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name),
+            syntax_grammar.variables[symbol.index].name,
             states.len(),
             width = max_symbol_name_length
         );
     }
-    info!("");
+    eprintln!();
 
     let state_indices = if report_symbol_name == "*" {
         Some(&all_state_indices)
@@ -512,9 +447,7 @@ fn report_state_info<'a>(
         symbols_with_state_indices
             .iter()
             .find_map(|(symbol, state_indices)| {
-                if str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name)
-                    == report_symbol_name
-                {
+                if syntax_grammar.variables[symbol.index].name == report_symbol_name {
                     Some(state_indices)
                 } else {
                     None
@@ -528,36 +461,23 @@ fn report_state_info<'a>(
 
         for state_index in state_indices {
             let id = parse_table.states[state_index].id;
-            let preceding_symbols = &parse_state_info.preceding_symbols_by_id[id as usize];
-            let item_set = parse_state_info.item_set(id);
-            info!("state index: {state_index}");
-            info!("state id: {id}");
-            info!(
-                "symbol sequence: {}",
-                preceding_symbols
-                    .iter()
-                    .map(|symbol| {
-                        if symbol.is_terminal() {
-                            str_pool.resolve(lexical_grammar.variables[symbol.index as usize].name)
-                        } else if symbol.is_external() {
-                            str_pool
-                                .resolve(syntax_grammar.external_tokens[symbol.index as usize].name)
-                        } else {
-                            str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name)
-                        }
-                    })
-                    .collect::<Vec<_>>()
-                    .join(" ")
-            );
-            info!(
+            let (preceding_symbols, item_set) = &parse_state_info[id];
+            eprintln!("state index: {state_index}");
+            eprintln!("state id: {id}");
+            eprint!("symbol sequence:");
+            for symbol in preceding_symbols {
+                let name = if symbol.is_terminal() {
+                    &lexical_grammar.variables[symbol.index].name
+                } else if symbol.is_external() {
+                    &syntax_grammar.external_tokens[symbol.index].name
+                } else {
+                    &syntax_grammar.variables[symbol.index].name
+                };
+                eprint!(" {name}");
+            }
+            eprintln!(
                 "\nitems:\n{}",
-                item::ParseItemSetDisplay(
-                    item_set,
-                    syntax_grammar,
-                    lexical_grammar,
-                    str_pool,
-                    &parse_state_info.lookaheads
-                ),
+                self::item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar,),
             );
         }
     }

cli/generate/src/build_tables/token_conflicts.rs

@@ -0,0 +1,529 @@
+use std::{cmp::Ordering, collections::HashSet, fmt};
+
+use crate::{
+    build_tables::item::TokenSetDisplay,
+    grammars::{LexicalGrammar, SyntaxGrammar},
+    nfa::{CharacterSet, NfaCursor, NfaTransition},
+    rules::TokenSet,
+};
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+struct TokenConflictStatus {
+    matches_prefix: bool,
+    does_match_continuation: bool,
+    does_match_valid_continuation: bool,
+    does_match_separators: bool,
+    matches_same_string: bool,
+    matches_different_string: bool,
+}
+
+pub struct TokenConflictMap<'a> {
+    n: usize,
+    status_matrix: Vec<TokenConflictStatus>,
+    following_tokens: Vec<TokenSet>,
+    starting_chars_by_index: Vec<CharacterSet>,
+    following_chars_by_index: Vec<CharacterSet>,
+    grammar: &'a LexicalGrammar,
+}
+
+impl<'a> TokenConflictMap<'a> {
+    /// Create a token conflict map based on a lexical grammar, which describes the structure
+    /// each token, and a `following_token` map, which indicates which tokens may be appear
+    /// immediately after each other token.
+    ///
+    /// This analyzes the possible kinds of overlap between each pair of tokens and stores
+    /// them in a matrix.
+    pub fn new(grammar: &'a LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
+        let mut cursor = NfaCursor::new(&grammar.nfa, Vec::new());
+        let starting_chars = get_starting_chars(&mut cursor, grammar);
+        let following_chars = get_following_chars(&starting_chars, &following_tokens);
+
+        let n = grammar.variables.len();
+        let mut status_matrix = vec![TokenConflictStatus::default(); n * n];
+        for i in 0..grammar.variables.len() {
+            for j in 0..i {
+                let status = compute_conflict_status(&mut cursor, grammar, &following_chars, i, j);
+                status_matrix[matrix_index(n, i, j)] = status.0;
+                status_matrix[matrix_index(n, j, i)] = status.1;
+            }
+        }
+
+        TokenConflictMap {
+            n,
+            status_matrix,
+            following_tokens,
+            starting_chars_by_index: starting_chars,
+            following_chars_by_index: following_chars,
+            grammar,
+        }
+    }
+
+    /// Does token `i` match any strings that token `j` also matches, such that token `i`
+    /// is preferred over token `j`?
+    pub fn has_same_conflict_status(&self, a: usize, b: usize, other: usize) -> bool {
+        let left = &self.status_matrix[matrix_index(self.n, a, other)];
+        let right = &self.status_matrix[matrix_index(self.n, b, other)];
+        left == right
+    }
+
+    /// Does token `i` match any strings that token `j` does *not* match?
+    pub fn does_match_different_string(&self, i: usize, j: usize) -> bool {
+        self.status_matrix[matrix_index(self.n, i, j)].matches_different_string
+    }
+
+    /// Does token `i` match any strings that token `j` also matches, where
+    /// token `i` is preferred over token `j`?
+    pub fn does_match_same_string(&self, i: usize, j: usize) -> bool {
+        self.status_matrix[matrix_index(self.n, i, j)].matches_same_string
+    }
+
+    pub fn does_conflict(&self, i: usize, j: usize) -> bool {
+        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
+        entry.does_match_valid_continuation
+            || entry.does_match_separators
+            || entry.matches_same_string
+    }
+
+    /// Does token `i` match any strings that are *prefixes* of strings matched by `j`?
+    pub fn does_match_prefix(&self, i: usize, j: usize) -> bool {
+        self.status_matrix[matrix_index(self.n, i, j)].matches_prefix
+    }
+
+    pub fn does_match_shorter_or_longer(&self, i: usize, j: usize) -> bool {
+        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
+        let reverse_entry = &self.status_matrix[matrix_index(self.n, j, i)];
+        (entry.does_match_valid_continuation || entry.does_match_separators)
+            && !reverse_entry.does_match_separators
+    }
+
+    pub fn does_overlap(&self, i: usize, j: usize) -> bool {
+        let status = &self.status_matrix[matrix_index(self.n, i, j)];
+        status.does_match_separators
+            || status.matches_prefix
+            || status.matches_same_string
+            || status.does_match_continuation
+    }
+
+    pub fn prefer_token(grammar: &LexicalGrammar, left: (i32, usize), right: (i32, usize)) -> bool {
+        match left.0.cmp(&right.0) {
+            Ordering::Less => false,
+            Ordering::Greater => true,
+            Ordering::Equal => match grammar.variables[left.1]
+                .implicit_precedence
+                .cmp(&grammar.variables[right.1].implicit_precedence)
+            {
+                Ordering::Less => false,
+                Ordering::Greater => true,
+                Ordering::Equal => left.1 < right.1,
+            },
+        }
+    }
+
+    pub fn prefer_transition(
+        grammar: &LexicalGrammar,
+        t: &NfaTransition,
+        completed_id: usize,
+        completed_precedence: i32,
+        has_separator_transitions: bool,
+    ) -> bool {
+        if t.precedence < completed_precedence {
+            return false;
+        }
+        if t.precedence == completed_precedence {
+            if t.is_separator {
+                return false;
+            }
+            if has_separator_transitions
+                && !grammar
+                    .variable_indices_for_nfa_states(&t.states)
+                    .any(|i| i == completed_id)
+            {
+                return false;
+            }
+        }
+        true
+    }
+}
+
+impl<'a> fmt::Debug for TokenConflictMap<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "TokenConflictMap {{")?;
+
+        let syntax_grammar = SyntaxGrammar::default();
+
+        writeln!(f, "  following_tokens: {{")?;
+        for (i, following_tokens) in self.following_tokens.iter().enumerate() {
+            writeln!(
+                f,
+                "    follow({:?}): {},",
+                self.grammar.variables[i].name,
+                TokenSetDisplay(following_tokens, &syntax_grammar, self.grammar)
+            )?;
+        }
+        writeln!(f, "  }},")?;
+
+        writeln!(f, "  starting_characters: {{")?;
+        for i in 0..self.n {
+            writeln!(
+                f,
+                "    {:?}: {:?},",
+                self.grammar.variables[i].name, self.starting_chars_by_index[i]
+            )?;
+        }
+        writeln!(f, "  }},")?;
+
+        writeln!(f, "  following_characters: {{")?;
+        for i in 0..self.n {
+            writeln!(
+                f,
+                "    {:?}: {:?},",
+                self.grammar.variables[i].name, self.following_chars_by_index[i]
+            )?;
+        }
+        writeln!(f, "  }},")?;
+
+        writeln!(f, "  status_matrix: {{")?;
+        for i in 0..self.n {
+            writeln!(f, "    {:?}: {{", self.grammar.variables[i].name)?;
+            for j in 0..self.n {
+                writeln!(
+                    f,
+                    "      {:?}: {:?},",
+                    self.grammar.variables[j].name,
+                    self.status_matrix[matrix_index(self.n, i, j)]
+                )?;
+            }
+            writeln!(f, "    }},")?;
+        }
+        write!(f, "  }},")?;
+        write!(f, "}}")?;
+        Ok(())
+    }
+}
+
+const fn matrix_index(variable_count: usize, i: usize, j: usize) -> usize {
+    variable_count * i + j
+}
+
+fn get_starting_chars(cursor: &mut NfaCursor, grammar: &LexicalGrammar) -> Vec<CharacterSet> {
+    let mut result = Vec::with_capacity(grammar.variables.len());
+    for variable in &grammar.variables {
+        cursor.reset(vec![variable.start_state]);
+        let mut all_chars = CharacterSet::empty();
+        for (chars, _) in cursor.transition_chars() {
+            all_chars = all_chars.add(chars);
+        }
+        result.push(all_chars);
+    }
+    result
+}
+
+fn get_following_chars(
+    starting_chars: &[CharacterSet],
+    following_tokens: &[TokenSet],
+) -> Vec<CharacterSet> {
+    following_tokens
+        .iter()
+        .map(|following_tokens| {
+            let mut chars = CharacterSet::empty();
+            for token in following_tokens.iter() {
+                if token.is_terminal() {
+                    chars = chars.add(&starting_chars[token.index]);
+                }
+            }
+            chars
+        })
+        .collect()
+}
+
+fn compute_conflict_status(
+    cursor: &mut NfaCursor,
+    grammar: &LexicalGrammar,
+    following_chars: &[CharacterSet],
+    i: usize,
+    j: usize,
+) -> (TokenConflictStatus, TokenConflictStatus) {
+    let mut visited_state_sets = HashSet::new();
+    let mut state_set_queue = vec![vec![
+        grammar.variables[i].start_state,
+        grammar.variables[j].start_state,
+    ]];
+    let mut result = (
+        TokenConflictStatus::default(),
+        TokenConflictStatus::default(),
+    );
+
+    while let Some(state_set) = state_set_queue.pop() {
+        let mut live_variable_indices = grammar.variable_indices_for_nfa_states(&state_set);
+
+        // If only one of the two tokens could possibly match from this state, then
+        // there is no reason to analyze any of its successors. Just record the fact
+        // that the token matches a string that the other token does not match.
+        let first_live_variable_index = live_variable_indices.next().unwrap();
+        if live_variable_indices.count() == 0 {
+            if first_live_variable_index == i {
+                result.0.matches_different_string = true;
+            } else {
+                result.1.matches_different_string = true;
+            }
+            continue;
+        }
+
+        // Don't pursue states where there's no potential for conflict.
+        cursor.reset(state_set);
+        let within_separator = cursor.transition_chars().any(|(_, sep)| sep);
+
+        // Examine each possible completed token in this state.
+        let mut completion = None;
+        for (id, precedence) in cursor.completions() {
+            if within_separator {
+                if id == i {
+                    result.0.does_match_separators = true;
+                } else {
+                    result.1.does_match_separators = true;
+                }
+            }
+
+            // If the other token has already completed, then this is
+            // a same-string conflict.
+            if let Some((prev_id, prev_precedence)) = completion {
+                if id == prev_id {
+                    continue;
+                }
+
+                // Determine which of the two tokens is preferred.
+                let preferred_id;
+                if TokenConflictMap::prefer_token(
+                    grammar,
+                    (prev_precedence, prev_id),
+                    (precedence, id),
+                ) {
+                    preferred_id = prev_id;
+                } else {
+                    preferred_id = id;
+                    completion = Some((id, precedence));
+                }
+
+                if preferred_id == i {
+                    result.0.matches_same_string = true;
+                } else {
+                    result.1.matches_same_string = true;
+                }
+            } else {
+                completion = Some((id, precedence));
+            }
+        }
+
+        // Examine each possible transition from this state to detect substring conflicts.
+        for transition in cursor.transitions() {
+            let mut can_advance = true;
+
+            // If there is already a completed token in this state, then determine
+            // if the next state can also match the completed token. If so, then
+            // this is *not* a conflict.
+            if let Some((completed_id, completed_precedence)) = completion {
+                let mut advanced_id = None;
+                let mut successor_contains_completed_id = false;
+                for variable_id in grammar.variable_indices_for_nfa_states(&transition.states) {
+                    if variable_id == completed_id {
+                        successor_contains_completed_id = true;
+                        break;
+                    }
+                    advanced_id = Some(variable_id);
+                }
+
+                // Determine which action is preferred: matching the already complete
+                // token, or continuing on to try and match the other longer token.
+                if let (Some(advanced_id), false) = (advanced_id, successor_contains_completed_id) {
+                    if TokenConflictMap::prefer_transition(
+                        grammar,
+                        &transition,
+                        completed_id,
+                        completed_precedence,
+                        within_separator,
+                    ) {
+                        can_advance = true;
+                        if advanced_id == i {
+                            result.0.does_match_continuation = true;
+                            if transition.characters.does_intersect(&following_chars[j]) {
+                                result.0.does_match_valid_continuation = true;
+                            }
+                        } else {
+                            result.1.does_match_continuation = true;
+                            if transition.characters.does_intersect(&following_chars[i]) {
+                                result.1.does_match_valid_continuation = true;
+                            }
+                        }
+                    } else if completed_id == i {
+                        result.0.matches_prefix = true;
+                    } else {
+                        result.1.matches_prefix = true;
+                    }
+                }
+            }
+
+            if can_advance && visited_state_sets.insert(transition.states.clone()) {
+                state_set_queue.push(transition.states);
+            }
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::{Variable, VariableType},
+        prepare_grammar::{expand_tokens, ExtractedLexicalGrammar},
+        rules::{Precedence, Rule, Symbol},
+    };
+
+    #[test]
+    fn test_starting_characters() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: Vec::new(),
+            variables: vec![
+                Variable {
+                    name: "token_0".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::pattern("[a-f]1|0x\\d", ""),
+                },
+                Variable {
+                    name: "token_1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::pattern("d*ef", ""),
+                },
+            ],
+        })
+        .unwrap();
+
+        let token_map = TokenConflictMap::new(&grammar, Vec::new());
+
+        assert_eq!(
+            token_map.starting_chars_by_index[0],
+            CharacterSet::empty().add_range('a', 'f').add_char('0')
+        );
+        assert_eq!(
+            token_map.starting_chars_by_index[1],
+            CharacterSet::empty().add_range('d', 'e')
+        );
+    }
+
+    #[test]
+    fn test_token_conflicts() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: Vec::new(),
+            variables: vec![
+                Variable {
+                    name: "in".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("in"),
+                },
+                Variable {
+                    name: "identifier".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::pattern("\\w+", ""),
+                },
+                Variable {
+                    name: "instanceof".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("instanceof"),
+                },
+            ],
+        })
+        .unwrap();
+
+        let var = |name| index_of_var(&grammar, name);
+
+        let token_map = TokenConflictMap::new(
+            &grammar,
+            vec![
+                std::iter::once(&Symbol::terminal(var("identifier")))
+                    .copied()
+                    .collect(),
+                std::iter::once(&Symbol::terminal(var("in")))
+                    .copied()
+                    .collect(),
+                std::iter::once(&Symbol::terminal(var("identifier")))
+                    .copied()
+                    .collect(),
+            ],
+        );
+
+        // Given the string "in", the `in` token is preferred over the `identifier` token
+        assert!(token_map.does_match_same_string(var("in"), var("identifier")));
+        assert!(!token_map.does_match_same_string(var("identifier"), var("in")));
+
+        // Depending on what character follows, the string "in" may be treated as part of an
+        // `identifier` token.
+        assert!(token_map.does_conflict(var("identifier"), var("in")));
+
+        // Depending on what character follows, the string "instanceof" may be treated as part of
+        // an `identifier` token.
+        assert!(token_map.does_conflict(var("identifier"), var("instanceof")));
+        assert!(token_map.does_conflict(var("instanceof"), var("in")));
+    }
+
+    #[test]
+    fn test_token_conflicts_with_separators() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: vec![Rule::pattern("\\s", "")],
+            variables: vec![
+                Variable {
+                    name: "x".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                Variable {
+                    name: "newline".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("\n"),
+                },
+            ],
+        })
+        .unwrap();
+
+        let var = |name| index_of_var(&grammar, name);
+
+        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
+
+        assert!(token_map.does_conflict(var("newline"), var("x")));
+        assert!(!token_map.does_conflict(var("x"), var("newline")));
+    }
+
+    #[test]
+    fn test_token_conflicts_with_open_ended_tokens() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: vec![Rule::pattern("\\s", "")],
+            variables: vec![
+                Variable {
+                    name: "x".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                Variable {
+                    name: "anything".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*", "")),
+                },
+            ],
+        })
+        .unwrap();
+
+        let var = |name| index_of_var(&grammar, name);
+
+        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
+
+        assert!(token_map.does_match_shorter_or_longer(var("anything"), var("x")));
+        assert!(!token_map.does_match_shorter_or_longer(var("x"), var("anything")));
+    }
+
+    fn index_of_var(grammar: &LexicalGrammar, name: &str) -> usize {
+        grammar
+            .variables
+            .iter()
+            .position(|v| v.name == name)
+            .unwrap()
+    }
+}

cli/generate/src/dedup.rs

@@ -0,0 +1,63 @@
+pub fn split_state_id_groups<S>(
+    states: &[S],
+    state_ids_by_group_id: &mut Vec<Vec<usize>>,
+    group_ids_by_state_id: &mut [usize],
+    start_group_id: usize,
+    mut f: impl FnMut(&S, &S, &[usize]) -> bool,
+) -> bool {
+    let mut result = false;
+
+    let mut group_id = start_group_id;
+    while group_id < state_ids_by_group_id.len() {
+        let state_ids = &state_ids_by_group_id[group_id];
+        let mut split_state_ids = Vec::new();
+
+        let mut i = 0;
+        while i < state_ids.len() {
+            let left_state_id = state_ids[i];
+            if split_state_ids.contains(&left_state_id) {
+                i += 1;
+                continue;
+            }
+
+            let left_state = &states[left_state_id];
+
+            // Identify all of the other states in the group that are incompatible with
+            // this state.
+            let mut j = i + 1;
+            while j < state_ids.len() {
+                let right_state_id = state_ids[j];
+                if split_state_ids.contains(&right_state_id) {
+                    j += 1;
+                    continue;
+                }
+                let right_state = &states[right_state_id];
+
+                if f(left_state, right_state, group_ids_by_state_id) {
+                    split_state_ids.push(right_state_id);
+                }
+
+                j += 1;
+            }
+
+            i += 1;
+        }
+
+        // If any states were removed from the group, add them all as a new group.
+        if !split_state_ids.is_empty() {
+            result = true;
+            state_ids_by_group_id[group_id].retain(|i| !split_state_ids.contains(i));
+
+            let new_group_id = state_ids_by_group_id.len();
+            for id in &split_state_ids {
+                group_ids_by_state_id[*id] = new_group_id;
+            }
+
+            state_ids_by_group_id.push(split_state_ids);
+        }
+
+        group_id += 1;
+    }
+
+    result
+}

cli/generate/src/dsl.js

@@ -16,7 +16,6 @@ function alias(rule, value) {
       result.value = value.symbol.name;
       return result;
     case Object:
-    case GrammarSymbol:
       if (typeof value.type === 'string' && value.type === 'SYMBOL') {
         result.named = true;
         result.value = value.name;
@@ -33,16 +32,7 @@ function blank() {
   };
 }
 
-function eof() {
-  return {
-    type: "EOF"
-  };
-}
-
 function field(name, rule) {
-  if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
-    throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`);
-  }
   return {
     type: "FIELD",
     name,
@@ -79,7 +69,7 @@ function prec(number, rule) {
   };
 }
 
-prec.left = function (number, rule) {
+prec.left = function(number, rule) {
   if (rule == null) {
     rule = number;
     number = 0;
@@ -101,7 +91,7 @@ prec.left = function (number, rule) {
   };
 }
 
-prec.right = function (number, rule) {
+prec.right = function(number, rule) {
   if (rule == null) {
     rule = number;
     number = 0;
@@ -123,7 +113,7 @@ prec.right = function (number, rule) {
   };
 }
 
-prec.dynamic = function (number, rule) {
+prec.dynamic = function(number, rule) {
   checkPrecedence(number);
   checkArguments(
     arguments,
@@ -163,26 +153,11 @@ function seq(...elements) {
   };
 }
 
-class GrammarSymbol {
-  constructor(name) {
-    this.type = "SYMBOL";
-    this.name = name;
-  }
-}
-
-function reserved(wordset, rule) {
-  if (typeof wordset !== 'string') {
-    throw new Error('Invalid reserved word set name: ' + wordset)
-  }
-  return {
-    type: "RESERVED",
-    content: normalize(rule),
-    context_name: wordset,
-  }
-}
-
 function sym(name) {
-  return new GrammarSymbol(name);
+  return {
+    type: "SYMBOL",
+    name
+  };
 }
 
 function token(value) {
@@ -193,7 +168,7 @@ function token(value) {
   };
 }
 
-token.immediate = function (value) {
+token.immediate = function(value) {
   checkArguments(arguments, arguments.length, token.immediate, 'token.immediate', '', 'literal');
   return {
     type: "IMMEDIATE_TOKEN",
@@ -220,11 +195,6 @@ function normalize(value) {
         type: 'PATTERN',
         value: value.source
       };
-    case RustRegex:
-      return {
-        type: 'PATTERN',
-        value: value.value
-      };
     case ReferenceError:
       throw value
     default:
@@ -266,7 +236,6 @@ function grammar(baseGrammar, options) {
       inline: [],
       supertypes: [],
       precedences: [],
-      reserved: {},
     };
   } else {
     baseGrammar = baseGrammar.grammar;
@@ -340,28 +309,6 @@ function grammar(baseGrammar, options) {
     }
   }
 
-  let reserved = { ...baseGrammar.reserved };
-  if (options.reserved) {
-    if (typeof options.reserved !== "object") {
-      throw new Error("Grammar's 'reserved' property must be an object.");
-    }
-
-    for (const reservedWordSetName of Object.keys(options.reserved)) {
-      const reservedWordSetFn = options.reserved[reservedWordSetName]
-      if (typeof reservedWordSetFn !== "function") {
-        throw new Error(`Grammar reserved word sets must all be functions. '${reservedWordSetName}' is not.`);
-      }
-
-      const reservedTokens = reservedWordSetFn.call(ruleBuilder, ruleBuilder, baseGrammar.reserved[reservedWordSetName]);
-
-      if (!Array.isArray(reservedTokens)) {
-        throw new Error(`Grammar's reserved word set functions must all return arrays of rules. '${reservedWordSetName}' does not.`);
-      }
-
-      reserved[reservedWordSetName] = reservedTokens.map(normalize);
-    }
-  }
-
   let extras = baseGrammar.extras.slice();
   if (options.extras) {
     if (typeof options.extras !== "function") {
@@ -380,16 +327,14 @@ function grammar(baseGrammar, options) {
 
   let word = baseGrammar.word;
   if (options.word) {
-    const wordRule = options.word.call(ruleBuilder, ruleBuilder);
-    if (wordRule?.name === 'ReferenceError') {
-      throw new Error("Grammar's 'word' property must be a valid named rule.");
-    }
-
-    if (wordRule?.type !== 'SYMBOL' || typeof wordRule.name !== 'string') {
+    word = options.word.call(ruleBuilder, ruleBuilder).name;
+    if (typeof word != 'string') {
       throw new Error("Grammar's 'word' property must be a named rule.");
     }
 
-    word = wordRule.name;
+    if (word === 'ReferenceError') {
+      throw new Error("Grammar's 'word' property must be a valid rule name.");
+    }
   }
 
   let conflicts = baseGrammar.conflicts;
@@ -410,13 +355,7 @@ function grammar(baseGrammar, options) {
         throw new Error("Grammar's conflicts must be an array of arrays of rules.");
       }
 
-      return conflictSet.map(symbol => {
-        const rule = normalize(symbol);
-        if (rule.type !== 'SYMBOL') {
-          throw new Error("Grammar's conflicts must contain only named rules.");
-        }
-        return rule.name;
-      });
+      return conflictSet.map(symbol => normalize(symbol).name);
     });
   }
 
@@ -434,15 +373,12 @@ function grammar(baseGrammar, options) {
     }
 
     inline = inlineRules.filter((symbol, index, self) => {
-      if (symbol?.name === 'ReferenceError') {
-        console.log(`Warning: inline rule '${symbol.symbol.name}' is not defined.`);
+      if (self.findIndex(s => s.name === symbol.name) !== index) {
+        console.log(`Warning: duplicate inline rule '${symbol.name}'`);
         return false;
       }
-      if (symbol?.type !== 'SYMBOL' || typeof symbol.name !== 'string') {
-        throw new Error("Grammar's inline property must contain only named rules.");
-      }
-      if (self.findIndex(s => s?.name === symbol.name) !== index) {
-        console.log(`Warning: duplicate inline rule '${symbol.name}'`);
+      if (symbol.name === 'ReferenceError') {
+        console.log(`Warning: inline rule '${symbol.symbol.name}' is not defined.`);
         return false;
       }
       return true;
@@ -463,12 +399,9 @@ function grammar(baseGrammar, options) {
     }
 
     supertypes = supertypeRules.map(symbol => {
-      if (symbol?.name === 'ReferenceError') {
+      if (symbol.name === 'ReferenceError') {
         throw new Error(`Supertype rule \`${symbol.symbol.name}\` is not defined.`);
       }
-      if (symbol?.type !== 'SYMBOL' || typeof symbol.name !== 'string') {
-        throw new Error("Grammar's supertypes property must contain only named rules.");
-      }
       return symbol.name;
     });
   }
@@ -486,15 +419,7 @@ function grammar(baseGrammar, options) {
       if (!Array.isArray(list)) {
         throw new Error("Grammar's precedences must be an array of arrays of rules.");
       }
-      return list.map(entry => {
-        const rule = normalize(entry);
-        if (rule.type !== 'STRING' && rule.type !== 'SYMBOL') {
-          throw new Error(
-            "Grammar's precedences must contain only precedence names or named rules."
-          );
-        }
-        return rule;
-      });
+      return list.map(normalize);
     });
   }
 
@@ -514,17 +439,10 @@ function grammar(baseGrammar, options) {
       externals,
       inline,
       supertypes,
-      reserved,
     },
   };
 }
 
-class RustRegex {
-  constructor(value) {
-    this.value = value;
-  }
-}
-
 function checkArguments(args, ruleCount, caller, callerName, suffix = '', argType = 'rule') {
   // Allow for .map() usage where additional arguments are index and the entire array.
   const isMapCall = ruleCount === 3 && typeof args[1] === 'number' && Array.isArray(args[2]);
@@ -548,7 +466,6 @@ function checkPrecedence(value) {
 }
 
 function getEnv(name) {
-  if (globalThis.native) return globalThis.__ts_grammar_path;
   if (globalThis.process) return process.env[name]; // Node/Bun
   if (globalThis.Deno) return Deno.env.get(name); // Deno
   throw Error("Unsupported JS runtime");
@@ -556,37 +473,25 @@ function getEnv(name) {
 
 globalThis.alias = alias;
 globalThis.blank = blank;
-globalThis.eof = eof;
 globalThis.choice = choice;
 globalThis.optional = optional;
 globalThis.prec = prec;
 globalThis.repeat = repeat;
 globalThis.repeat1 = repeat1;
-globalThis.reserved = reserved;
 globalThis.seq = seq;
 globalThis.sym = sym;
 globalThis.token = token;
 globalThis.grammar = grammar;
 globalThis.field = field;
-globalThis.RustRegex = RustRegex;
-
-const grammarPath = getEnv("TREE_SITTER_GRAMMAR_PATH");
-let result = await import(grammarPath);
-let grammarObj = result.default?.grammar ?? result.grammar;
-
-if (globalThis.native && !grammarObj) {
-  grammarObj = module.exports.grammar;
-}
 
+const result = await import(getEnv("TREE_SITTER_GRAMMAR_PATH"));
 const object = {
   "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
-  ...grammarObj,
+  ...(result.default?.grammar ?? result.grammar)
 };
 const output = JSON.stringify(object);
 
-if (globalThis.native) {
-  globalThis.output = output;
-} else if (globalThis.process) { // Node/Bun
+if (globalThis.process) { // Node/Bun
   process.stdout.write(output);
 } else if (globalThis.Deno) { // Deno
   Deno.stdout.writeSync(new TextEncoder().encode(output));

cli/generate/src/grammar_files.rs

@@ -0,0 +1 @@
+

cli/generate/src/grammars.rs

@@ -0,0 +1,261 @@
+use std::{collections::HashMap, fmt};
+
+use super::{
+    nfa::Nfa,
+    rules::{Alias, Associativity, Precedence, Rule, Symbol},
+};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum VariableType {
+    Hidden,
+    Auxiliary,
+    Anonymous,
+    Named,
+}
+
+// Input grammar
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Variable {
+    pub name: String,
+    pub kind: VariableType,
+    pub rule: Rule,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum PrecedenceEntry {
+    Name(String),
+    Symbol(String),
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct InputGrammar {
+    pub name: String,
+    pub variables: Vec<Variable>,
+    pub extra_symbols: Vec<Rule>,
+    pub expected_conflicts: Vec<Vec<String>>,
+    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
+    pub external_tokens: Vec<Rule>,
+    pub variables_to_inline: Vec<String>,
+    pub supertype_symbols: Vec<String>,
+    pub word_token: Option<String>,
+}
+
+// Extracted lexical grammar
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct LexicalVariable {
+    pub name: String,
+    pub kind: VariableType,
+    pub implicit_precedence: i32,
+    pub start_state: u32,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct LexicalGrammar {
+    pub nfa: Nfa,
+    pub variables: Vec<LexicalVariable>,
+}
+
+// Extracted syntax grammar
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct ProductionStep {
+    pub symbol: Symbol,
+    pub precedence: Precedence,
+    pub associativity: Option<Associativity>,
+    pub alias: Option<Alias>,
+    pub field_name: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Production {
+    pub steps: Vec<ProductionStep>,
+    pub dynamic_precedence: i32,
+}
+
+#[derive(Default)]
+pub struct InlinedProductionMap {
+    pub productions: Vec<Production>,
+    pub production_map: HashMap<(*const Production, u32), Vec<usize>>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SyntaxVariable {
+    pub name: String,
+    pub kind: VariableType,
+    pub productions: Vec<Production>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ExternalToken {
+    pub name: String,
+    pub kind: VariableType,
+    pub corresponding_internal_token: Option<Symbol>,
+}
+
+#[derive(Debug, Default)]
+pub struct SyntaxGrammar {
+    pub variables: Vec<SyntaxVariable>,
+    pub extra_symbols: Vec<Symbol>,
+    pub expected_conflicts: Vec<Vec<Symbol>>,
+    pub external_tokens: Vec<ExternalToken>,
+    pub supertype_symbols: Vec<Symbol>,
+    pub variables_to_inline: Vec<Symbol>,
+    pub word_token: Option<Symbol>,
+    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
+}
+
+#[cfg(test)]
+impl ProductionStep {
+    #[must_use]
+    pub const fn new(symbol: Symbol) -> Self {
+        Self {
+            symbol,
+            precedence: Precedence::None,
+            associativity: None,
+            alias: None,
+            field_name: None,
+        }
+    }
+
+    pub fn with_prec(self, precedence: Precedence, associativity: Option<Associativity>) -> Self {
+        Self {
+            symbol: self.symbol,
+            precedence,
+            associativity,
+            alias: self.alias,
+            field_name: self.field_name,
+        }
+    }
+
+    pub fn with_alias(self, value: &str, is_named: bool) -> Self {
+        Self {
+            symbol: self.symbol,
+            precedence: self.precedence,
+            associativity: self.associativity,
+            alias: Some(Alias {
+                value: value.to_string(),
+                is_named,
+            }),
+            field_name: self.field_name,
+        }
+    }
+    pub fn with_field_name(self, name: &str) -> Self {
+        Self {
+            symbol: self.symbol,
+            precedence: self.precedence,
+            associativity: self.associativity,
+            alias: self.alias,
+            field_name: Some(name.to_string()),
+        }
+    }
+}
+
+impl Production {
+    pub fn first_symbol(&self) -> Option<Symbol> {
+        self.steps.first().map(|s| s.symbol)
+    }
+}
+
+#[cfg(test)]
+impl Variable {
+    pub fn named(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Named,
+            rule,
+        }
+    }
+
+    pub fn auxiliary(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Auxiliary,
+            rule,
+        }
+    }
+
+    pub fn hidden(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Hidden,
+            rule,
+        }
+    }
+
+    pub fn anonymous(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Anonymous,
+            rule,
+        }
+    }
+}
+
+impl VariableType {
+    pub fn is_visible(self) -> bool {
+        self == Self::Named || self == Self::Anonymous
+    }
+}
+
+impl LexicalGrammar {
+    pub fn variable_indices_for_nfa_states<'a>(
+        &'a self,
+        state_ids: &'a [u32],
+    ) -> impl Iterator<Item = usize> + 'a {
+        let mut prev = None;
+        state_ids.iter().filter_map(move |state_id| {
+            let variable_id = self.variable_index_for_nfa_state(*state_id);
+            if prev == Some(variable_id) {
+                None
+            } else {
+                prev = Some(variable_id);
+                prev
+            }
+        })
+    }
+
+    pub fn variable_index_for_nfa_state(&self, state_id: u32) -> usize {
+        self.variables
+            .iter()
+            .position(|v| v.start_state >= state_id)
+            .unwrap()
+    }
+}
+
+impl SyntaxVariable {
+    pub fn is_auxiliary(&self) -> bool {
+        self.kind == VariableType::Auxiliary
+    }
+
+    pub fn is_hidden(&self) -> bool {
+        self.kind == VariableType::Hidden || self.kind == VariableType::Auxiliary
+    }
+}
+
+impl InlinedProductionMap {
+    pub fn inlined_productions<'a>(
+        &'a self,
+        production: &Production,
+        step_index: u32,
+    ) -> Option<impl Iterator<Item = &'a Production> + 'a> {
+        self.production_map
+            .get(&(production as *const Production, step_index))
+            .map(|production_indices| {
+                production_indices
+                    .iter()
+                    .copied()
+                    .map(move |index| &self.productions[index])
+            })
+    }
+}
+
+impl fmt::Display for PrecedenceEntry {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Name(n) => write!(f, "'{n}'"),
+            Self::Symbol(s) => write!(f, "$.{s}"),
+        }
+    }
+}

cli/generate/src/lib.rs

@@ -0,0 +1,252 @@
+use std::{
+    env, fs,
+    io::Write,
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+};
+
+use anyhow::{anyhow, Context, Result};
+use build_tables::build_tables;
+use grammars::InputGrammar;
+use lazy_static::lazy_static;
+use parse_grammar::parse_grammar;
+use prepare_grammar::prepare_grammar;
+use regex::{Regex, RegexBuilder};
+use render::render_c_code;
+use semver::Version;
+
+mod build_tables;
+mod dedup;
+mod grammar_files;
+mod grammars;
+mod nfa;
+mod node_types;
+pub mod parse_grammar;
+mod prepare_grammar;
+mod render;
+mod rules;
+mod tables;
+
+lazy_static! {
+    static ref JSON_COMMENT_REGEX: Regex = RegexBuilder::new("^\\s*//.*")
+        .multi_line(true)
+        .build()
+        .unwrap();
+}
+
+struct GeneratedParser {
+    c_code: String,
+    node_types_json: String,
+}
+
+pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h");
+pub const ARRAY_HEADER: &str = include_str!("templates/array.h");
+
+pub fn generate_parser_in_directory(
+    repo_path: &Path,
+    grammar_path: Option<&str>,
+    abi_version: usize,
+    report_symbol_name: Option<&str>,
+    js_runtime: Option<&str>,
+) -> Result<()> {
+    let mut repo_path = repo_path.to_owned();
+    let mut grammar_path = grammar_path;
+
+    // Populate a new empty grammar directory.
+    if let Some(path) = grammar_path {
+        let path = PathBuf::from(path);
+        if !path
+            .try_exists()
+            .with_context(|| "Some error with specified path")?
+        {
+            fs::create_dir_all(&path)?;
+            grammar_path = None;
+            repo_path = path;
+        }
+    }
+
+    let grammar_path = grammar_path
+        .map(PathBuf::from)
+        .unwrap_or_else(|| repo_path.join("grammar.js"));
+
+    // Read the grammar file.
+    let grammar_json = load_grammar_file(&grammar_path, js_runtime)?;
+
+    let src_path = repo_path.join("src");
+    let header_path = src_path.join("tree_sitter");
+
+    // Ensure that the output directories exist.
+    fs::create_dir_all(&src_path)?;
+    fs::create_dir_all(&header_path)?;
+
+    if grammar_path.file_name().unwrap() != "grammar.json" {
+        fs::write(src_path.join("grammar.json"), &grammar_json)
+            .with_context(|| format!("Failed to write grammar.json to {src_path:?}"))?;
+    }
+
+    // Parse and preprocess the grammar.
+    let input_grammar = parse_grammar(&grammar_json)?;
+
+    // Generate the parser and related files.
+    let GeneratedParser {
+        c_code,
+        node_types_json,
+    } = generate_parser_for_grammar_with_opts(&input_grammar, abi_version, report_symbol_name)?;
+
+    write_file(&src_path.join("parser.c"), c_code)?;
+    write_file(&src_path.join("node-types.json"), node_types_json)?;
+    write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
+    write_file(&header_path.join("array.h"), ARRAY_HEADER)?;
+    write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
+
+    Ok(())
+}
+
+pub fn generate_parser_for_grammar(grammar_json: &str) -> Result<(String, String)> {
+    let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n");
+    let input_grammar = parse_grammar(&grammar_json)?;
+    let parser =
+        generate_parser_for_grammar_with_opts(&input_grammar, tree_sitter::LANGUAGE_VERSION, None)?;
+    Ok((input_grammar.name, parser.c_code))
+}
+
+fn generate_parser_for_grammar_with_opts(
+    input_grammar: &InputGrammar,
+    abi_version: usize,
+    report_symbol_name: Option<&str>,
+) -> Result<GeneratedParser> {
+    let (syntax_grammar, lexical_grammar, inlines, simple_aliases) =
+        prepare_grammar(input_grammar)?;
+    let variable_info =
+        node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?;
+    let node_types_json = node_types::generate_node_types_json(
+        &syntax_grammar,
+        &lexical_grammar,
+        &simple_aliases,
+        &variable_info,
+    );
+    let tables = build_tables(
+        &syntax_grammar,
+        &lexical_grammar,
+        &simple_aliases,
+        &variable_info,
+        &inlines,
+        report_symbol_name,
+    )?;
+    let c_code = render_c_code(
+        &input_grammar.name,
+        tables,
+        syntax_grammar,
+        lexical_grammar,
+        simple_aliases,
+        abi_version,
+    );
+    Ok(GeneratedParser {
+        c_code,
+        node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(),
+    })
+}
+
+pub fn load_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result<String> {
+    if grammar_path.is_dir() {
+        return Err(anyhow!(
+            "Path to a grammar file with `.js` or `.json` extension is required"
+        ));
+    }
+    match grammar_path.extension().and_then(|e| e.to_str()) {
+        Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)
+            .with_context(|| "Failed to load grammar.js")?),
+        Some("json") => {
+            Ok(fs::read_to_string(grammar_path).with_context(|| "Failed to load grammar.json")?)
+        }
+        _ => Err(anyhow!("Unknown grammar file extension: {grammar_path:?}",)),
+    }
+}
+
+fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result<String> {
+    let grammar_path = fs::canonicalize(grammar_path)?;
+
+    #[cfg(windows)]
+    let grammar_path = url::Url::from_file_path(grammar_path)
+        .expect("Failed to convert path to URL")
+        .to_string();
+
+    let js_runtime = js_runtime.unwrap_or("node");
+
+    let mut js_command = Command::new(js_runtime);
+    match js_runtime {
+        "node" => {
+            js_command.args(["--input-type=module", "-"]);
+        }
+        "bun" => {
+            js_command.arg("-");
+        }
+        "deno" => {
+            js_command.args(["run", "--allow-all", "-"]);
+        }
+        _ => {}
+    }
+
+    let mut js_process = js_command
+        .env("TREE_SITTER_GRAMMAR_PATH", grammar_path)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .spawn()
+        .with_context(|| format!("Failed to run `{js_runtime}`"))?;
+
+    let mut js_stdin = js_process
+        .stdin
+        .take()
+        .with_context(|| format!("Failed to open stdin for {js_runtime}"))?;
+    let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))
+        .with_context(|| "Could not parse this package's version as semver.")?;
+    write!(
+        js_stdin,
+        "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {};
+         globalThis.TREE_SITTER_CLI_VERSION_MINOR = {};
+         globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
+        cli_version.major, cli_version.minor, cli_version.patch,
+    )
+    .with_context(|| format!("Failed to write tree-sitter version to {js_runtime}'s stdin"))?;
+    js_stdin
+        .write(include_bytes!("./dsl.js"))
+        .with_context(|| format!("Failed to write grammar dsl to {js_runtime}'s stdin"))?;
+    drop(js_stdin);
+
+    let output = js_process
+        .wait_with_output()
+        .with_context(|| format!("Failed to read output from {js_runtime}"))?;
+    match output.status.code() {
+        None => panic!("{js_runtime} process was killed"),
+        Some(0) => {
+            let stdout = String::from_utf8(output.stdout)
+                .with_context(|| format!("Got invalid UTF8 from {js_runtime}"))?;
+
+            let mut grammar_json = &stdout[..];
+
+            if let Some(pos) = stdout.rfind('\n') {
+                // If there's a newline, split the last line from the rest of the output
+                let node_output = &stdout[..pos];
+                grammar_json = &stdout[pos + 1..];
+
+                let mut stdout = std::io::stdout().lock();
+                stdout.write_all(node_output.as_bytes())?;
+                stdout.write_all(b"\n")?;
+                stdout.flush()?;
+            }
+
+            Ok(serde_json::to_string_pretty(
+                &serde_json::from_str::<serde_json::Value>(grammar_json)
+                    .with_context(|| "Failed to parse grammar JSON")?,
+            )
+            .with_context(|| "Failed to serialize grammar JSON")?
+                + "\n")
+        }
+        Some(code) => Err(anyhow!("{js_runtime} process exited with status {code}")),
+    }
+}
+
+pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> Result<()> {
+    fs::write(path, body)
+        .with_context(|| format!("Failed to write {:?}", path.file_name().unwrap()))
+}

cli/generate/src/nfa.rs

@@ -1,5 +1,6 @@
 use std::{
-    cmp::{Ordering, max},
+    char,
+    cmp::{max, Ordering},
     fmt,
     iter::ExactSizeIterator,
     mem::{self, swap},
@@ -51,18 +52,13 @@ const END: u32 = char::MAX as u32 + 1;
 
 impl CharacterSet {
     /// Create a character set with a single character.
-    #[must_use]
     pub const fn empty() -> Self {
         Self { ranges: Vec::new() }
     }
 
     /// Create a character set with a given *inclusive* range of characters.
-    #[expect(
-        clippy::single_range_in_vec_init,
-        reason = "Vec is the backing store for CharacterSet"
-    )]
-    #[cfg(test)]
-    fn from_range(mut first: char, mut last: char) -> Self {
+    #[allow(clippy::single_range_in_vec_init)]
+    pub fn from_range(mut first: char, mut last: char) -> Self {
         if first > last {
             swap(&mut first, &mut last);
         }
@@ -72,11 +68,7 @@ impl CharacterSet {
     }
 
     /// Create a character set with a single character.
-    #[must_use]
-    #[expect(
-        clippy::single_range_in_vec_init,
-        reason = "Vec is the backing store for CharacterSet"
-    )]
+    #[allow(clippy::single_range_in_vec_init)]
     pub fn from_char(c: char) -> Self {
         Self {
             ranges: vec![(c as u32)..(c as u32 + 1)],
@@ -85,7 +77,6 @@ impl CharacterSet {
 
     /// Create a character set containing all characters *not* present
     /// in this character set.
-    #[must_use]
     pub fn negate(mut self) -> Self {
         let mut i = 0;
         let mut previous_end = 0;
@@ -106,20 +97,16 @@ impl CharacterSet {
         self
     }
 
-    #[must_use]
     pub fn add_char(mut self, c: char) -> Self {
         self.add_int_range(0, c as u32, c as u32 + 1);
         self
     }
 
-    #[must_use]
     pub fn add_range(mut self, start: char, end: char) -> Self {
         self.add_int_range(0, start as u32, end as u32 + 1);
         self
     }
 
-    #[must_use]
-    #[allow(clippy::should_implement_trait)]
     pub fn add(mut self, other: &Self) -> Self {
         let mut index = 0;
         for range in &other.ranges {
@@ -158,7 +145,6 @@ impl CharacterSet {
         i
     }
 
-    #[must_use]
     pub fn does_intersect(&self, other: &Self) -> bool {
         let mut left_ranges = self.ranges.iter();
         let mut right_ranges = other.ranges.iter();
@@ -179,7 +165,6 @@ impl CharacterSet {
     /// Get the set of characters that are present in both this set
     /// and the other set. Remove those common characters from both
     /// of the operands.
-    #[allow(clippy::return_self_not_must_use)]
     pub fn remove_intersection(&mut self, other: &mut Self) -> Self {
         let mut intersection = Vec::new();
         let mut left_i = 0;
@@ -294,11 +279,6 @@ impl CharacterSet {
 
     /// Produces a `CharacterSet` containing every character in `self` that is not present in
     /// `other`.
-    #[allow(
-        clippy::must_use_candidate,
-        clippy::return_self_not_must_use,
-        dead_code
-    )]
     pub fn difference(mut self, mut other: Self) -> Self {
         self.remove_intersection(&mut other);
         self
@@ -306,8 +286,7 @@ impl CharacterSet {
 
     /// Produces a `CharacterSet` containing every character that is in _exactly one_ of `self` or
     /// `other`, but is not present in both sets.
-    #[cfg(test)]
-    fn symmetric_difference(mut self, mut other: Self) -> Self {
+    pub fn symmetric_difference(mut self, mut other: Self) -> Self {
         self.remove_intersection(&mut other);
         self.add(&other)
     }
@@ -320,8 +299,7 @@ impl CharacterSet {
         self.char_codes().filter_map(char::from_u32)
     }
 
-    #[must_use]
-    pub const fn range_count(&self) -> usize {
+    pub fn range_count(&self) -> usize {
         self.ranges.len()
     }
 
@@ -333,14 +311,12 @@ impl CharacterSet {
         })
     }
 
-    #[must_use]
-    pub const fn is_empty(&self) -> bool {
+    pub fn is_empty(&self) -> bool {
         self.ranges.is_empty()
     }
 
     /// Get a reduced list of character ranges, assuming that a given
     /// set of characters can be safely ignored.
-    #[must_use]
     pub fn simplify_ignoring(&self, ruled_out_characters: &Self) -> Self {
         let mut prev_range: Option<Range<u32>> = None;
         Self {
@@ -355,12 +331,13 @@ impl CharacterSet {
                             return None;
                         }
 
-                        if let Some(prev_range) = &mut prev_range
-                            && ruled_out_characters
+                        if let Some(prev_range) = &mut prev_range {
+                            if ruled_out_characters
                                 .contains_codepoint_range(prev_range.end..range.start)
-                        {
-                            prev_range.end = range.end;
-                            return None;
+                            {
+                                prev_range.end = range.end;
+                                return None;
+                            }
                         }
                     }
 
@@ -372,7 +349,6 @@ impl CharacterSet {
         }
     }
 
-    #[must_use]
     pub fn contains_codepoint_range(&self, seek_range: Range<u32>) -> bool {
         let ix = match self.ranges.binary_search_by(|probe| {
             if probe.end <= seek_range.start {
@@ -385,12 +361,11 @@ impl CharacterSet {
         }) {
             Ok(ix) | Err(ix) => ix,
         };
-        self.ranges
-            .get(ix)
-            .is_some_and(|range| range.start <= seek_range.start && range.end >= seek_range.end)
+        self.ranges.get(ix).map_or(false, |range| {
+            range.start <= seek_range.start && range.end >= seek_range.end
+        })
     }
 
-    #[must_use]
     pub fn contains(&self, c: char) -> bool {
         self.contains_codepoint_range(c as u32..c as u32 + 1)
     }
@@ -456,9 +431,7 @@ impl Nfa {
         Self { states: Vec::new() }
     }
 
-    #[must_use]
     pub fn last_state_id(&self) -> u32 {
-        assert_ne!(self.states, [] as [NfaState; 0]);
         self.states.len() as u32 - 1
     }
 }
@@ -475,7 +448,6 @@ impl fmt::Debug for Nfa {
 }
 
 impl<'a> NfaCursor<'a> {
-    #[must_use]
     pub fn new(nfa: &'a Nfa, mut states: Vec<u32>) -> Self {
         let mut result = Self {
             nfa,
@@ -498,25 +470,10 @@ impl<'a> NfaCursor<'a> {
         self.raw_transitions().map(|t| (t.0, t.1))
     }
 
-    #[must_use]
     pub fn transitions(&self) -> Vec<NfaTransition> {
         Self::group_transitions(self.raw_transitions())
     }
 
-    /// Like [`transitions()`](Self::transitions) but also returns whether any raw NFA transition
-    /// is a separator. This is computed in the same pass, avoiding a second
-    /// iteration over `state_ids` for callers that need both.
-    #[must_use]
-    pub fn transitions_and_any_sep(&self) -> (Vec<NfaTransition>, bool) {
-        let mut any_sep = false;
-        let result =
-            Self::group_transitions(self.raw_transitions().map(|(chars, is_sep, prec, state)| {
-                any_sep |= is_sep;
-                (chars, is_sep, prec, state)
-            }));
-        (result, any_sep)
-    }
-
     fn raw_transitions(&self) -> impl Iterator<Item = (&CharacterSet, bool, i32, u32)> {
         self.state_ids.iter().filter_map(move |id| {
             if let NfaState::Advance {
@@ -537,22 +494,13 @@ impl<'a> NfaCursor<'a> {
         iter: impl Iterator<Item = (&'b CharacterSet, bool, i32, u32)>,
     ) -> Vec<NfaTransition> {
         let mut result = Vec::<NfaTransition>::new();
-        // Reuse a single CharacterSet buffer across iterations to avoid one
-        // malloc per raw transition. `assign` refills it in-place; `mem::take`
-        // donates the allocation to a result entry when chars has a remainder.
-        let mut chars = CharacterSet::empty();
-        for (input_chars, is_sep, prec, state) in iter {
-            chars.assign(input_chars);
+        for (chars, is_sep, prec, state) in iter {
+            let mut chars = chars.clone();
             let mut i = 0;
             while i < result.len() && !chars.is_empty() {
                 let intersection = result[i].characters.remove_intersection(&mut chars);
                 if !intersection.is_empty() {
-                    let chars_is_empty = result[i].characters.is_empty();
-                    let mut intersection_states = if chars_is_empty {
-                        mem::take(&mut result[i].states)
-                    } else {
-                        result[i].states.clone()
-                    };
+                    let mut intersection_states = result[i].states.clone();
                     if let Err(j) = intersection_states.binary_search(&state) {
                         intersection_states.insert(j, state);
                     }
@@ -562,23 +510,18 @@ impl<'a> NfaCursor<'a> {
                         precedence: max(result[i].precedence, prec),
                         states: intersection_states,
                     };
-                    if chars_is_empty {
+                    if result[i].characters.is_empty() {
                         result[i] = intersection_transition;
                     } else {
-                        // Push to the tail instead of inserting at i (which
-                        // would be O(n)).  After remove_intersection, the new
-                        // `chars` (C') and `intersection` (I = A∩C) are
-                        // disjoint, so when the loop later reaches I at the
-                        // tail, remove_intersection(I, C'') will be a no-op.
-                        // The final sort makes mid-loop ordering irrelevant.
-                        result.push(intersection_transition);
+                        result.insert(i, intersection_transition);
+                        i += 1;
                     }
                 }
                 i += 1;
             }
             if !chars.is_empty() {
                 result.push(NfaTransition {
-                    characters: mem::take(&mut chars),
+                    characters: chars,
                     precedence: prec,
                     states: vec![state],
                     is_separator: is_sep,
@@ -595,7 +538,7 @@ impl<'a> NfaCursor<'a> {
                 {
                     let characters = mem::take(&mut result[j].characters);
                     result[j].characters = characters.add(&result[i].characters);
-                    result.swap_remove(i);
+                    result.remove(i);
                     i -= 1;
                     break;
                 }
@@ -1004,19 +947,20 @@ mod tests {
             assert_eq!(
                 left.remove_intersection(&mut right),
                 row.intersection,
-                "row {i}a: {:?} && {:?}",
+                "row {}a: {:?} && {:?}",
+                i,
                 row.left,
                 row.right
             );
             assert_eq!(
                 left, row.left_only,
-                "row {i}a: {:?} - {:?}",
-                row.left, row.right
+                "row {}a: {:?} - {:?}",
+                i, row.left, row.right
             );
             assert_eq!(
                 right, row.right_only,
-                "row {i}a: {:?} - {:?}",
-                row.right, row.left
+                "row {}a: {:?} - {:?}",
+                i, row.right, row.left
             );
 
             let mut left = row.left.clone();
@@ -1024,25 +968,27 @@ mod tests {
             assert_eq!(
                 right.remove_intersection(&mut left),
                 row.intersection,
-                "row {i}b: {:?} && {:?}",
+                "row {}b: {:?} && {:?}",
+                i,
                 row.left,
                 row.right
             );
             assert_eq!(
                 left, row.left_only,
-                "row {i}b: {:?} - {:?}",
-                row.left, row.right
+                "row {}b: {:?} - {:?}",
+                i, row.left, row.right
             );
             assert_eq!(
                 right, row.right_only,
-                "row {i}b: {:?} - {:?}",
-                row.right, row.left
+                "row {}b: {:?} - {:?}",
+                i, row.right, row.left
             );
 
             assert_eq!(
                 row.left.clone().difference(row.right.clone()),
                 row.left_only,
-                "row {i}b: {:?} -- {:?}",
+                "row {}b: {:?} -- {:?}",
+                i,
                 row.left,
                 row.right
             );
@@ -1115,10 +1061,7 @@ mod tests {
     }
 
     #[test]
-    #[expect(
-        clippy::single_range_in_vec_init,
-        reason = "test data intentionally uses single-element ranges"
-    )]
+    #[allow(clippy::single_range_in_vec_init)]
     fn test_character_set_simplify_ignoring() {
         struct Row {
             chars: Vec<char>,

cli/generate/src/parse_grammar.rs

@@ -0,0 +1,343 @@
+use std::collections::HashSet;
+
+use anyhow::{anyhow, Result};
+use serde::Deserialize;
+use serde_json::{Map, Value};
+
+use super::{
+    grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType},
+    rules::{Precedence, Rule},
+};
+
+#[derive(Deserialize)]
+#[serde(tag = "type")]
+#[allow(non_camel_case_types)]
+#[allow(clippy::upper_case_acronyms)]
+enum RuleJSON {
+    ALIAS {
+        content: Box<RuleJSON>,
+        named: bool,
+        value: String,
+    },
+    BLANK,
+    STRING {
+        value: String,
+    },
+    PATTERN {
+        value: String,
+        flags: Option<String>,
+    },
+    SYMBOL {
+        name: String,
+    },
+    CHOICE {
+        members: Vec<RuleJSON>,
+    },
+    FIELD {
+        name: String,
+        content: Box<RuleJSON>,
+    },
+    SEQ {
+        members: Vec<RuleJSON>,
+    },
+    REPEAT {
+        content: Box<RuleJSON>,
+    },
+    REPEAT1 {
+        content: Box<RuleJSON>,
+    },
+    PREC_DYNAMIC {
+        value: i32,
+        content: Box<RuleJSON>,
+    },
+    PREC_LEFT {
+        value: PrecedenceValueJSON,
+        content: Box<RuleJSON>,
+    },
+    PREC_RIGHT {
+        value: PrecedenceValueJSON,
+        content: Box<RuleJSON>,
+    },
+    PREC {
+        value: PrecedenceValueJSON,
+        content: Box<RuleJSON>,
+    },
+    TOKEN {
+        content: Box<RuleJSON>,
+    },
+    IMMEDIATE_TOKEN {
+        content: Box<RuleJSON>,
+    },
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum PrecedenceValueJSON {
+    Integer(i32),
+    Name(String),
+}
+
+#[derive(Deserialize)]
+pub struct GrammarJSON {
+    pub name: String,
+    rules: Map<String, Value>,
+    #[serde(default)]
+    precedences: Vec<Vec<RuleJSON>>,
+    #[serde(default)]
+    conflicts: Vec<Vec<String>>,
+    #[serde(default)]
+    externals: Vec<RuleJSON>,
+    #[serde(default)]
+    extras: Vec<RuleJSON>,
+    #[serde(default)]
+    inline: Vec<String>,
+    #[serde(default)]
+    supertypes: Vec<String>,
+    word: Option<String>,
+}
+
+fn rule_is_referenced(rule: &Rule, target: &str) -> bool {
+    match rule {
+        Rule::NamedSymbol(name) => name == target,
+        Rule::Choice(rules) | Rule::Seq(rules) => {
+            rules.iter().any(|r| rule_is_referenced(r, target))
+        }
+        Rule::Metadata { rule, .. } => rule_is_referenced(rule, target),
+        Rule::Repeat(inner) => rule_is_referenced(inner, target),
+        Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false,
+    }
+}
+
+fn variable_is_used(
+    grammar_rules: &[(String, Rule)],
+    other_rules: (&[Rule], &[Rule]),
+    target_name: &str,
+    in_progress: &mut HashSet<String>,
+) -> bool {
+    let root = &grammar_rules.first().unwrap().0;
+    if target_name == root {
+        return true;
+    }
+
+    if other_rules
+        .0
+        .iter()
+        .chain(other_rules.1.iter())
+        .any(|rule| rule_is_referenced(rule, target_name))
+    {
+        return true;
+    }
+
+    in_progress.insert(target_name.to_string());
+    let result = grammar_rules
+        .iter()
+        .filter(|(key, _)| *key != target_name)
+        .any(|(name, rule)| {
+            if !rule_is_referenced(rule, target_name) || in_progress.contains(name) {
+                return false;
+            }
+            variable_is_used(grammar_rules, other_rules, name, in_progress)
+        });
+    in_progress.remove(target_name);
+
+    result
+}
+
+pub(crate) fn parse_grammar(input: &str) -> Result<InputGrammar> {
+    let mut grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
+
+    let mut extra_symbols =
+        grammar_json
+            .extras
+            .into_iter()
+            .try_fold(Vec::new(), |mut acc, item| {
+                let rule = parse_rule(item);
+                if let Rule::String(ref value) = rule {
+                    if value.is_empty() {
+                        return Err(anyhow!(
+                            "Rules in the `extras` array must not contain empty strings"
+                        ));
+                    }
+                }
+                acc.push(rule);
+                Ok(acc)
+            })?;
+
+    let mut external_tokens = grammar_json
+        .externals
+        .into_iter()
+        .map(parse_rule)
+        .collect::<Vec<_>>();
+
+    let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len());
+    for list in grammar_json.precedences {
+        let mut ordering = Vec::with_capacity(list.len());
+        for entry in list {
+            ordering.push(match entry {
+                RuleJSON::STRING { value } => PrecedenceEntry::Name(value),
+                RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name),
+                _ => {
+                    return Err(anyhow!(
+                        "Invalid rule in precedences array. Only strings and symbols are allowed"
+                    ))
+                }
+            });
+        }
+        precedence_orderings.push(ordering);
+    }
+
+    let mut variables = Vec::with_capacity(grammar_json.rules.len());
+
+    let rules = grammar_json
+        .rules
+        .into_iter()
+        .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?))))
+        .collect::<Result<Vec<_>>>()?;
+
+    let mut in_progress = HashSet::new();
+
+    for (name, rule) in &rules {
+        if !variable_is_used(
+            &rules,
+            (&extra_symbols, &external_tokens),
+            name,
+            &mut in_progress,
+        ) && grammar_json.word.as_ref().map_or(true, |w| w != name)
+        {
+            grammar_json.conflicts.retain(|r| !r.contains(name));
+            grammar_json.supertypes.retain(|r| r != name);
+            grammar_json.inline.retain(|r| r != name);
+            extra_symbols.retain(|r| !rule_is_referenced(r, name));
+            external_tokens.retain(|r| !rule_is_referenced(r, name));
+            precedence_orderings.retain(|r| {
+                !r.iter().any(|e| {
+                    let PrecedenceEntry::Symbol(s) = e else {
+                        return false;
+                    };
+                    s == name
+                })
+            });
+            continue;
+        }
+        variables.push(Variable {
+            name: name.clone(),
+            kind: VariableType::Named,
+            rule: rule.clone(),
+        });
+    }
+
+    Ok(InputGrammar {
+        name: grammar_json.name,
+        word_token: grammar_json.word,
+        expected_conflicts: grammar_json.conflicts,
+        supertype_symbols: grammar_json.supertypes,
+        variables_to_inline: grammar_json.inline,
+        precedence_orderings,
+        variables,
+        extra_symbols,
+        external_tokens,
+    })
+}
+
+fn parse_rule(json: RuleJSON) -> Rule {
+    match json {
+        RuleJSON::ALIAS {
+            content,
+            value,
+            named,
+        } => Rule::alias(parse_rule(*content), value, named),
+        RuleJSON::BLANK => Rule::Blank,
+        RuleJSON::STRING { value } => Rule::String(value),
+        RuleJSON::PATTERN { value, flags } => Rule::Pattern(
+            value,
+            flags.map_or(String::new(), |f| {
+                f.matches(|c| {
+                    if c == 'i' {
+                        true
+                    } else {
+                        // silently ignore unicode flags
+                        if c != 'u' && c != 'v' {
+                            eprintln!("Warning: unsupported flag {c}");
+                        }
+                        false
+                    }
+                })
+                .collect()
+            }),
+        ),
+        RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name),
+        RuleJSON::CHOICE { members } => Rule::choice(members.into_iter().map(parse_rule).collect()),
+        RuleJSON::FIELD { content, name } => Rule::field(name, parse_rule(*content)),
+        RuleJSON::SEQ { members } => Rule::seq(members.into_iter().map(parse_rule).collect()),
+        RuleJSON::REPEAT1 { content } => Rule::repeat(parse_rule(*content)),
+        RuleJSON::REPEAT { content } => {
+            Rule::choice(vec![Rule::repeat(parse_rule(*content)), Rule::Blank])
+        }
+        RuleJSON::PREC { value, content } => Rule::prec(value.into(), parse_rule(*content)),
+        RuleJSON::PREC_LEFT { value, content } => {
+            Rule::prec_left(value.into(), parse_rule(*content))
+        }
+        RuleJSON::PREC_RIGHT { value, content } => {
+            Rule::prec_right(value.into(), parse_rule(*content))
+        }
+        RuleJSON::PREC_DYNAMIC { value, content } => {
+            Rule::prec_dynamic(value, parse_rule(*content))
+        }
+        RuleJSON::TOKEN { content } => Rule::token(parse_rule(*content)),
+        RuleJSON::IMMEDIATE_TOKEN { content } => Rule::immediate_token(parse_rule(*content)),
+    }
+}
+
+impl From<PrecedenceValueJSON> for Precedence {
+    fn from(val: PrecedenceValueJSON) -> Self {
+        match val {
+            PrecedenceValueJSON::Integer(i) => Self::Integer(i),
+            PrecedenceValueJSON::Name(i) => Self::Name(i),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_grammar() {
+        let grammar = parse_grammar(
+            r#"{
+            "name": "my_lang",
+            "rules": {
+                "file": {
+                    "type": "REPEAT1",
+                    "content": {
+                        "type": "SYMBOL",
+                        "name": "statement"
+                    }
+                },
+                "statement": {
+                    "type": "STRING",
+                    "value": "foo"
+                }
+            }
+        }"#,
+        )
+        .unwrap();
+
+        assert_eq!(grammar.name, "my_lang");
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable {
+                    name: "file".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::repeat(Rule::NamedSymbol("statement".to_string()))
+                },
+                Variable {
+                    name: "statement".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::String("foo".to_string())
+                },
+            ]
+        );
+    }
+}

cli/generate/src/prepare_grammar/expand_repeats.rs

@@ -0,0 +1,289 @@
+use std::{collections::HashMap, mem};
+
+use super::ExtractedSyntaxGrammar;
+use crate::{
+    grammars::{Variable, VariableType},
+    rules::{Rule, Symbol},
+};
+
+struct Expander {
+    variable_name: String,
+    repeat_count_in_variable: usize,
+    preceding_symbol_count: usize,
+    auxiliary_variables: Vec<Variable>,
+    existing_repeats: HashMap<Rule, Symbol>,
+}
+
+impl Expander {
+    fn expand_variable(&mut self, index: usize, variable: &mut Variable) -> bool {
+        self.variable_name.clear();
+        self.variable_name.push_str(&variable.name);
+        self.repeat_count_in_variable = 0;
+        let mut rule = Rule::Blank;
+        mem::swap(&mut rule, &mut variable.rule);
+
+        // In the special case of a hidden variable with a repetition at its top level,
+        // convert that rule itself into a binary tree structure instead of introducing
+        // another auxiliary rule.
+        if let (VariableType::Hidden, Rule::Repeat(repeated_content)) = (variable.kind, &rule) {
+            let inner_rule = self.expand_rule(repeated_content);
+            variable.rule = self.wrap_rule_in_binary_tree(Symbol::non_terminal(index), inner_rule);
+            variable.kind = VariableType::Auxiliary;
+            return true;
+        }
+
+        variable.rule = self.expand_rule(&rule);
+        false
+    }
+
+    fn expand_rule(&mut self, rule: &Rule) -> Rule {
+        match rule {
+            // For choices, sequences, and metadata, descend into the child rules,
+            // replacing any nested repetitions.
+            Rule::Choice(elements) => Rule::Choice(
+                elements
+                    .iter()
+                    .map(|element| self.expand_rule(element))
+                    .collect(),
+            ),
+
+            Rule::Seq(elements) => Rule::Seq(
+                elements
+                    .iter()
+                    .map(|element| self.expand_rule(element))
+                    .collect(),
+            ),
+
+            Rule::Metadata { rule, params } => Rule::Metadata {
+                rule: Box::new(self.expand_rule(rule)),
+                params: params.clone(),
+            },
+
+            // For repetitions, introduce an auxiliary rule that contains the
+            // repeated content, but can also contain a recursive binary tree structure.
+            Rule::Repeat(content) => {
+                let inner_rule = self.expand_rule(content);
+
+                if let Some(existing_symbol) = self.existing_repeats.get(&inner_rule) {
+                    return Rule::Symbol(*existing_symbol);
+                }
+
+                self.repeat_count_in_variable += 1;
+                let rule_name = format!(
+                    "{}_repeat{}",
+                    self.variable_name, self.repeat_count_in_variable
+                );
+                let repeat_symbol = Symbol::non_terminal(
+                    self.preceding_symbol_count + self.auxiliary_variables.len(),
+                );
+                self.existing_repeats
+                    .insert(inner_rule.clone(), repeat_symbol);
+                self.auxiliary_variables.push(Variable {
+                    name: rule_name,
+                    kind: VariableType::Auxiliary,
+                    rule: self.wrap_rule_in_binary_tree(repeat_symbol, inner_rule),
+                });
+
+                Rule::Symbol(repeat_symbol)
+            }
+
+            // For primitive rules, don't change anything.
+            _ => rule.clone(),
+        }
+    }
+
+    fn wrap_rule_in_binary_tree(&self, symbol: Symbol, rule: Rule) -> Rule {
+        Rule::choice(vec![
+            Rule::Seq(vec![Rule::Symbol(symbol), Rule::Symbol(symbol)]),
+            rule,
+        ])
+    }
+}
+
+pub(super) fn expand_repeats(mut grammar: ExtractedSyntaxGrammar) -> ExtractedSyntaxGrammar {
+    let mut expander = Expander {
+        variable_name: String::new(),
+        repeat_count_in_variable: 0,
+        preceding_symbol_count: grammar.variables.len(),
+        auxiliary_variables: Vec::new(),
+        existing_repeats: HashMap::new(),
+    };
+
+    for (i, variable) in grammar.variables.iter_mut().enumerate() {
+        let expanded_top_level_repetition = expander.expand_variable(i, variable);
+
+        // If a hidden variable had a top-level repetition and it was converted to
+        // a recursive rule, then it can't be inlined.
+        if expanded_top_level_repetition {
+            grammar
+                .variables_to_inline
+                .retain(|symbol| *symbol != Symbol::non_terminal(i));
+        }
+    }
+
+    grammar.variables.extend(expander.auxiliary_variables);
+    grammar
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_repeat_expansion() {
+        // Repeats nested inside of sequences and choices are expanded.
+        let grammar = expand_repeats(build_grammar(vec![Variable::named(
+            "rule0",
+            Rule::seq(vec![
+                Rule::terminal(10),
+                Rule::choice(vec![
+                    Rule::repeat(Rule::terminal(11)),
+                    Rule::repeat(Rule::terminal(12)),
+                ]),
+                Rule::terminal(13),
+            ]),
+        )]));
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "rule0",
+                    Rule::seq(vec![
+                        Rule::terminal(10),
+                        Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),]),
+                        Rule::terminal(13),
+                    ])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
+                        Rule::terminal(11),
+                    ])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat2",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
+                        Rule::terminal(12),
+                    ])
+                ),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_repeat_deduplication() {
+        // Terminal 4 appears inside of a repeat in three different places.
+        let grammar = expand_repeats(build_grammar(vec![
+            Variable::named(
+                "rule0",
+                Rule::choice(vec![
+                    Rule::seq(vec![Rule::terminal(1), Rule::repeat(Rule::terminal(4))]),
+                    Rule::seq(vec![Rule::terminal(2), Rule::repeat(Rule::terminal(4))]),
+                ]),
+            ),
+            Variable::named(
+                "rule1",
+                Rule::seq(vec![Rule::terminal(3), Rule::repeat(Rule::terminal(4))]),
+            ),
+        ]));
+
+        // Only one auxiliary rule is created for repeating terminal 4.
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "rule0",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::terminal(1), Rule::non_terminal(2)]),
+                        Rule::seq(vec![Rule::terminal(2), Rule::non_terminal(2)]),
+                    ])
+                ),
+                Variable::named(
+                    "rule1",
+                    Rule::seq(vec![Rule::terminal(3), Rule::non_terminal(2),])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
+                        Rule::terminal(4),
+                    ])
+                )
+            ]
+        );
+    }
+
+    #[test]
+    fn test_expansion_of_nested_repeats() {
+        let grammar = expand_repeats(build_grammar(vec![Variable::named(
+            "rule0",
+            Rule::seq(vec![
+                Rule::terminal(10),
+                Rule::repeat(Rule::seq(vec![
+                    Rule::terminal(11),
+                    Rule::repeat(Rule::terminal(12)),
+                ])),
+            ]),
+        )]));
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "rule0",
+                    Rule::seq(vec![Rule::terminal(10), Rule::non_terminal(2),])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
+                        Rule::terminal(12),
+                    ])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat2",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
+                        Rule::seq(vec![Rule::terminal(11), Rule::non_terminal(1),]),
+                    ])
+                ),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_expansion_of_repeats_at_top_of_hidden_rules() {
+        let grammar = expand_repeats(build_grammar(vec![
+            Variable::named("rule0", Rule::non_terminal(1)),
+            Variable::hidden(
+                "_rule1",
+                Rule::repeat(Rule::choice(vec![Rule::terminal(11), Rule::terminal(12)])),
+            ),
+        ]));
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named("rule0", Rule::non_terminal(1),),
+                Variable::auxiliary(
+                    "_rule1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1)]),
+                        Rule::terminal(11),
+                        Rule::terminal(12),
+                    ]),
+                ),
+            ]
+        );
+    }
+
+    fn build_grammar(variables: Vec<Variable>) -> ExtractedSyntaxGrammar {
+        ExtractedSyntaxGrammar {
+            variables,
+            ..Default::default()
+        }
+    }
+}

cli/generate/src/prepare_grammar/expand_tokens.rs

@@ -0,0 +1,940 @@
+use std::collections::HashMap;
+
+use anyhow::{anyhow, Context, Result};
+use lazy_static::lazy_static;
+use regex_syntax::ast::{
+    parse, Ast, ClassPerlKind, ClassSet, ClassSetBinaryOpKind, ClassSetItem, ClassUnicodeKind,
+    RepetitionKind, RepetitionRange,
+};
+
+use super::ExtractedLexicalGrammar;
+use crate::{
+    grammars::{LexicalGrammar, LexicalVariable},
+    nfa::{CharacterSet, Nfa, NfaState},
+    rules::{Precedence, Rule},
+};
+
+lazy_static! {
+    static ref UNICODE_CATEGORIES: HashMap<&'static str, Vec<u32>> =
+        serde_json::from_str(UNICODE_CATEGORIES_JSON).unwrap();
+    static ref UNICODE_PROPERTIES: HashMap<&'static str, Vec<u32>> =
+        serde_json::from_str(UNICODE_PROPERTIES_JSON).unwrap();
+    static ref UNICODE_CATEGORY_ALIASES: HashMap<&'static str, String> =
+        serde_json::from_str(UNICODE_CATEGORY_ALIASES_JSON).unwrap();
+    static ref UNICODE_PROPERTY_ALIASES: HashMap<&'static str, String> =
+        serde_json::from_str(UNICODE_PROPERTY_ALIASES_JSON).unwrap();
+}
+
+const UNICODE_CATEGORIES_JSON: &str = include_str!("./unicode-categories.json");
+const UNICODE_PROPERTIES_JSON: &str = include_str!("./unicode-properties.json");
+const UNICODE_CATEGORY_ALIASES_JSON: &str = include_str!("./unicode-category-aliases.json");
+const UNICODE_PROPERTY_ALIASES_JSON: &str = include_str!("./unicode-property-aliases.json");
+
+struct NfaBuilder {
+    nfa: Nfa,
+    is_sep: bool,
+    precedence_stack: Vec<i32>,
+}
+
+fn get_implicit_precedence(rule: &Rule) -> i32 {
+    match rule {
+        Rule::String(_) => 2,
+        Rule::Metadata { rule, params } => {
+            if params.is_main_token {
+                get_implicit_precedence(rule) + 1
+            } else {
+                get_implicit_precedence(rule)
+            }
+        }
+        _ => 0,
+    }
+}
+
+const fn get_completion_precedence(rule: &Rule) -> i32 {
+    if let Rule::Metadata { params, .. } = rule {
+        if let Precedence::Integer(p) = params.precedence {
+            return p;
+        }
+    }
+    0
+}
+
+pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> Result<LexicalGrammar> {
+    let mut builder = NfaBuilder {
+        nfa: Nfa::new(),
+        is_sep: true,
+        precedence_stack: vec![0],
+    };
+
+    let separator_rule = if grammar.separators.is_empty() {
+        Rule::Blank
+    } else {
+        grammar.separators.push(Rule::Blank);
+        Rule::repeat(Rule::choice(grammar.separators))
+    };
+
+    let mut variables = Vec::new();
+    for (i, variable) in grammar.variables.into_iter().enumerate() {
+        let is_immediate_token = match &variable.rule {
+            Rule::Metadata { params, .. } => params.is_main_token,
+            _ => false,
+        };
+
+        builder.is_sep = false;
+        builder.nfa.states.push(NfaState::Accept {
+            variable_index: i,
+            precedence: get_completion_precedence(&variable.rule),
+        });
+        let last_state_id = builder.nfa.last_state_id();
+        builder
+            .expand_rule(&variable.rule, last_state_id)
+            .with_context(|| format!("Error processing rule {}", variable.name))?;
+
+        if !is_immediate_token {
+            builder.is_sep = true;
+            let last_state_id = builder.nfa.last_state_id();
+            builder.expand_rule(&separator_rule, last_state_id)?;
+        }
+
+        variables.push(LexicalVariable {
+            name: variable.name,
+            kind: variable.kind,
+            implicit_precedence: get_implicit_precedence(&variable.rule),
+            start_state: builder.nfa.last_state_id(),
+        });
+    }
+
+    Ok(LexicalGrammar {
+        nfa: builder.nfa,
+        variables,
+    })
+}
+
+impl NfaBuilder {
+    fn expand_rule(&mut self, rule: &Rule, mut next_state_id: u32) -> Result<bool> {
+        match rule {
+            Rule::Pattern(s, f) => {
+                let ast = parse::Parser::new().parse(s)?;
+                self.expand_regex(&ast, next_state_id, f.contains('i'))
+            }
+            Rule::String(s) => {
+                for c in s.chars().rev() {
+                    self.push_advance(CharacterSet::empty().add_char(c), next_state_id);
+                    next_state_id = self.nfa.last_state_id();
+                }
+                Ok(!s.is_empty())
+            }
+            Rule::Choice(elements) => {
+                let mut alternative_state_ids = Vec::new();
+                for element in elements {
+                    if self.expand_rule(element, next_state_id)? {
+                        alternative_state_ids.push(self.nfa.last_state_id());
+                    } else {
+                        alternative_state_ids.push(next_state_id);
+                    }
+                }
+                alternative_state_ids.sort_unstable();
+                alternative_state_ids.dedup();
+                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
+                for alternative_state_id in alternative_state_ids {
+                    self.push_split(alternative_state_id);
+                }
+                Ok(true)
+            }
+            Rule::Seq(elements) => {
+                let mut result = false;
+                for element in elements.iter().rev() {
+                    if self.expand_rule(element, next_state_id)? {
+                        result = true;
+                    }
+                    next_state_id = self.nfa.last_state_id();
+                }
+                Ok(result)
+            }
+            Rule::Repeat(rule) => {
+                self.nfa.states.push(NfaState::Accept {
+                    variable_index: 0,
+                    precedence: 0,
+                }); // Placeholder for split
+                let split_state_id = self.nfa.last_state_id();
+                if self.expand_rule(rule, split_state_id)? {
+                    self.nfa.states[split_state_id as usize] =
+                        NfaState::Split(self.nfa.last_state_id(), next_state_id);
+                    Ok(true)
+                } else {
+                    Ok(false)
+                }
+            }
+            Rule::Metadata { rule, params } => {
+                let has_precedence = if let Precedence::Integer(precedence) = &params.precedence {
+                    self.precedence_stack.push(*precedence);
+                    true
+                } else {
+                    false
+                };
+                let result = self.expand_rule(rule, next_state_id);
+                if has_precedence {
+                    self.precedence_stack.pop();
+                }
+                result
+            }
+            Rule::Blank => Ok(false),
+            _ => Err(anyhow!("Grammar error: Unexpected rule {rule:?}")),
+        }
+    }
+
+    fn expand_regex(
+        &mut self,
+        ast: &Ast,
+        mut next_state_id: u32,
+        case_insensitive: bool,
+    ) -> Result<bool> {
+        const fn inverse_char(c: char) -> char {
+            match c {
+                'a'..='z' => (c as u8 - b'a' + b'A') as char,
+                'A'..='Z' => (c as u8 - b'A' + b'a') as char,
+                c => c,
+            }
+        }
+
+        fn with_inverse_char(mut chars: CharacterSet) -> CharacterSet {
+            for char in chars.clone().chars() {
+                let inverted = inverse_char(char);
+                if char != inverted {
+                    chars = chars.add_char(inverted);
+                }
+            }
+            chars
+        }
+
+        match ast {
+            Ast::Empty(_) => Ok(false),
+            Ast::Flags(_) => Err(anyhow!("Regex error: Flags are not supported")),
+            Ast::Literal(literal) => {
+                let mut char_set = CharacterSet::from_char(literal.c);
+                if case_insensitive {
+                    let inverted = inverse_char(literal.c);
+                    if literal.c != inverted {
+                        char_set = char_set.add_char(inverted);
+                    }
+                }
+                self.push_advance(char_set, next_state_id);
+                Ok(true)
+            }
+            Ast::Dot(_) => {
+                self.push_advance(CharacterSet::from_char('\n').negate(), next_state_id);
+                Ok(true)
+            }
+            Ast::Assertion(_) => Err(anyhow!("Regex error: Assertions are not supported")),
+            Ast::ClassUnicode(class) => {
+                let mut chars = self.expand_unicode_character_class(&class.kind)?;
+                if class.negated {
+                    chars = chars.negate();
+                }
+                if case_insensitive {
+                    chars = with_inverse_char(chars);
+                }
+                self.push_advance(chars, next_state_id);
+                Ok(true)
+            }
+            Ast::ClassPerl(class) => {
+                let mut chars = self.expand_perl_character_class(&class.kind);
+                if class.negated {
+                    chars = chars.negate();
+                }
+                if case_insensitive {
+                    chars = with_inverse_char(chars);
+                }
+                self.push_advance(chars, next_state_id);
+                Ok(true)
+            }
+            Ast::ClassBracketed(class) => {
+                let mut chars = self.translate_class_set(&class.kind)?;
+                if class.negated {
+                    chars = chars.negate();
+                }
+                if case_insensitive {
+                    chars = with_inverse_char(chars);
+                }
+                self.push_advance(chars, next_state_id);
+                Ok(true)
+            }
+            Ast::Repetition(repetition) => match repetition.op.kind {
+                RepetitionKind::ZeroOrOne => {
+                    self.expand_zero_or_one(&repetition.ast, next_state_id, case_insensitive)
+                }
+                RepetitionKind::OneOrMore => {
+                    self.expand_one_or_more(&repetition.ast, next_state_id, case_insensitive)
+                }
+                RepetitionKind::ZeroOrMore => {
+                    self.expand_zero_or_more(&repetition.ast, next_state_id, case_insensitive)
+                }
+                RepetitionKind::Range(RepetitionRange::Exactly(count)) => {
+                    self.expand_count(&repetition.ast, count, next_state_id, case_insensitive)
+                }
+                RepetitionKind::Range(RepetitionRange::AtLeast(min)) => {
+                    if self.expand_zero_or_more(&repetition.ast, next_state_id, case_insensitive)? {
+                        self.expand_count(&repetition.ast, min, next_state_id, case_insensitive)
+                    } else {
+                        Ok(false)
+                    }
+                }
+                RepetitionKind::Range(RepetitionRange::Bounded(min, max)) => {
+                    let mut result =
+                        self.expand_count(&repetition.ast, min, next_state_id, case_insensitive)?;
+                    for _ in min..max {
+                        if result {
+                            next_state_id = self.nfa.last_state_id();
+                        }
+                        if self.expand_zero_or_one(
+                            &repetition.ast,
+                            next_state_id,
+                            case_insensitive,
+                        )? {
+                            result = true;
+                        }
+                    }
+                    Ok(result)
+                }
+            },
+            Ast::Group(group) => self.expand_regex(&group.ast, next_state_id, case_insensitive),
+            Ast::Alternation(alternation) => {
+                let mut alternative_state_ids = Vec::new();
+                for ast in &alternation.asts {
+                    if self.expand_regex(ast, next_state_id, case_insensitive)? {
+                        alternative_state_ids.push(self.nfa.last_state_id());
+                    } else {
+                        alternative_state_ids.push(next_state_id);
+                    }
+                }
+                alternative_state_ids.sort_unstable();
+                alternative_state_ids.dedup();
+                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
+
+                for alternative_state_id in alternative_state_ids {
+                    self.push_split(alternative_state_id);
+                }
+                Ok(true)
+            }
+            Ast::Concat(concat) => {
+                let mut result = false;
+                for ast in concat.asts.iter().rev() {
+                    if self.expand_regex(ast, next_state_id, case_insensitive)? {
+                        result = true;
+                        next_state_id = self.nfa.last_state_id();
+                    }
+                }
+                Ok(result)
+            }
+        }
+    }
+
+    fn translate_class_set(&self, class_set: &ClassSet) -> Result<CharacterSet> {
+        match &class_set {
+            ClassSet::Item(item) => self.expand_character_class(item),
+            ClassSet::BinaryOp(binary_op) => {
+                let mut lhs_char_class = self.translate_class_set(&binary_op.lhs)?;
+                let mut rhs_char_class = self.translate_class_set(&binary_op.rhs)?;
+                match binary_op.kind {
+                    ClassSetBinaryOpKind::Intersection => {
+                        Ok(lhs_char_class.remove_intersection(&mut rhs_char_class))
+                    }
+                    ClassSetBinaryOpKind::Difference => {
+                        Ok(lhs_char_class.difference(rhs_char_class))
+                    }
+                    ClassSetBinaryOpKind::SymmetricDifference => {
+                        Ok(lhs_char_class.symmetric_difference(rhs_char_class))
+                    }
+                }
+            }
+        }
+    }
+
+    fn expand_one_or_more(
+        &mut self,
+        ast: &Ast,
+        next_state_id: u32,
+        case_insensitive: bool,
+    ) -> Result<bool> {
+        self.nfa.states.push(NfaState::Accept {
+            variable_index: 0,
+            precedence: 0,
+        }); // Placeholder for split
+        let split_state_id = self.nfa.last_state_id();
+        if self.expand_regex(ast, split_state_id, case_insensitive)? {
+            self.nfa.states[split_state_id as usize] =
+                NfaState::Split(self.nfa.last_state_id(), next_state_id);
+            Ok(true)
+        } else {
+            self.nfa.states.pop();
+            Ok(false)
+        }
+    }
+
+    fn expand_zero_or_one(
+        &mut self,
+        ast: &Ast,
+        next_state_id: u32,
+        case_insensitive: bool,
+    ) -> Result<bool> {
+        if self.expand_regex(ast, next_state_id, case_insensitive)? {
+            self.push_split(next_state_id);
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    fn expand_zero_or_more(
+        &mut self,
+        ast: &Ast,
+        next_state_id: u32,
+        case_insensitive: bool,
+    ) -> Result<bool> {
+        if self.expand_one_or_more(ast, next_state_id, case_insensitive)? {
+            self.push_split(next_state_id);
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    fn expand_count(
+        &mut self,
+        ast: &Ast,
+        count: u32,
+        mut next_state_id: u32,
+        case_insensitive: bool,
+    ) -> Result<bool> {
+        let mut result = false;
+        for _ in 0..count {
+            if self.expand_regex(ast, next_state_id, case_insensitive)? {
+                result = true;
+                next_state_id = self.nfa.last_state_id();
+            }
+        }
+        Ok(result)
+    }
+
+    fn expand_character_class(&self, item: &ClassSetItem) -> Result<CharacterSet> {
+        match item {
+            ClassSetItem::Empty(_) => Ok(CharacterSet::empty()),
+            ClassSetItem::Literal(literal) => Ok(CharacterSet::from_char(literal.c)),
+            ClassSetItem::Range(range) => Ok(CharacterSet::from_range(range.start.c, range.end.c)),
+            ClassSetItem::Union(union) => {
+                let mut result = CharacterSet::empty();
+                for item in &union.items {
+                    result = result.add(&self.expand_character_class(item)?);
+                }
+                Ok(result)
+            }
+            ClassSetItem::Perl(class) => Ok(self.expand_perl_character_class(&class.kind)),
+            ClassSetItem::Unicode(class) => {
+                let mut set = self.expand_unicode_character_class(&class.kind)?;
+                if class.negated {
+                    set = set.negate();
+                }
+                Ok(set)
+            }
+            ClassSetItem::Bracketed(class) => {
+                let mut set = self.translate_class_set(&class.kind)?;
+                if class.negated {
+                    set = set.negate();
+                }
+                Ok(set)
+            }
+            ClassSetItem::Ascii(_) => Err(anyhow!(
+                "Regex error: Unsupported character class syntax {item:?}",
+            )),
+        }
+    }
+
+    fn expand_unicode_character_class(&self, class: &ClassUnicodeKind) -> Result<CharacterSet> {
+        let mut chars = CharacterSet::empty();
+
+        let category_letter;
+        match class {
+            ClassUnicodeKind::OneLetter(le) => {
+                category_letter = le.to_string();
+            }
+            ClassUnicodeKind::Named(class_name) => {
+                let actual_class_name = UNICODE_CATEGORY_ALIASES
+                    .get(class_name.as_str())
+                    .or_else(|| UNICODE_PROPERTY_ALIASES.get(class_name.as_str()))
+                    .unwrap_or(class_name);
+                if actual_class_name.len() == 1 {
+                    category_letter = actual_class_name.clone();
+                } else {
+                    let code_points =
+                        UNICODE_CATEGORIES
+                            .get(actual_class_name.as_str())
+                            .or_else(|| UNICODE_PROPERTIES.get(actual_class_name.as_str()))
+                            .ok_or_else(|| {
+                                anyhow!(
+                                    "Regex error: Unsupported unicode character class {class_name}",
+                                )
+                            })?;
+                    for c in code_points {
+                        if let Some(c) = char::from_u32(*c) {
+                            chars = chars.add_char(c);
+                        }
+                    }
+
+                    return Ok(chars);
+                }
+            }
+            ClassUnicodeKind::NamedValue { .. } => {
+                return Err(anyhow!(
+                    "Regex error: Key-value unicode properties are not supported"
+                ))
+            }
+        }
+
+        for (category, code_points) in UNICODE_CATEGORIES.iter() {
+            if category.starts_with(&category_letter) {
+                for c in code_points {
+                    if let Some(c) = char::from_u32(*c) {
+                        chars = chars.add_char(c);
+                    }
+                }
+            }
+        }
+
+        Ok(chars)
+    }
+
+    fn expand_perl_character_class(&self, item: &ClassPerlKind) -> CharacterSet {
+        match item {
+            ClassPerlKind::Digit => CharacterSet::from_range('0', '9'),
+            ClassPerlKind::Space => CharacterSet::empty()
+                .add_char(' ')
+                .add_char('\t')
+                .add_char('\r')
+                .add_char('\n')
+                .add_char('\x0B')
+                .add_char('\x0C'),
+            ClassPerlKind::Word => CharacterSet::empty()
+                .add_char('_')
+                .add_range('A', 'Z')
+                .add_range('a', 'z')
+                .add_range('0', '9'),
+        }
+    }
+
+    fn push_advance(&mut self, chars: CharacterSet, state_id: u32) {
+        let precedence = *self.precedence_stack.last().unwrap();
+        self.nfa.states.push(NfaState::Advance {
+            chars,
+            state_id,
+            precedence,
+            is_sep: self.is_sep,
+        });
+    }
+
+    fn push_split(&mut self, state_id: u32) {
+        let last_state_id = self.nfa.last_state_id();
+        self.nfa
+            .states
+            .push(NfaState::Split(state_id, last_state_id));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::Variable,
+        nfa::{NfaCursor, NfaTransition},
+    };
+
+    fn simulate_nfa<'a>(grammar: &'a LexicalGrammar, s: &'a str) -> Option<(usize, &'a str)> {
+        let start_states = grammar.variables.iter().map(|v| v.start_state).collect();
+        let mut cursor = NfaCursor::new(&grammar.nfa, start_states);
+
+        let mut result = None;
+        let mut result_precedence = i32::MIN;
+        let mut start_char = 0;
+        let mut end_char = 0;
+        for c in s.chars() {
+            for (id, precedence) in cursor.completions() {
+                if result.is_none() || result_precedence <= precedence {
+                    result = Some((id, &s[start_char..end_char]));
+                    result_precedence = precedence;
+                }
+            }
+            if let Some(NfaTransition {
+                states,
+                is_separator,
+                ..
+            }) = cursor
+                .transitions()
+                .into_iter()
+                .find(|t| t.characters.contains(c) && t.precedence >= result_precedence)
+            {
+                cursor.reset(states);
+                end_char += c.len_utf8();
+                if is_separator {
+                    start_char = end_char;
+                }
+            } else {
+                break;
+            }
+        }
+
+        for (id, precedence) in cursor.completions() {
+            if result.is_none() || result_precedence <= precedence {
+                result = Some((id, &s[start_char..end_char]));
+                result_precedence = precedence;
+            }
+        }
+
+        result
+    }
+
+    #[test]
+    fn test_rule_expansion() {
+        struct Row {
+            rules: Vec<Rule>,
+            separators: Vec<Rule>,
+            examples: Vec<(&'static str, Option<(usize, &'static str)>)>,
+        }
+
+        let table = [
+            // regex with sequences and alternatives
+            Row {
+                rules: vec![Rule::pattern("(a|b|c)d(e|f|g)h?", "")],
+                separators: vec![],
+                examples: vec![
+                    ("ade1", Some((0, "ade"))),
+                    ("bdf1", Some((0, "bdf"))),
+                    ("bdfh1", Some((0, "bdfh"))),
+                    ("ad1", None),
+                ],
+            },
+            // regex with repeats
+            Row {
+                rules: vec![Rule::pattern("a*", "")],
+                separators: vec![],
+                examples: vec![("aaa1", Some((0, "aaa"))), ("b", Some((0, "")))],
+            },
+            // regex with repeats in sequences
+            Row {
+                rules: vec![Rule::pattern("a((bc)+|(de)*)f", "")],
+                separators: vec![],
+                examples: vec![
+                    ("af1", Some((0, "af"))),
+                    ("adedef1", Some((0, "adedef"))),
+                    ("abcbcbcf1", Some((0, "abcbcbcf"))),
+                    ("a", None),
+                ],
+            },
+            // regex with character ranges
+            Row {
+                rules: vec![Rule::pattern("[a-fA-F0-9]+", "")],
+                separators: vec![],
+                examples: vec![("A1ff0.", Some((0, "A1ff0")))],
+            },
+            // regex with perl character classes
+            Row {
+                rules: vec![Rule::pattern("\\w\\d\\s", "")],
+                separators: vec![],
+                examples: vec![("_0  ", Some((0, "_0 ")))],
+            },
+            // string
+            Row {
+                rules: vec![Rule::string("abc")],
+                separators: vec![],
+                examples: vec![("abcd", Some((0, "abc"))), ("ab", None)],
+            },
+            // complex rule containing strings and regexes
+            Row {
+                rules: vec![Rule::repeat(Rule::seq(vec![
+                    Rule::string("{"),
+                    Rule::pattern("[a-f]+", ""),
+                    Rule::string("}"),
+                ]))],
+                separators: vec![],
+                examples: vec![
+                    ("{a}{", Some((0, "{a}"))),
+                    ("{a}{d", Some((0, "{a}"))),
+                    ("ab", None),
+                ],
+            },
+            // longest match rule
+            Row {
+                rules: vec![
+                    Rule::pattern("a|bc", ""),
+                    Rule::pattern("aa", ""),
+                    Rule::pattern("bcd", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("a.", Some((0, "a"))),
+                    ("bc.", Some((0, "bc"))),
+                    ("aa.", Some((1, "aa"))),
+                    ("bcd?", Some((2, "bcd"))),
+                    ("b.", None),
+                    ("c.", None),
+                ],
+            },
+            // regex with an alternative including the empty string
+            Row {
+                rules: vec![Rule::pattern("a(b|)+c", "")],
+                separators: vec![],
+                examples: vec![
+                    ("ac.", Some((0, "ac"))),
+                    ("abc.", Some((0, "abc"))),
+                    ("abbc.", Some((0, "abbc"))),
+                ],
+            },
+            // separators
+            Row {
+                rules: vec![Rule::pattern("[a-f]+", "")],
+                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
+                examples: vec![
+                    ("  a", Some((0, "a"))),
+                    ("  \nb", Some((0, "b"))),
+                    ("  \\a", None),
+                    ("  \\\na", Some((0, "a"))),
+                ],
+            },
+            // shorter tokens with higher precedence
+            Row {
+                rules: vec![
+                    Rule::prec(Precedence::Integer(2), Rule::pattern("abc", "")),
+                    Rule::prec(Precedence::Integer(1), Rule::pattern("ab[cd]e", "")),
+                    Rule::pattern("[a-e]+", ""),
+                ],
+                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
+                examples: vec![
+                    ("abceef", Some((0, "abc"))),
+                    ("abdeef", Some((1, "abde"))),
+                    ("aeeeef", Some((2, "aeeee"))),
+                ],
+            },
+            // immediate tokens with higher precedence
+            Row {
+                rules: vec![
+                    Rule::prec(Precedence::Integer(1), Rule::pattern("[^a]+", "")),
+                    Rule::immediate_token(Rule::prec(
+                        Precedence::Integer(2),
+                        Rule::pattern("[^ab]+", ""),
+                    )),
+                ],
+                separators: vec![Rule::pattern("\\s", "")],
+                examples: vec![("cccb", Some((1, "ccc")))],
+            },
+            Row {
+                rules: vec![Rule::seq(vec![
+                    Rule::string("a"),
+                    Rule::choice(vec![Rule::string("b"), Rule::string("c")]),
+                    Rule::string("d"),
+                ])],
+                separators: vec![],
+                examples: vec![
+                    ("abd", Some((0, "abd"))),
+                    ("acd", Some((0, "acd"))),
+                    ("abc", None),
+                    ("ad", None),
+                    ("d", None),
+                    ("a", None),
+                ],
+            },
+            // nested choices within sequences
+            Row {
+                rules: vec![Rule::seq(vec![
+                    Rule::pattern("[0-9]+", ""),
+                    Rule::choice(vec![
+                        Rule::Blank,
+                        Rule::choice(vec![Rule::seq(vec![
+                            Rule::choice(vec![Rule::string("e"), Rule::string("E")]),
+                            Rule::choice(vec![
+                                Rule::Blank,
+                                Rule::choice(vec![Rule::string("+"), Rule::string("-")]),
+                            ]),
+                            Rule::pattern("[0-9]+", ""),
+                        ])]),
+                    ]),
+                ])],
+                separators: vec![],
+                examples: vec![
+                    ("12", Some((0, "12"))),
+                    ("12e", Some((0, "12"))),
+                    ("12g", Some((0, "12"))),
+                    ("12e3", Some((0, "12e3"))),
+                    ("12e+", Some((0, "12"))),
+                    ("12E+34 +", Some((0, "12E+34"))),
+                    ("12e34", Some((0, "12e34"))),
+                ],
+            },
+            // nested groups
+            Row {
+                rules: vec![Rule::seq(vec![Rule::pattern(r"([^x\\]|\\(.|\n))+", "")])],
+                separators: vec![],
+                examples: vec![("abcx", Some((0, "abc"))), ("abc\\0x", Some((0, "abc\\0")))],
+            },
+            // allowing unrecognized escape sequences
+            Row {
+                rules: vec![
+                    // Escaped forward slash (used in JS because '/' is the regex delimiter)
+                    Rule::pattern(r"\/", ""),
+                    // Escaped quotes
+                    Rule::pattern(r#"\"\'"#, ""),
+                    // Quote preceded by a literal backslash
+                    Rule::pattern(r"[\\']+", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("/", Some((0, "/"))),
+                    ("\"\'", Some((1, "\"\'"))),
+                    (r"'\'a", Some((2, r"'\'"))),
+                ],
+            },
+            // unicode property escapes
+            Row {
+                rules: vec![
+                    Rule::pattern(r"\p{L}+\P{L}+", ""),
+                    Rule::pattern(r"\p{White_Space}+\P{White_Space}+[\p{White_Space}]*", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("  123   abc", Some((1, "  123   "))),
+                    ("ბΨƁ___ƀƔ", Some((0, "ბΨƁ___"))),
+                ],
+            },
+            // unicode property escapes in bracketed sets
+            Row {
+                rules: vec![Rule::pattern(r"[\p{L}\p{Nd}]+", "")],
+                separators: vec![],
+                examples: vec![("abΨ12٣٣, ok", Some((0, "abΨ12٣٣")))],
+            },
+            // unicode character escapes
+            Row {
+                rules: vec![
+                    Rule::pattern(r"\u{00dc}", ""),
+                    Rule::pattern(r"\U{000000dd}", ""),
+                    Rule::pattern(r"\u00de", ""),
+                    Rule::pattern(r"\U000000df", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("\u{00dc}", Some((0, "\u{00dc}"))),
+                    ("\u{00dd}", Some((1, "\u{00dd}"))),
+                    ("\u{00de}", Some((2, "\u{00de}"))),
+                    ("\u{00df}", Some((3, "\u{00df}"))),
+                ],
+            },
+            Row {
+                rules: vec![
+                    Rule::pattern(r"u\{[0-9a-fA-F]+\}", ""),
+                    // Already-escaped curly braces
+                    Rule::pattern(r"\{[ab]{3}\}", ""),
+                    // Unicode codepoints
+                    Rule::pattern(r"\u{1000A}", ""),
+                    // Unicode codepoints (lowercase)
+                    Rule::pattern(r"\u{1000b}", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("u{1234} ok", Some((0, "u{1234}"))),
+                    ("{aba}}", Some((1, "{aba}"))),
+                    ("\u{1000A}", Some((2, "\u{1000A}"))),
+                    ("\u{1000b}", Some((3, "\u{1000b}"))),
+                ],
+            },
+            // Emojis
+            Row {
+                rules: vec![Rule::pattern(r"\p{Emoji}+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("🐎", Some((0, "🐎"))),
+                    ("🐴🐴", Some((0, "🐴🐴"))),
+                    ("#0", Some((0, "#0"))), // These chars are technically emojis!
+                    ("⻢", None),
+                    ("♞", None),
+                    ("horse", None),
+                ],
+            },
+            // Intersection
+            Row {
+                rules: vec![Rule::pattern(r"[[0-7]&&[4-9]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("456", Some((0, "456"))),
+                    ("64", Some((0, "64"))),
+                    ("452", Some((0, "45"))),
+                    ("91", None),
+                    ("8", None),
+                    ("3", None),
+                ],
+            },
+            // Difference
+            Row {
+                rules: vec![Rule::pattern(r"[[0-9]--[4-7]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("123", Some((0, "123"))),
+                    ("83", Some((0, "83"))),
+                    ("9", Some((0, "9"))),
+                    ("124", Some((0, "12"))),
+                    ("67", None),
+                    ("4", None),
+                ],
+            },
+            // Symmetric difference
+            Row {
+                rules: vec![Rule::pattern(r"[[0-7]~~[4-9]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("123", Some((0, "123"))),
+                    ("83", Some((0, "83"))),
+                    ("9", Some((0, "9"))),
+                    ("124", Some((0, "12"))),
+                    ("67", None),
+                    ("4", None),
+                ],
+            },
+            // Nested set operations
+            Row {
+                //               0 1 2 3 4 5 6 7 8 9
+                // [0-5]:        y y y y y y
+                // [2-4]:            y y y
+                // [0-5]--[2-4]: y y       y
+                // [3-9]:              y y y y y y y
+                // [6-7]:                    y y
+                // [3-9]--[5-7]:       y y y     y y
+                // final regex:  y y   y y       y y
+                rules: vec![Rule::pattern(r"[[[0-5]--[2-4]]~~[[3-9]--[6-7]]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("01", Some((0, "01"))),
+                    ("432", Some((0, "43"))),
+                    ("8", Some((0, "8"))),
+                    ("9", Some((0, "9"))),
+                    ("2", None),
+                    ("567", None),
+                ],
+            },
+        ];
+
+        for Row {
+            rules,
+            separators,
+            examples,
+        } in &table
+        {
+            let grammar = expand_tokens(ExtractedLexicalGrammar {
+                separators: separators.clone(),
+                variables: rules
+                    .iter()
+                    .map(|rule| Variable::named("", rule.clone()))
+                    .collect(),
+            })
+            .unwrap();
+
+            for (haystack, needle) in examples {
+                assert_eq!(simulate_nfa(&grammar, haystack), *needle);
+            }
+        }
+    }
+}

cli/generate/src/prepare_grammar/extract_default_aliases.rs

@@ -0,0 +1,304 @@
+use crate::{
+    grammars::{LexicalGrammar, SyntaxGrammar},
+    rules::{Alias, AliasMap, Symbol, SymbolType},
+};
+
+#[derive(Clone, Default)]
+struct SymbolStatus {
+    aliases: Vec<(Alias, usize)>,
+    appears_unaliased: bool,
+}
+
+// Update the grammar by finding symbols that always are aliased, and for each such symbol,
+// promoting one of its aliases to a "default alias", which is applied globally instead
+// of in a context-specific way.
+//
+// This has two benefits:
+// * It reduces the overhead of storing production-specific alias info in the parse table.
+// * Within an `ERROR` node, no context-specific aliases will be applied. This transformation
+//   ensures that the children of an `ERROR` node have symbols that are consistent with the way that
+//   they would appear in a valid syntax tree.
+pub(super) fn extract_default_aliases(
+    syntax_grammar: &mut SyntaxGrammar,
+    lexical_grammar: &LexicalGrammar,
+) -> AliasMap {
+    let mut terminal_status_list = vec![SymbolStatus::default(); lexical_grammar.variables.len()];
+    let mut non_terminal_status_list =
+        vec![SymbolStatus::default(); syntax_grammar.variables.len()];
+    let mut external_status_list =
+        vec![SymbolStatus::default(); syntax_grammar.external_tokens.len()];
+
+    // For each grammar symbol, find all of the aliases under which the symbol appears,
+    // and determine whether or not the symbol ever appears *unaliased*.
+    for variable in &syntax_grammar.variables {
+        for production in &variable.productions {
+            for step in &production.steps {
+                let status = match step.symbol.kind {
+                    SymbolType::External => &mut external_status_list[step.symbol.index],
+                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
+                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
+                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                        panic!("Unexpected end token")
+                    }
+                };
+
+                // Default aliases don't work for inlined variables.
+                if syntax_grammar.variables_to_inline.contains(&step.symbol) {
+                    continue;
+                }
+
+                if let Some(alias) = &step.alias {
+                    if let Some(count_for_alias) = status
+                        .aliases
+                        .iter_mut()
+                        .find_map(|(a, count)| if a == alias { Some(count) } else { None })
+                    {
+                        *count_for_alias += 1;
+                    } else {
+                        status.aliases.push((alias.clone(), 1));
+                    }
+                } else {
+                    status.appears_unaliased = true;
+                }
+            }
+        }
+    }
+
+    for symbol in &syntax_grammar.extra_symbols {
+        let status = match symbol.kind {
+            SymbolType::External => &mut external_status_list[symbol.index],
+            SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
+            SymbolType::Terminal => &mut terminal_status_list[symbol.index],
+            SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                panic!("Unexpected end token")
+            }
+        };
+        status.appears_unaliased = true;
+    }
+
+    let symbols_with_statuses = (terminal_status_list
+        .iter_mut()
+        .enumerate()
+        .map(|(i, status)| (Symbol::terminal(i), status)))
+    .chain(
+        non_terminal_status_list
+            .iter_mut()
+            .enumerate()
+            .map(|(i, status)| (Symbol::non_terminal(i), status)),
+    )
+    .chain(
+        external_status_list
+            .iter_mut()
+            .enumerate()
+            .map(|(i, status)| (Symbol::external(i), status)),
+    );
+
+    // For each symbol that always appears aliased, find the alias the occurs most often,
+    // and designate that alias as the symbol's "default alias". Store all of these
+    // default aliases in a map that will be returned.
+    let mut result = AliasMap::new();
+    for (symbol, status) in symbols_with_statuses {
+        if status.appears_unaliased {
+            status.aliases.clear();
+        } else if let Some(default_entry) = status
+            .aliases
+            .iter()
+            .enumerate()
+            .max_by_key(|(i, (_, count))| (count, -(*i as i64)))
+            .map(|(_, entry)| entry.clone())
+        {
+            status.aliases.clear();
+            status.aliases.push(default_entry.clone());
+            result.insert(symbol, default_entry.0);
+        }
+    }
+
+    // Wherever a symbol is aliased as its default alias, remove the usage of the alias,
+    // because it will now be redundant.
+    let mut alias_positions_to_clear = Vec::new();
+    for variable in &mut syntax_grammar.variables {
+        alias_positions_to_clear.clear();
+
+        for (i, production) in variable.productions.iter().enumerate() {
+            for (j, step) in production.steps.iter().enumerate() {
+                let status = match step.symbol.kind {
+                    SymbolType::External => &mut external_status_list[step.symbol.index],
+                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
+                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
+                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                        panic!("Unexpected end token")
+                    }
+                };
+
+                // If this step is aliased as the symbol's default alias, then remove that alias.
+                if step.alias.is_some()
+                    && step.alias.as_ref() == status.aliases.first().map(|t| &t.0)
+                {
+                    let mut other_productions_must_use_this_alias_at_this_index = false;
+                    for (other_i, other_production) in variable.productions.iter().enumerate() {
+                        if other_i != i
+                            && other_production.steps.len() > j
+                            && other_production.steps[j].alias == step.alias
+                            && result.get(&other_production.steps[j].symbol) != step.alias.as_ref()
+                        {
+                            other_productions_must_use_this_alias_at_this_index = true;
+                            break;
+                        }
+                    }
+
+                    if !other_productions_must_use_this_alias_at_this_index {
+                        alias_positions_to_clear.push((i, j));
+                    }
+                }
+            }
+        }
+
+        for (production_index, step_index) in &alias_positions_to_clear {
+            variable.productions[*production_index].steps[*step_index].alias = None;
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType},
+        nfa::Nfa,
+    };
+
+    #[test]
+    fn test_extract_simple_aliases() {
+        let mut syntax_grammar = SyntaxGrammar {
+            variables: vec![
+                SyntaxVariable {
+                    name: "v1".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
+                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
+                            ProductionStep::new(Symbol::terminal(2)).with_alias("a3", true),
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "v2".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            // Token 0 is always aliased as "a1".
+                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
+                            // Token 1 is aliased within rule `v1` above, but not here.
+                            ProductionStep::new(Symbol::terminal(1)),
+                            // Token 2 is aliased differently here than in `v1`. The alias from
+                            // `v1` should be promoted to the default alias, because `v1` appears
+                            // first in the grammar.
+                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
+                            // Token 3 is also aliased differently here than in `v1`. In this case,
+                            // this alias should be promoted to the default alias, because it is
+                            // used a greater number of times (twice).
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
+                        ],
+                    }],
+                },
+            ],
+            ..Default::default()
+        };
+
+        let lexical_grammar = LexicalGrammar {
+            nfa: Nfa::new(),
+            variables: vec![
+                LexicalVariable {
+                    name: "t0".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+                LexicalVariable {
+                    name: "t1".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+                LexicalVariable {
+                    name: "t2".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+                LexicalVariable {
+                    name: "t3".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+            ],
+        };
+
+        let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
+        assert_eq!(default_aliases.len(), 3);
+
+        assert_eq!(
+            default_aliases.get(&Symbol::terminal(0)),
+            Some(&Alias {
+                value: "a1".to_string(),
+                is_named: true,
+            })
+        );
+        assert_eq!(
+            default_aliases.get(&Symbol::terminal(2)),
+            Some(&Alias {
+                value: "a3".to_string(),
+                is_named: true,
+            })
+        );
+        assert_eq!(
+            default_aliases.get(&Symbol::terminal(3)),
+            Some(&Alias {
+                value: "a6".to_string(),
+                is_named: true,
+            })
+        );
+        assert_eq!(default_aliases.get(&Symbol::terminal(1)), None);
+
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![
+                SyntaxVariable {
+                    name: "v1".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(0)),
+                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
+                            ProductionStep::new(Symbol::terminal(2)),
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
+                        ],
+                    },],
+                },
+                SyntaxVariable {
+                    name: "v2".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(0)),
+                            ProductionStep::new(Symbol::terminal(1)),
+                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
+                            ProductionStep::new(Symbol::terminal(3)),
+                            ProductionStep::new(Symbol::terminal(3)),
+                        ],
+                    },],
+                },
+            ]
+        );
+    }
+}

cli/generate/src/prepare_grammar/extract_tokens.rs

@@ -0,0 +1,554 @@
+use std::{collections::HashMap, mem};
+
+use anyhow::{anyhow, Result};
+
+use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar};
+use crate::{
+    grammars::{ExternalToken, Variable, VariableType},
+    rules::{MetadataParams, Rule, Symbol, SymbolType},
+};
+
+pub(super) fn extract_tokens(
+    mut grammar: InternedGrammar,
+) -> Result<(ExtractedSyntaxGrammar, ExtractedLexicalGrammar)> {
+    let mut extractor = TokenExtractor {
+        current_variable_name: String::new(),
+        current_variable_token_count: 0,
+        is_first_rule: false,
+        extracted_variables: Vec::new(),
+        extracted_usage_counts: Vec::new(),
+    };
+
+    for (i, variable) in &mut grammar.variables.iter_mut().enumerate() {
+        extractor.extract_tokens_in_variable(i == 0, variable)?;
+    }
+
+    for variable in &mut grammar.external_tokens {
+        extractor.extract_tokens_in_variable(false, variable)?;
+    }
+
+    let mut lexical_variables = Vec::with_capacity(extractor.extracted_variables.len());
+    for variable in extractor.extracted_variables {
+        lexical_variables.push(variable);
+    }
+
+    // If a variable's entire rule was extracted as a token and that token didn't
+    // appear within any other rule, then remove that variable from the syntax
+    // grammar, giving its name to the token in the lexical grammar. Any symbols
+    // that pointed to that variable will need to be updated to point to the
+    // variable in the lexical grammar. Symbols that pointed to later variables
+    // will need to have their indices decremented.
+    let mut variables = Vec::new();
+    let mut symbol_replacer = SymbolReplacer {
+        replacements: HashMap::new(),
+    };
+    for (i, variable) in grammar.variables.into_iter().enumerate() {
+        if let Rule::Symbol(Symbol {
+            kind: SymbolType::Terminal,
+            index,
+        }) = variable.rule
+        {
+            if i > 0 && extractor.extracted_usage_counts[index] == 1 {
+                let lexical_variable = &mut lexical_variables[index];
+                if lexical_variable.kind == VariableType::Auxiliary
+                    || variable.kind != VariableType::Hidden
+                {
+                    lexical_variable.kind = variable.kind;
+                    lexical_variable.name = variable.name;
+                    symbol_replacer.replacements.insert(i, index);
+                    continue;
+                }
+            }
+        }
+        variables.push(variable);
+    }
+
+    for variable in &mut variables {
+        variable.rule = symbol_replacer.replace_symbols_in_rule(&variable.rule);
+    }
+
+    let expected_conflicts = grammar
+        .expected_conflicts
+        .into_iter()
+        .map(|conflict| {
+            let mut result = conflict
+                .iter()
+                .map(|symbol| symbol_replacer.replace_symbol(*symbol))
+                .collect::<Vec<_>>();
+            result.sort_unstable();
+            result.dedup();
+            result
+        })
+        .collect();
+
+    let supertype_symbols = grammar
+        .supertype_symbols
+        .into_iter()
+        .map(|symbol| symbol_replacer.replace_symbol(symbol))
+        .collect();
+
+    let variables_to_inline = grammar
+        .variables_to_inline
+        .into_iter()
+        .map(|symbol| symbol_replacer.replace_symbol(symbol))
+        .collect();
+
+    let mut separators = Vec::new();
+    let mut extra_symbols = Vec::new();
+    for rule in grammar.extra_symbols {
+        if let Rule::Symbol(symbol) = rule {
+            extra_symbols.push(symbol_replacer.replace_symbol(symbol));
+        } else if let Some(index) = lexical_variables.iter().position(|v| v.rule == rule) {
+            extra_symbols.push(Symbol::terminal(index));
+        } else {
+            separators.push(rule);
+        }
+    }
+
+    let mut external_tokens = Vec::new();
+    for external_token in grammar.external_tokens {
+        let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule);
+        if let Rule::Symbol(symbol) = rule {
+            if symbol.is_non_terminal() {
+                return Err(anyhow!(
+                    "Rule '{}' cannot be used as both an external token and a non-terminal rule",
+                    &variables[symbol.index].name,
+                ));
+            }
+
+            if symbol.is_external() {
+                external_tokens.push(ExternalToken {
+                    name: external_token.name,
+                    kind: external_token.kind,
+                    corresponding_internal_token: None,
+                });
+            } else {
+                external_tokens.push(ExternalToken {
+                    name: lexical_variables[symbol.index].name.clone(),
+                    kind: external_token.kind,
+                    corresponding_internal_token: Some(symbol),
+                });
+            }
+        } else {
+            return Err(anyhow!(
+                "Non-symbol rules cannot be used as external tokens"
+            ));
+        }
+    }
+
+    let mut word_token = None;
+    if let Some(token) = grammar.word_token {
+        let token = symbol_replacer.replace_symbol(token);
+        if token.is_non_terminal() {
+            return Err(anyhow!(
+                "Non-terminal symbol '{}' cannot be used as the word token",
+                &variables[token.index].name
+            ));
+        }
+        word_token = Some(token);
+    }
+
+    Ok((
+        ExtractedSyntaxGrammar {
+            variables,
+            expected_conflicts,
+            extra_symbols,
+            variables_to_inline,
+            supertype_symbols,
+            external_tokens,
+            word_token,
+            precedence_orderings: grammar.precedence_orderings,
+        },
+        ExtractedLexicalGrammar {
+            variables: lexical_variables,
+            separators,
+        },
+    ))
+}
+
+struct TokenExtractor {
+    current_variable_name: String,
+    current_variable_token_count: usize,
+    is_first_rule: bool,
+    extracted_variables: Vec<Variable>,
+    extracted_usage_counts: Vec<usize>,
+}
+
+struct SymbolReplacer {
+    replacements: HashMap<usize, usize>,
+}
+
+impl TokenExtractor {
+    fn extract_tokens_in_variable(
+        &mut self,
+        is_first: bool,
+        variable: &mut Variable,
+    ) -> Result<()> {
+        self.current_variable_name.clear();
+        self.current_variable_name.push_str(&variable.name);
+        self.current_variable_token_count = 0;
+        self.is_first_rule = is_first;
+        let mut rule = Rule::Blank;
+        mem::swap(&mut rule, &mut variable.rule);
+        variable.rule = self.extract_tokens_in_rule(&rule)?;
+        Ok(())
+    }
+
+    fn extract_tokens_in_rule(&mut self, input: &Rule) -> Result<Rule> {
+        match input {
+            Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()),
+            Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()),
+            Rule::Metadata { params, rule } => {
+                if params.is_token {
+                    let mut params = params.clone();
+                    params.is_token = false;
+
+                    let mut string_value = None;
+                    if let Rule::String(value) = rule.as_ref() {
+                        string_value = Some(value);
+                    }
+
+                    let rule_to_extract = if params == MetadataParams::default() {
+                        rule.as_ref()
+                    } else {
+                        input
+                    };
+
+                    Ok(self.extract_token(rule_to_extract, string_value)?.into())
+                } else {
+                    Ok(Rule::Metadata {
+                        params: params.clone(),
+                        rule: Box::new(self.extract_tokens_in_rule(rule)?),
+                    })
+                }
+            }
+            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(
+                self.extract_tokens_in_rule(content)?,
+            ))),
+            Rule::Seq(elements) => Ok(Rule::Seq(
+                elements
+                    .iter()
+                    .map(|e| self.extract_tokens_in_rule(e))
+                    .collect::<Result<Vec<_>>>()?,
+            )),
+            Rule::Choice(elements) => Ok(Rule::Choice(
+                elements
+                    .iter()
+                    .map(|e| self.extract_tokens_in_rule(e))
+                    .collect::<Result<Vec<_>>>()?,
+            )),
+            _ => Ok(input.clone()),
+        }
+    }
+
+    fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Result<Symbol> {
+        for (i, variable) in self.extracted_variables.iter_mut().enumerate() {
+            if variable.rule == *rule {
+                self.extracted_usage_counts[i] += 1;
+                return Ok(Symbol::terminal(i));
+            }
+        }
+
+        let index = self.extracted_variables.len();
+        let variable = if let Some(string_value) = string_value {
+            if string_value.is_empty() && !self.is_first_rule {
+                return Err(anyhow!(
+                    "The rule `{}` contains an empty string.
+
+Tree-sitter does not support syntactic rules that contain an empty string
+unless they are used only as the grammar's start rule.
+",
+                    self.current_variable_name
+                ));
+            }
+            Variable {
+                name: string_value.clone(),
+                kind: VariableType::Anonymous,
+                rule: rule.clone(),
+            }
+        } else {
+            self.current_variable_token_count += 1;
+            Variable {
+                name: format!(
+                    "{}_token{}",
+                    &self.current_variable_name, self.current_variable_token_count
+                ),
+                kind: VariableType::Auxiliary,
+                rule: rule.clone(),
+            }
+        };
+
+        self.extracted_variables.push(variable);
+        self.extracted_usage_counts.push(1);
+        Ok(Symbol::terminal(index))
+    }
+}
+
+impl SymbolReplacer {
+    fn replace_symbols_in_rule(&mut self, rule: &Rule) -> Rule {
+        match rule {
+            Rule::Symbol(symbol) => self.replace_symbol(*symbol).into(),
+            Rule::Choice(elements) => Rule::Choice(
+                elements
+                    .iter()
+                    .map(|e| self.replace_symbols_in_rule(e))
+                    .collect(),
+            ),
+            Rule::Seq(elements) => Rule::Seq(
+                elements
+                    .iter()
+                    .map(|e| self.replace_symbols_in_rule(e))
+                    .collect(),
+            ),
+            Rule::Repeat(content) => Rule::Repeat(Box::new(self.replace_symbols_in_rule(content))),
+            Rule::Metadata { rule, params } => Rule::Metadata {
+                params: params.clone(),
+                rule: Box::new(self.replace_symbols_in_rule(rule)),
+            },
+            _ => rule.clone(),
+        }
+    }
+
+    fn replace_symbol(&self, symbol: Symbol) -> Symbol {
+        if !symbol.is_non_terminal() {
+            return symbol;
+        }
+
+        if let Some(replacement) = self.replacements.get(&symbol.index) {
+            return Symbol::terminal(*replacement);
+        }
+
+        let mut adjusted_index = symbol.index;
+        for replaced_index in self.replacements.keys() {
+            if *replaced_index < symbol.index {
+                adjusted_index -= 1;
+            }
+        }
+
+        Symbol::non_terminal(adjusted_index)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_extraction() {
+        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
+            Variable::named(
+                "rule_0",
+                Rule::repeat(Rule::seq(vec![
+                    Rule::string("a"),
+                    Rule::pattern("b", ""),
+                    Rule::choice(vec![
+                        Rule::non_terminal(1),
+                        Rule::non_terminal(2),
+                        Rule::token(Rule::repeat(Rule::choice(vec![
+                            Rule::string("c"),
+                            Rule::string("d"),
+                        ]))),
+                    ]),
+                ])),
+            ),
+            Variable::named("rule_1", Rule::pattern("e", "")),
+            Variable::named("rule_2", Rule::pattern("b", "")),
+            Variable::named(
+                "rule_3",
+                Rule::seq(vec![Rule::non_terminal(2), Rule::Blank]),
+            ),
+        ]))
+        .unwrap();
+
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![
+                Variable::named(
+                    "rule_0",
+                    Rule::repeat(Rule::seq(vec![
+                        // The string "a" was replaced by a symbol referencing the lexical grammar
+                        Rule::terminal(0),
+                        // The pattern "b" was replaced by a symbol referencing the lexical grammar
+                        Rule::terminal(1),
+                        Rule::choice(vec![
+                            // The symbol referencing `rule_1` was replaced by a symbol referencing
+                            // the lexical grammar.
+                            Rule::terminal(3),
+                            // The symbol referencing `rule_2` had its index decremented because
+                            // `rule_1` was moved to the lexical grammar.
+                            Rule::non_terminal(1),
+                            // The rule wrapped in `token` was replaced by a symbol referencing
+                            // the lexical grammar.
+                            Rule::terminal(2),
+                        ])
+                    ]))
+                ),
+                // The pattern "e" was only used in once place: as the definition of `rule_1`,
+                // so that rule was moved to the lexical grammar. The pattern "b" appeared in
+                // two places, so it was not moved into the lexical grammar.
+                Variable::named("rule_2", Rule::terminal(1)),
+                Variable::named(
+                    "rule_3",
+                    Rule::seq(vec![Rule::non_terminal(1), Rule::Blank,])
+                ),
+            ]
+        );
+
+        assert_eq!(
+            lexical_grammar.variables,
+            vec![
+                Variable::anonymous("a", Rule::string("a")),
+                Variable::auxiliary("rule_0_token1", Rule::pattern("b", "")),
+                Variable::auxiliary(
+                    "rule_0_token2",
+                    Rule::repeat(Rule::choice(vec![Rule::string("c"), Rule::string("d"),]))
+                ),
+                Variable::named("rule_1", Rule::pattern("e", "")),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_start_rule_is_token() {
+        let (syntax_grammar, lexical_grammar) =
+            extract_tokens(build_grammar(vec![Variable::named(
+                "rule_0",
+                Rule::string("hello"),
+            )]))
+            .unwrap();
+
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![Variable::named("rule_0", Rule::terminal(0)),]
+        );
+        assert_eq!(
+            lexical_grammar.variables,
+            vec![Variable::anonymous("hello", Rule::string("hello")),]
+        );
+    }
+
+    #[test]
+    fn test_extracting_extra_symbols() {
+        let mut grammar = build_grammar(vec![
+            Variable::named("rule_0", Rule::string("x")),
+            Variable::named("comment", Rule::pattern("//.*", "")),
+        ]);
+        grammar.extra_symbols = vec![Rule::string(" "), Rule::non_terminal(1)];
+
+        let (syntax_grammar, lexical_grammar) = extract_tokens(grammar).unwrap();
+        assert_eq!(syntax_grammar.extra_symbols, vec![Symbol::terminal(1),]);
+        assert_eq!(lexical_grammar.separators, vec![Rule::string(" "),]);
+    }
+
+    #[test]
+    fn test_extract_externals() {
+        let mut grammar = build_grammar(vec![
+            Variable::named(
+                "rule_0",
+                Rule::seq(vec![
+                    Rule::external(0),
+                    Rule::string("a"),
+                    Rule::non_terminal(1),
+                    Rule::non_terminal(2),
+                ]),
+            ),
+            Variable::named("rule_1", Rule::string("b")),
+            Variable::named("rule_2", Rule::string("c")),
+        ]);
+        grammar.external_tokens = vec![
+            Variable::named("external_0", Rule::external(0)),
+            Variable::anonymous("a", Rule::string("a")),
+            Variable::named("rule_2", Rule::non_terminal(2)),
+        ];
+
+        let (syntax_grammar, _) = extract_tokens(grammar).unwrap();
+
+        assert_eq!(
+            syntax_grammar.external_tokens,
+            vec![
+                ExternalToken {
+                    name: "external_0".to_string(),
+                    kind: VariableType::Named,
+                    corresponding_internal_token: None,
+                },
+                ExternalToken {
+                    name: "a".to_string(),
+                    kind: VariableType::Anonymous,
+                    corresponding_internal_token: Some(Symbol::terminal(0)),
+                },
+                ExternalToken {
+                    name: "rule_2".to_string(),
+                    kind: VariableType::Named,
+                    corresponding_internal_token: Some(Symbol::terminal(2)),
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_error_on_external_with_same_name_as_non_terminal() {
+        let mut grammar = build_grammar(vec![
+            Variable::named(
+                "rule_0",
+                Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
+            ),
+            Variable::named(
+                "rule_1",
+                Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2)]),
+            ),
+            Variable::named("rule_2", Rule::string("a")),
+        ]);
+        grammar.external_tokens = vec![Variable::named("rule_1", Rule::non_terminal(1))];
+
+        match extract_tokens(grammar) {
+            Err(e) => {
+                assert_eq!(e.to_string(), "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule");
+            }
+            _ => {
+                panic!("Expected an error but got no error");
+            }
+        }
+    }
+
+    #[test]
+    fn test_extraction_on_hidden_terminal() {
+        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
+            Variable::named("rule_0", Rule::non_terminal(1)),
+            Variable::hidden("_rule_1", Rule::string("a")),
+        ]))
+        .unwrap();
+
+        // The rule `_rule_1` should not "absorb" the
+        // terminal "a", since it is hidden,
+        // so we expect two variables still
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![
+                Variable::named("rule_0", Rule::non_terminal(1)),
+                Variable::hidden("_rule_1", Rule::terminal(0)),
+            ]
+        );
+
+        // We should not have a hidden rule in our lexical grammar, only the terminal "a"
+        assert_eq!(
+            lexical_grammar.variables,
+            vec![Variable::anonymous("a", Rule::string("a"))]
+        );
+    }
+
+    #[test]
+    fn test_extraction_with_empty_string() {
+        assert!(extract_tokens(build_grammar(vec![
+            Variable::named("rule_0", Rule::non_terminal(1)),
+            Variable::hidden("_rule_1", Rule::string("")),
+        ]))
+        .is_err());
+    }
+
+    fn build_grammar(variables: Vec<Variable>) -> InternedGrammar {
+        InternedGrammar {
+            variables,
+            ..Default::default()
+        }
+    }
+}

cli/generate/src/prepare_grammar/flatten_grammar.rs

@@ -0,0 +1,453 @@
+use anyhow::{anyhow, Result};
+
+use super::ExtractedSyntaxGrammar;
+use crate::{
+    grammars::{Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable},
+    rules::{Alias, Associativity, Precedence, Rule, Symbol},
+};
+
+struct RuleFlattener {
+    production: Production,
+    precedence_stack: Vec<Precedence>,
+    associativity_stack: Vec<Associativity>,
+    alias_stack: Vec<Alias>,
+    field_name_stack: Vec<String>,
+}
+
+impl RuleFlattener {
+    const fn new() -> Self {
+        Self {
+            production: Production {
+                steps: Vec::new(),
+                dynamic_precedence: 0,
+            },
+            precedence_stack: Vec::new(),
+            associativity_stack: Vec::new(),
+            alias_stack: Vec::new(),
+            field_name_stack: Vec::new(),
+        }
+    }
+
+    fn flatten(mut self, rule: Rule) -> Production {
+        self.apply(rule, true);
+        self.production
+    }
+
+    fn apply(&mut self, rule: Rule, at_end: bool) -> bool {
+        match rule {
+            Rule::Seq(members) => {
+                let mut result = false;
+                let last_index = members.len() - 1;
+                for (i, member) in members.into_iter().enumerate() {
+                    result |= self.apply(member, i == last_index && at_end);
+                }
+                result
+            }
+            Rule::Metadata { rule, params } => {
+                let mut has_precedence = false;
+                if !params.precedence.is_none() {
+                    has_precedence = true;
+                    self.precedence_stack.push(params.precedence);
+                }
+
+                let mut has_associativity = false;
+                if let Some(associativity) = params.associativity {
+                    has_associativity = true;
+                    self.associativity_stack.push(associativity);
+                }
+
+                let mut has_alias = false;
+                if let Some(alias) = params.alias {
+                    has_alias = true;
+                    self.alias_stack.push(alias);
+                }
+
+                let mut has_field_name = false;
+                if let Some(field_name) = params.field_name {
+                    has_field_name = true;
+                    self.field_name_stack.push(field_name);
+                }
+
+                if params.dynamic_precedence.abs() > self.production.dynamic_precedence.abs() {
+                    self.production.dynamic_precedence = params.dynamic_precedence;
+                }
+
+                let did_push = self.apply(*rule, at_end);
+
+                if has_precedence {
+                    self.precedence_stack.pop();
+                    if did_push && !at_end {
+                        self.production.steps.last_mut().unwrap().precedence = self
+                            .precedence_stack
+                            .last()
+                            .cloned()
+                            .unwrap_or(Precedence::None);
+                    }
+                }
+
+                if has_associativity {
+                    self.associativity_stack.pop();
+                    if did_push && !at_end {
+                        self.production.steps.last_mut().unwrap().associativity =
+                            self.associativity_stack.last().copied();
+                    }
+                }
+
+                if has_alias {
+                    self.alias_stack.pop();
+                }
+
+                if has_field_name {
+                    self.field_name_stack.pop();
+                }
+
+                did_push
+            }
+            Rule::Symbol(symbol) => {
+                self.production.steps.push(ProductionStep {
+                    symbol,
+                    precedence: self
+                        .precedence_stack
+                        .last()
+                        .cloned()
+                        .unwrap_or(Precedence::None),
+                    associativity: self.associativity_stack.last().copied(),
+                    alias: self.alias_stack.last().cloned(),
+                    field_name: self.field_name_stack.last().cloned(),
+                });
+                true
+            }
+            _ => false,
+        }
+    }
+}
+
+fn extract_choices(rule: Rule) -> Vec<Rule> {
+    match rule {
+        Rule::Seq(elements) => {
+            let mut result = vec![Rule::Blank];
+            for element in elements {
+                let extraction = extract_choices(element);
+                let mut next_result = Vec::new();
+                for entry in result {
+                    for extraction_entry in &extraction {
+                        next_result.push(Rule::Seq(vec![entry.clone(), extraction_entry.clone()]));
+                    }
+                }
+                result = next_result;
+            }
+            result
+        }
+        Rule::Choice(elements) => {
+            let mut result = Vec::new();
+            for element in elements {
+                for rule in extract_choices(element) {
+                    result.push(rule);
+                }
+            }
+            result
+        }
+        Rule::Metadata { rule, params } => extract_choices(*rule)
+            .into_iter()
+            .map(|rule| Rule::Metadata {
+                rule: Box::new(rule),
+                params: params.clone(),
+            })
+            .collect(),
+        _ => vec![rule],
+    }
+}
+
+fn flatten_variable(variable: Variable) -> SyntaxVariable {
+    let mut productions = Vec::new();
+    for rule in extract_choices(variable.rule) {
+        let production = RuleFlattener::new().flatten(rule);
+        if !productions.contains(&production) {
+            productions.push(production);
+        }
+    }
+    SyntaxVariable {
+        name: variable.name,
+        kind: variable.kind,
+        productions,
+    }
+}
+
+fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool {
+    for variable in variables {
+        for production in &variable.productions {
+            for step in &production.steps {
+                if step.symbol == symbol {
+                    return true;
+                }
+            }
+        }
+    }
+    false
+}
+
+pub(super) fn flatten_grammar(grammar: ExtractedSyntaxGrammar) -> Result<SyntaxGrammar> {
+    let mut variables = Vec::new();
+    for variable in grammar.variables {
+        variables.push(flatten_variable(variable));
+    }
+    for (i, variable) in variables.iter().enumerate() {
+        let symbol = Symbol::non_terminal(i);
+
+        for production in &variable.productions {
+            if production.steps.is_empty() && symbol_is_used(&variables, symbol) {
+                return Err(anyhow!(
+                    "The rule `{}` matches the empty string.
+
+Tree-sitter does not support syntactic rules that match the empty string
+unless they are used only as the grammar's start rule.
+",
+                    variable.name
+                ));
+            }
+
+            if grammar.variables_to_inline.contains(&symbol)
+                && production.steps.iter().any(|step| step.symbol == symbol)
+            {
+                return Err(anyhow!(
+                    "Rule `{}` cannot be inlined because it contains a reference to itself.",
+                    variable.name,
+                ));
+            }
+        }
+    }
+    Ok(SyntaxGrammar {
+        extra_symbols: grammar.extra_symbols,
+        expected_conflicts: grammar.expected_conflicts,
+        variables_to_inline: grammar.variables_to_inline,
+        precedence_orderings: grammar.precedence_orderings,
+        external_tokens: grammar.external_tokens,
+        supertype_symbols: grammar.supertype_symbols,
+        word_token: grammar.word_token,
+        variables,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::grammars::VariableType;
+
+    #[test]
+    fn test_flatten_grammar() {
+        let result = flatten_variable(Variable {
+            name: "test".to_string(),
+            kind: VariableType::Named,
+            rule: Rule::seq(vec![
+                Rule::non_terminal(1),
+                Rule::prec_left(
+                    Precedence::Integer(101),
+                    Rule::seq(vec![
+                        Rule::non_terminal(2),
+                        Rule::choice(vec![
+                            Rule::prec_right(
+                                Precedence::Integer(102),
+                                Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
+                            ),
+                            Rule::non_terminal(5),
+                        ]),
+                        Rule::non_terminal(6),
+                    ]),
+                ),
+                Rule::non_terminal(7),
+            ]),
+        });
+
+        assert_eq!(
+            result.productions,
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(3))
+                            .with_prec(Precedence::Integer(102), Some(Associativity::Right)),
+                        ProductionStep::new(Symbol::non_terminal(4))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ]
+                },
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(5))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ]
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_maximum_dynamic_precedence() {
+        let result = flatten_variable(Variable {
+            name: "test".to_string(),
+            kind: VariableType::Named,
+            rule: Rule::seq(vec![
+                Rule::non_terminal(1),
+                Rule::prec_dynamic(
+                    101,
+                    Rule::seq(vec![
+                        Rule::non_terminal(2),
+                        Rule::choice(vec![
+                            Rule::prec_dynamic(
+                                102,
+                                Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
+                            ),
+                            Rule::non_terminal(5),
+                        ]),
+                        Rule::non_terminal(6),
+                    ]),
+                ),
+                Rule::non_terminal(7),
+            ]),
+        });
+
+        assert_eq!(
+            result.productions,
+            vec![
+                Production {
+                    dynamic_precedence: 102,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::non_terminal(3)),
+                        ProductionStep::new(Symbol::non_terminal(4)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ],
+                },
+                Production {
+                    dynamic_precedence: 101,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::non_terminal(5)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ],
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_final_precedence() {
+        let result = flatten_variable(Variable {
+            name: "test".to_string(),
+            kind: VariableType::Named,
+            rule: Rule::prec_left(
+                Precedence::Integer(101),
+                Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
+            ),
+        });
+
+        assert_eq!(
+            result.productions,
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    ProductionStep::new(Symbol::non_terminal(1))
+                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                    ProductionStep::new(Symbol::non_terminal(2))
+                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                ]
+            }]
+        );
+
+        let result = flatten_variable(Variable {
+            name: "test".to_string(),
+            kind: VariableType::Named,
+            rule: Rule::prec_left(
+                Precedence::Integer(101),
+                Rule::seq(vec![Rule::non_terminal(1)]),
+            ),
+        });
+
+        assert_eq!(
+            result.productions,
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![ProductionStep::new(Symbol::non_terminal(1))
+                    .with_prec(Precedence::Integer(101), Some(Associativity::Left)),]
+            }]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_field_names() {
+        let result = flatten_variable(Variable {
+            name: "test".to_string(),
+            kind: VariableType::Named,
+            rule: Rule::seq(vec![
+                Rule::field("first-thing".to_string(), Rule::terminal(1)),
+                Rule::terminal(2),
+                Rule::choice(vec![
+                    Rule::Blank,
+                    Rule::field("second-thing".to_string(), Rule::terminal(3)),
+                ]),
+            ]),
+        });
+
+        assert_eq!(
+            result.productions,
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
+                        ProductionStep::new(Symbol::terminal(2))
+                    ]
+                },
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
+                        ProductionStep::new(Symbol::terminal(2)),
+                        ProductionStep::new(Symbol::terminal(3)).with_field_name("second-thing"),
+                    ]
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_recursive_inline_variable() {
+        let result = flatten_grammar(ExtractedSyntaxGrammar {
+            extra_symbols: Vec::new(),
+            expected_conflicts: Vec::new(),
+            variables_to_inline: vec![Symbol::non_terminal(0)],
+            precedence_orderings: Vec::new(),
+            external_tokens: Vec::new(),
+            supertype_symbols: Vec::new(),
+            word_token: None,
+            variables: vec![Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::seq(vec![
+                    Rule::non_terminal(0),
+                    Rule::non_terminal(1),
+                    Rule::non_terminal(2),
+                ]),
+            }],
+        });
+
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Rule `test` cannot be inlined because it contains a reference to itself.",
+        );
+    }
+}

cli/generate/src/prepare_grammar/intern_symbols.rs

@@ -0,0 +1,256 @@
+use anyhow::{anyhow, Result};
+
+use super::InternedGrammar;
+use crate::{
+    grammars::{InputGrammar, Variable, VariableType},
+    rules::{Rule, Symbol},
+};
+
+pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result<InternedGrammar> {
+    let interner = Interner { grammar };
+
+    if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden {
+        return Err(anyhow!("A grammar's start rule must be visible."));
+    }
+
+    let mut variables = Vec::with_capacity(grammar.variables.len());
+    for variable in &grammar.variables {
+        variables.push(Variable {
+            name: variable.name.clone(),
+            kind: variable_type_for_name(&variable.name),
+            rule: interner.intern_rule(&variable.rule, Some(&variable.name))?,
+        });
+    }
+
+    let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len());
+    for external_token in &grammar.external_tokens {
+        let rule = interner.intern_rule(external_token, None)?;
+        let (name, kind) = if let Rule::NamedSymbol(name) = external_token {
+            (name.clone(), variable_type_for_name(name))
+        } else {
+            (String::new(), VariableType::Anonymous)
+        };
+        external_tokens.push(Variable { name, kind, rule });
+    }
+
+    let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len());
+    for extra_token in &grammar.extra_symbols {
+        extra_symbols.push(interner.intern_rule(extra_token, None)?);
+    }
+
+    let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len());
+    for supertype_symbol_name in &grammar.supertype_symbols {
+        supertype_symbols.push(interner.intern_name(supertype_symbol_name).ok_or_else(|| {
+            anyhow!("Undefined symbol `{supertype_symbol_name}` in grammar's supertypes array")
+        })?);
+    }
+
+    let mut expected_conflicts = Vec::new();
+    for conflict in &grammar.expected_conflicts {
+        let mut interned_conflict = Vec::with_capacity(conflict.len());
+        for name in conflict {
+            interned_conflict.push(interner.intern_name(name).ok_or_else(|| {
+                anyhow!("Undefined symbol `{name}` in grammar's conflicts array")
+            })?);
+        }
+        expected_conflicts.push(interned_conflict);
+    }
+
+    let mut variables_to_inline = Vec::new();
+    for name in &grammar.variables_to_inline {
+        if let Some(symbol) = interner.intern_name(name) {
+            variables_to_inline.push(symbol);
+        }
+    }
+
+    let mut word_token = None;
+    if let Some(name) = grammar.word_token.as_ref() {
+        word_token = Some(
+            interner
+                .intern_name(name)
+                .ok_or_else(|| anyhow!("Undefined symbol `{name}` as grammar's word token"))?,
+        );
+    }
+
+    for (i, variable) in variables.iter_mut().enumerate() {
+        if supertype_symbols.contains(&Symbol::non_terminal(i)) {
+            variable.kind = VariableType::Hidden;
+        }
+    }
+
+    Ok(InternedGrammar {
+        variables,
+        external_tokens,
+        extra_symbols,
+        expected_conflicts,
+        variables_to_inline,
+        supertype_symbols,
+        word_token,
+        precedence_orderings: grammar.precedence_orderings.clone(),
+    })
+}
+
+struct Interner<'a> {
+    grammar: &'a InputGrammar,
+}
+
+impl<'a> Interner<'a> {
+    fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> Result<Rule> {
+        match rule {
+            Rule::Choice(elements) => {
+                self.check_single(elements, name);
+                let mut result = Vec::with_capacity(elements.len());
+                for element in elements {
+                    result.push(self.intern_rule(element, name)?);
+                }
+                Ok(Rule::Choice(result))
+            }
+            Rule::Seq(elements) => {
+                self.check_single(elements, name);
+                let mut result = Vec::with_capacity(elements.len());
+                for element in elements {
+                    result.push(self.intern_rule(element, name)?);
+                }
+                Ok(Rule::Seq(result))
+            }
+            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(self.intern_rule(content, name)?))),
+            Rule::Metadata { rule, params } => Ok(Rule::Metadata {
+                rule: Box::new(self.intern_rule(rule, name)?),
+                params: params.clone(),
+            }),
+            Rule::NamedSymbol(name) => self.intern_name(name).map_or_else(
+                || Err(anyhow!("Undefined symbol `{name}`")),
+                |symbol| Ok(Rule::Symbol(symbol)),
+            ),
+            _ => Ok(rule.clone()),
+        }
+    }
+
+    fn intern_name(&self, symbol: &str) -> Option<Symbol> {
+        for (i, variable) in self.grammar.variables.iter().enumerate() {
+            if variable.name == symbol {
+                return Some(Symbol::non_terminal(i));
+            }
+        }
+
+        for (i, external_token) in self.grammar.external_tokens.iter().enumerate() {
+            if let Rule::NamedSymbol(name) = external_token {
+                if name == symbol {
+                    return Some(Symbol::external(i));
+                }
+            }
+        }
+
+        None
+    }
+
+    // In the case of a seq or choice rule of 1 element in a hidden rule, weird
+    // inconsistent behavior with queries can occur. So we should warn the user about it.
+    fn check_single(&self, elements: &[Rule], name: Option<&str>) {
+        if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) {
+            eprintln!(
+                "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.",
+                name.unwrap_or_default()
+            );
+        }
+    }
+}
+
+fn variable_type_for_name(name: &str) -> VariableType {
+    if name.starts_with('_') {
+        VariableType::Hidden
+    } else {
+        VariableType::Named
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_repeat_expansion() {
+        let grammar = intern_symbols(&build_grammar(vec![
+            Variable::named("x", Rule::choice(vec![Rule::named("y"), Rule::named("_z")])),
+            Variable::named("y", Rule::named("_z")),
+            Variable::named("_z", Rule::string("a")),
+        ]))
+        .unwrap();
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "x",
+                    Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),])
+                ),
+                Variable::named("y", Rule::non_terminal(2)),
+                Variable::hidden("_z", Rule::string("a")),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_interning_external_token_names() {
+        // Variable `y` is both an internal and an external token.
+        // Variable `z` is just an external token.
+        let mut input_grammar = build_grammar(vec![
+            Variable::named(
+                "w",
+                Rule::choice(vec![Rule::named("x"), Rule::named("y"), Rule::named("z")]),
+            ),
+            Variable::named("x", Rule::string("a")),
+            Variable::named("y", Rule::string("b")),
+        ]);
+        input_grammar
+            .external_tokens
+            .extend(vec![Rule::named("y"), Rule::named("z")]);
+
+        let grammar = intern_symbols(&input_grammar).unwrap();
+
+        // Variable `y` is referred to by its internal index.
+        // Variable `z` is referred to by its external index.
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "w",
+                    Rule::choice(vec![
+                        Rule::non_terminal(1),
+                        Rule::non_terminal(2),
+                        Rule::external(1),
+                    ])
+                ),
+                Variable::named("x", Rule::string("a")),
+                Variable::named("y", Rule::string("b")),
+            ]
+        );
+
+        // The external token for `y` refers back to its internal index.
+        assert_eq!(
+            grammar.external_tokens,
+            vec![
+                Variable::named("y", Rule::non_terminal(2)),
+                Variable::named("z", Rule::external(1)),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_grammar_with_undefined_symbols() {
+        let result = intern_symbols(&build_grammar(vec![Variable::named("x", Rule::named("y"))]));
+
+        match result {
+            Err(e) => assert_eq!(e.to_string(), "Undefined symbol `y`"),
+            _ => panic!("Expected an error but got none"),
+        }
+    }
+
+    fn build_grammar(variables: Vec<Variable>) -> InputGrammar {
+        InputGrammar {
+            variables,
+            name: "the_language".to_string(),
+            ..Default::default()
+        }
+    }
+}

cli/generate/src/prepare_grammar/mod.rs

@@ -0,0 +1,251 @@
+mod expand_repeats;
+mod expand_tokens;
+mod extract_default_aliases;
+mod extract_tokens;
+mod flatten_grammar;
+mod intern_symbols;
+mod process_inlines;
+
+use std::{
+    cmp::Ordering,
+    collections::{hash_map, HashMap, HashSet},
+    mem,
+};
+
+use anyhow::{anyhow, Result};
+
+pub use self::expand_tokens::expand_tokens;
+use self::{
+    expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
+    extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
+    intern_symbols::intern_symbols, process_inlines::process_inlines,
+};
+use super::{
+    grammars::{
+        ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
+        SyntaxGrammar, Variable,
+    },
+    rules::{AliasMap, Precedence, Rule, Symbol},
+};
+
+pub struct IntermediateGrammar<T, U> {
+    variables: Vec<Variable>,
+    extra_symbols: Vec<T>,
+    expected_conflicts: Vec<Vec<Symbol>>,
+    precedence_orderings: Vec<Vec<PrecedenceEntry>>,
+    external_tokens: Vec<U>,
+    variables_to_inline: Vec<Symbol>,
+    supertype_symbols: Vec<Symbol>,
+    word_token: Option<Symbol>,
+}
+
+pub type InternedGrammar = IntermediateGrammar<Rule, Variable>;
+
+pub type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct ExtractedLexicalGrammar {
+    pub variables: Vec<Variable>,
+    pub separators: Vec<Rule>,
+}
+
+impl<T, U> Default for IntermediateGrammar<T, U> {
+    fn default() -> Self {
+        Self {
+            variables: Vec::default(),
+            extra_symbols: Vec::default(),
+            expected_conflicts: Vec::default(),
+            precedence_orderings: Vec::default(),
+            external_tokens: Vec::default(),
+            variables_to_inline: Vec::default(),
+            supertype_symbols: Vec::default(),
+            word_token: Option::default(),
+        }
+    }
+}
+
+/// Transform an input grammar into separate components that are ready
+/// for parse table construction.
+pub fn prepare_grammar(
+    input_grammar: &InputGrammar,
+) -> Result<(
+    SyntaxGrammar,
+    LexicalGrammar,
+    InlinedProductionMap,
+    AliasMap,
+)> {
+    validate_precedences(input_grammar)?;
+
+    let interned_grammar = intern_symbols(input_grammar)?;
+    let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
+    let syntax_grammar = expand_repeats(syntax_grammar);
+    let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
+    let lexical_grammar = expand_tokens(lexical_grammar)?;
+    let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
+    let inlines = process_inlines(&syntax_grammar, &lexical_grammar)?;
+    Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
+}
+
+/// Check that all of the named precedences used in the grammar are declared
+/// within the `precedences` lists, and also that there are no conflicting
+/// precedence orderings declared in those lists.
+fn validate_precedences(grammar: &InputGrammar) -> Result<()> {
+    // Check that no rule contains a named precedence that is not present in
+    // any of the `precedences` lists.
+    fn validate(rule_name: &str, rule: &Rule, names: &HashSet<&String>) -> Result<()> {
+        match rule {
+            Rule::Repeat(rule) => validate(rule_name, rule, names),
+            Rule::Seq(elements) | Rule::Choice(elements) => elements
+                .iter()
+                .try_for_each(|e| validate(rule_name, e, names)),
+            Rule::Metadata { rule, params } => {
+                if let Precedence::Name(n) = &params.precedence {
+                    if !names.contains(n) {
+                        return Err(anyhow!("Undeclared precedence '{n}' in rule '{rule_name}'"));
+                    }
+                }
+                validate(rule_name, rule, names)?;
+                Ok(())
+            }
+            _ => Ok(()),
+        }
+    }
+
+    // For any two precedence names `a` and `b`, if `a` comes before `b`
+    // in some list, then it cannot come *after* `b` in any list.
+    let mut pairs = HashMap::new();
+    for list in &grammar.precedence_orderings {
+        for (i, mut entry1) in list.iter().enumerate() {
+            for mut entry2 in list.iter().skip(i + 1) {
+                if entry2 == entry1 {
+                    continue;
+                }
+                let mut ordering = Ordering::Greater;
+                if entry1 > entry2 {
+                    ordering = Ordering::Less;
+                    mem::swap(&mut entry1, &mut entry2);
+                }
+                match pairs.entry((entry1, entry2)) {
+                    hash_map::Entry::Vacant(e) => {
+                        e.insert(ordering);
+                    }
+                    hash_map::Entry::Occupied(e) => {
+                        if e.get() != &ordering {
+                            return Err(anyhow!(
+                                "Conflicting orderings for precedences {entry1} and {entry2}",
+                            ));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    let precedence_names = grammar
+        .precedence_orderings
+        .iter()
+        .flat_map(|l| l.iter())
+        .filter_map(|p| {
+            if let PrecedenceEntry::Name(n) = p {
+                Some(n)
+            } else {
+                None
+            }
+        })
+        .collect::<HashSet<&String>>();
+    for variable in &grammar.variables {
+        validate(&variable.name, &variable.rule, &precedence_names)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::grammars::VariableType;
+
+    #[test]
+    fn test_validate_precedences_with_undeclared_precedence() {
+        let grammar = InputGrammar {
+            precedence_orderings: vec![
+                vec![
+                    PrecedenceEntry::Name("a".to_string()),
+                    PrecedenceEntry::Name("b".to_string()),
+                ],
+                vec![
+                    PrecedenceEntry::Name("b".to_string()),
+                    PrecedenceEntry::Name("c".to_string()),
+                    PrecedenceEntry::Name("d".to_string()),
+                ],
+            ],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::Seq(vec![
+                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
+                    ]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::repeat(Rule::Choice(vec![
+                        Rule::prec_left(Precedence::Name("omg".to_string()), Rule::string("y")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
+                    ])),
+                },
+            ],
+            ..Default::default()
+        };
+
+        let result = validate_precedences(&grammar);
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Undeclared precedence 'omg' in rule 'v2'",
+        );
+    }
+
+    #[test]
+    fn test_validate_precedences_with_conflicting_order() {
+        let grammar = InputGrammar {
+            precedence_orderings: vec![
+                vec![
+                    PrecedenceEntry::Name("a".to_string()),
+                    PrecedenceEntry::Name("b".to_string()),
+                ],
+                vec![
+                    PrecedenceEntry::Name("b".to_string()),
+                    PrecedenceEntry::Name("c".to_string()),
+                    PrecedenceEntry::Name("a".to_string()),
+                ],
+            ],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::Seq(vec![
+                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
+                    ]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::repeat(Rule::Choice(vec![
+                        Rule::prec_left(Precedence::Name("a".to_string()), Rule::string("y")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
+                    ])),
+                },
+            ],
+            ..Default::default()
+        };
+
+        let result = validate_precedences(&grammar);
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Conflicting orderings for precedences 'a' and 'b'",
+        );
+    }
+}

cli/generate/src/prepare_grammar/process_inlines.rs

@@ -0,0 +1,547 @@
+use std::collections::HashMap;
+
+use anyhow::{anyhow, Result};
+
+use crate::{
+    grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
+    rules::SymbolType,
+};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+struct ProductionStepId {
+    // A `None` value here means that the production itself was produced via inlining,
+    // and is stored in the builder's `productions` vector, as opposed to being
+    // stored in one of the grammar's variables.
+    variable_index: Option<usize>,
+    production_index: usize,
+    step_index: usize,
+}
+
+struct InlinedProductionMapBuilder {
+    production_indices_by_step_id: HashMap<ProductionStepId, Vec<usize>>,
+    productions: Vec<Production>,
+}
+
+impl InlinedProductionMapBuilder {
+    fn build(mut self, grammar: &SyntaxGrammar) -> InlinedProductionMap {
+        let mut step_ids_to_process = Vec::new();
+        for (variable_index, variable) in grammar.variables.iter().enumerate() {
+            for production_index in 0..variable.productions.len() {
+                step_ids_to_process.push(ProductionStepId {
+                    variable_index: Some(variable_index),
+                    production_index,
+                    step_index: 0,
+                });
+                while !step_ids_to_process.is_empty() {
+                    let mut i = 0;
+                    while i < step_ids_to_process.len() {
+                        let step_id = step_ids_to_process[i];
+                        if let Some(step) = self.production_step_for_id(step_id, grammar) {
+                            if grammar.variables_to_inline.contains(&step.symbol) {
+                                let inlined_step_ids = self
+                                    .inline_production_at_step(step_id, grammar)
+                                    .iter()
+                                    .copied()
+                                    .map(|production_index| ProductionStepId {
+                                        variable_index: None,
+                                        production_index,
+                                        step_index: step_id.step_index,
+                                    });
+                                step_ids_to_process.splice(i..=i, inlined_step_ids);
+                            } else {
+                                step_ids_to_process[i] = ProductionStepId {
+                                    variable_index: step_id.variable_index,
+                                    production_index: step_id.production_index,
+                                    step_index: step_id.step_index + 1,
+                                };
+                                i += 1;
+                            }
+                        } else {
+                            step_ids_to_process.remove(i);
+                        }
+                    }
+                }
+            }
+        }
+
+        let productions = self.productions;
+        let production_indices_by_step_id = self.production_indices_by_step_id;
+        let production_map = production_indices_by_step_id
+            .into_iter()
+            .map(|(step_id, production_indices)| {
+                let production = step_id.variable_index.map_or_else(
+                    || &productions[step_id.production_index],
+                    |variable_index| {
+                        &grammar.variables[variable_index].productions[step_id.production_index]
+                    },
+                ) as *const Production;
+                ((production, step_id.step_index as u32), production_indices)
+            })
+            .collect();
+
+        InlinedProductionMap {
+            productions,
+            production_map,
+        }
+    }
+
+    fn inline_production_at_step<'a>(
+        &'a mut self,
+        step_id: ProductionStepId,
+        grammar: &'a SyntaxGrammar,
+    ) -> &'a [usize] {
+        // Build a list of productions produced by inlining rules.
+        let mut i = 0;
+        let step_index = step_id.step_index;
+        let mut productions_to_add = vec![self.production_for_id(step_id, grammar).clone()];
+        while i < productions_to_add.len() {
+            if let Some(step) = productions_to_add[i].steps.get(step_index) {
+                let symbol = step.symbol;
+                if grammar.variables_to_inline.contains(&symbol) {
+                    // Remove the production from the vector, replacing it with a placeholder.
+                    let production = productions_to_add
+                        .splice(i..=i, std::iter::once(&Production::default()).cloned())
+                        .next()
+                        .unwrap();
+
+                    // Replace the placeholder with the inlined productions.
+                    productions_to_add.splice(
+                        i..=i,
+                        grammar.variables[symbol.index].productions.iter().map(|p| {
+                            let mut production = production.clone();
+                            let removed_step = production
+                                .steps
+                                .splice(step_index..=step_index, p.steps.iter().cloned())
+                                .next()
+                                .unwrap();
+                            let inserted_steps =
+                                &mut production.steps[step_index..(step_index + p.steps.len())];
+                            if let Some(alias) = removed_step.alias {
+                                for inserted_step in inserted_steps.iter_mut() {
+                                    inserted_step.alias = Some(alias.clone());
+                                }
+                            }
+                            if let Some(field_name) = removed_step.field_name {
+                                for inserted_step in inserted_steps.iter_mut() {
+                                    inserted_step.field_name = Some(field_name.clone());
+                                }
+                            }
+                            if let Some(last_inserted_step) = inserted_steps.last_mut() {
+                                if last_inserted_step.precedence.is_none() {
+                                    last_inserted_step.precedence = removed_step.precedence;
+                                }
+                                if last_inserted_step.associativity.is_none() {
+                                    last_inserted_step.associativity = removed_step.associativity;
+                                }
+                            }
+                            if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() {
+                                production.dynamic_precedence = p.dynamic_precedence;
+                            }
+                            production
+                        }),
+                    );
+
+                    continue;
+                }
+            }
+            i += 1;
+        }
+
+        // Store all the computed productions.
+        let result = productions_to_add
+            .into_iter()
+            .map(|production| {
+                self.productions
+                    .iter()
+                    .position(|p| *p == production)
+                    .unwrap_or_else(|| {
+                        self.productions.push(production);
+                        self.productions.len() - 1
+                    })
+            })
+            .collect();
+
+        // Cache these productions based on the original production step.
+        self.production_indices_by_step_id
+            .entry(step_id)
+            .or_insert(result)
+    }
+
+    fn production_for_id<'a>(
+        &'a self,
+        id: ProductionStepId,
+        grammar: &'a SyntaxGrammar,
+    ) -> &'a Production {
+        id.variable_index.map_or_else(
+            || &self.productions[id.production_index],
+            |variable_index| &grammar.variables[variable_index].productions[id.production_index],
+        )
+    }
+
+    fn production_step_for_id<'a>(
+        &'a self,
+        id: ProductionStepId,
+        grammar: &'a SyntaxGrammar,
+    ) -> Option<&'a ProductionStep> {
+        self.production_for_id(id, grammar).steps.get(id.step_index)
+    }
+}
+
+pub(super) fn process_inlines(
+    grammar: &SyntaxGrammar,
+    lexical_grammar: &LexicalGrammar,
+) -> Result<InlinedProductionMap> {
+    for symbol in &grammar.variables_to_inline {
+        match symbol.kind {
+            SymbolType::External => {
+                return Err(anyhow!(
+                    "External token `{}` cannot be inlined",
+                    grammar.external_tokens[symbol.index].name
+                ))
+            }
+            SymbolType::Terminal => {
+                return Err(anyhow!(
+                    "Token `{}` cannot be inlined",
+                    lexical_grammar.variables[symbol.index].name,
+                ))
+            }
+            SymbolType::NonTerminal if symbol.index == 0 => {
+                return Err(anyhow!(
+                    "Rule `{}` cannot be inlined because it is the first rule",
+                    grammar.variables[symbol.index].name,
+                ))
+            }
+            _ => {}
+        }
+    }
+
+    Ok(InlinedProductionMapBuilder {
+        productions: Vec::new(),
+        production_indices_by_step_id: HashMap::new(),
+    }
+    .build(grammar))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::{LexicalVariable, SyntaxVariable, VariableType},
+        rules::{Associativity, Precedence, Symbol},
+    };
+
+    #[test]
+    fn test_basic_inlining() {
+        let grammar = SyntaxGrammar {
+            variables_to_inline: vec![Symbol::non_terminal(1)],
+            variables: vec![
+                SyntaxVariable {
+                    name: "non-terminal-0".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(10)),
+                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
+                            ProductionStep::new(Symbol::terminal(11)),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-1".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![
+                        Production {
+                            dynamic_precedence: 0,
+                            steps: vec![
+                                ProductionStep::new(Symbol::terminal(12)),
+                                ProductionStep::new(Symbol::terminal(13)),
+                            ],
+                        },
+                        Production {
+                            dynamic_precedence: -2,
+                            steps: vec![ProductionStep::new(Symbol::terminal(14))],
+                        },
+                    ],
+                },
+            ],
+            ..Default::default()
+        };
+
+        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
+
+        // Nothing to inline at step 0.
+        assert!(inline_map
+            .inlined_productions(&grammar.variables[0].productions[0], 0)
+            .is_none());
+
+        // Inlining variable 1 yields two productions.
+        assert_eq!(
+            inline_map
+                .inlined_productions(&grammar.variables[0].productions[0], 1)
+                .unwrap()
+                .cloned()
+                .collect::<Vec<_>>(),
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(12)),
+                        ProductionStep::new(Symbol::terminal(13)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                    ],
+                },
+                Production {
+                    dynamic_precedence: -2,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(14)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                    ],
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_nested_inlining() {
+        let grammar = SyntaxGrammar {
+            variables: vec![
+                SyntaxVariable {
+                    name: "non-terminal-0".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(10)),
+                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
+                            ProductionStep::new(Symbol::terminal(11)),
+                            ProductionStep::new(Symbol::non_terminal(2)), // inlined
+                            ProductionStep::new(Symbol::terminal(12)),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-1".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![
+                        Production {
+                            dynamic_precedence: 0,
+                            steps: vec![ProductionStep::new(Symbol::terminal(13))],
+                        },
+                        Production {
+                            dynamic_precedence: 0,
+                            steps: vec![
+                                ProductionStep::new(Symbol::non_terminal(3)), // inlined
+                                ProductionStep::new(Symbol::terminal(14)),
+                            ],
+                        },
+                    ],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-2".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![ProductionStep::new(Symbol::terminal(15))],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-3".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![ProductionStep::new(Symbol::terminal(16))],
+                    }],
+                },
+            ],
+            variables_to_inline: vec![
+                Symbol::non_terminal(1),
+                Symbol::non_terminal(2),
+                Symbol::non_terminal(3),
+            ],
+            ..Default::default()
+        };
+
+        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
+
+        let productions = inline_map
+            .inlined_productions(&grammar.variables[0].productions[0], 1)
+            .unwrap()
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            productions.iter().copied().cloned().collect::<Vec<_>>(),
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(13)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::terminal(12)),
+                    ],
+                },
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(16)),
+                        ProductionStep::new(Symbol::terminal(14)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::terminal(12)),
+                    ],
+                },
+            ]
+        );
+
+        assert_eq!(
+            inline_map
+                .inlined_productions(productions[0], 3)
+                .unwrap()
+                .cloned()
+                .collect::<Vec<_>>(),
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    ProductionStep::new(Symbol::terminal(10)),
+                    ProductionStep::new(Symbol::terminal(13)),
+                    ProductionStep::new(Symbol::terminal(11)),
+                    ProductionStep::new(Symbol::terminal(15)),
+                    ProductionStep::new(Symbol::terminal(12)),
+                ],
+            },]
+        );
+    }
+
+    #[test]
+    fn test_inlining_with_precedence_and_alias() {
+        let grammar = SyntaxGrammar {
+            variables_to_inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
+            variables: vec![
+                SyntaxVariable {
+                    name: "non-terminal-0".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            // inlined
+                            ProductionStep::new(Symbol::non_terminal(1))
+                                .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
+                            ProductionStep::new(Symbol::terminal(10)),
+                            // inlined
+                            ProductionStep::new(Symbol::non_terminal(2))
+                                .with_alias("outer_alias", true),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-1".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(11))
+                                .with_prec(Precedence::Integer(2), None)
+                                .with_alias("inner_alias", true),
+                            ProductionStep::new(Symbol::terminal(12)),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-2".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![ProductionStep::new(Symbol::terminal(13))],
+                    }],
+                },
+            ],
+            ..Default::default()
+        };
+
+        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
+
+        let productions = inline_map
+            .inlined_productions(&grammar.variables[0].productions[0], 0)
+            .unwrap()
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            productions.iter().copied().cloned().collect::<Vec<_>>(),
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    // The first step in the inlined production retains its precedence
+                    // and alias.
+                    ProductionStep::new(Symbol::terminal(11))
+                        .with_prec(Precedence::Integer(2), None)
+                        .with_alias("inner_alias", true),
+                    // The final step of the inlined production inherits the precedence of
+                    // the inlined step.
+                    ProductionStep::new(Symbol::terminal(12))
+                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
+                    ProductionStep::new(Symbol::terminal(10)),
+                    ProductionStep::new(Symbol::non_terminal(2)).with_alias("outer_alias", true),
+                ]
+            }],
+        );
+
+        assert_eq!(
+            inline_map
+                .inlined_productions(productions[0], 3)
+                .unwrap()
+                .cloned()
+                .collect::<Vec<_>>(),
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    ProductionStep::new(Symbol::terminal(11))
+                        .with_prec(Precedence::Integer(2), None)
+                        .with_alias("inner_alias", true),
+                    ProductionStep::new(Symbol::terminal(12))
+                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
+                    ProductionStep::new(Symbol::terminal(10)),
+                    // All steps of the inlined production inherit their alias from the
+                    // inlined step.
+                    ProductionStep::new(Symbol::terminal(13)).with_alias("outer_alias", true),
+                ]
+            }],
+        );
+    }
+
+    #[test]
+    fn test_error_when_inlining_tokens() {
+        let lexical_grammar = LexicalGrammar {
+            variables: vec![LexicalVariable {
+                name: "something".to_string(),
+                kind: VariableType::Named,
+                implicit_precedence: 0,
+                start_state: 0,
+            }],
+            ..Default::default()
+        };
+
+        let grammar = SyntaxGrammar {
+            variables_to_inline: vec![Symbol::terminal(0)],
+            variables: vec![SyntaxVariable {
+                name: "non-terminal-0".to_string(),
+                kind: VariableType::Named,
+                productions: vec![Production {
+                    dynamic_precedence: 0,
+                    steps: vec![ProductionStep::new(Symbol::terminal(0))],
+                }],
+            }],
+            ..Default::default()
+        };
+
+        if let Err(error) = process_inlines(&grammar, &lexical_grammar) {
+            assert_eq!(error.to_string(), "Token `something` cannot be inlined");
+        } else {
+            panic!("expected an error, but got none");
+        }
+    }
+}

cli/generate/src/prepare_grammar/unicode-category-aliases.json

@@ -0,0 +1 @@
+{"Other":"C","Control":"Cc","cntrl":"Cc","Format":"Cf","Unassigned":"Cn","Private_Use":"Co","Surrogate":"Cs","Letter":"L","Cased_Letter":"LC","Lowercase_Letter":"Ll","Modifier_Letter":"Lm","Other_Letter":"Lo","Titlecase_Letter":"Lt","Uppercase_Letter":"Lu","Mark":"M","Combining_Mark":"M","Spacing_Mark":"Mc","Enclosing_Mark":"Me","Nonspacing_Mark":"Mn","Number":"N","Decimal_Number":"Nd","digit":"Nd","Letter_Number":"Nl","Other_Number":"No","Punctuation":"P","punct":"P","Connector_Punctuation":"Pc","Dash_Punctuation":"Pd","Close_Punctuation":"Pe","Final_Punctuation":"Pf","Initial_Punctuation":"Pi","Other_Punctuation":"Po","Open_Punctuation":"Ps","Symbol":"S","Currency_Symbol":"Sc","Modifier_Symbol":"Sk","Math_Symbol":"Sm","Other_Symbol":"So","Separator":"Z","Line_Separator":"Zl","Paragraph_Separator":"Zp","Space_Separator":"Zs"}

cli/generate/src/prepare_grammar/unicode-property-aliases.json

@@ -0,0 +1 @@
+{"cjkAccountingNumeric":"kAccountingNumeric","cjkOtherNumeric":"kOtherNumeric","cjkPrimaryNumeric":"kPrimaryNumeric","nv":"Numeric_Value","bmg":"Bidi_Mirroring_Glyph","bpb":"Bidi_Paired_Bracket","cf":"Case_Folding","cjkCompatibilityVariant":"kCompatibilityVariant","dm":"Decomposition_Mapping","EqUIdeo":"Equivalent_Unified_Ideograph","FC_NFKC":"FC_NFKC_Closure","lc":"Lowercase_Mapping","NFKC_CF":"NFKC_Casefold","NFKC_SCF":"NFKC_Simple_Casefold","scf":"Simple_Case_Folding","sfc":"Simple_Case_Folding","slc":"Simple_Lowercase_Mapping","stc":"Simple_Titlecase_Mapping","suc":"Simple_Uppercase_Mapping","tc":"Titlecase_Mapping","uc":"Uppercase_Mapping","cjkIICore":"kIICore","cjkIRG_GSource":"kIRG_GSource","cjkIRG_HSource":"kIRG_HSource","cjkIRG_JSource":"kIRG_JSource","cjkIRG_KPSource":"kIRG_KPSource","cjkIRG_KSource":"kIRG_KSource","cjkIRG_MSource":"kIRG_MSource","cjkIRG_SSource":"kIRG_SSource","cjkIRG_TSource":"kIRG_TSource","cjkIRG_UKSource":"kIRG_UKSource","cjkIRG_USource":"kIRG_USource","cjkIRG_VSource":"kIRG_VSource","cjkRSUnicode":"kRSUnicode","Unicode_Radical_Stroke":"kRSUnicode","URS":"kRSUnicode","isc":"ISO_Comment","JSN":"Jamo_Short_Name","na":"Name","na1":"Unicode_1_Name","Name_Alias":"Name_Alias","scx":"Script_Extensions","age":"Age","blk":"Block","sc":"Script","bc":"Bidi_Class","bpt":"Bidi_Paired_Bracket_Type","ccc":"Canonical_Combining_Class","dt":"Decomposition_Type","ea":"East_Asian_Width","gc":"General_Category","GCB":"Grapheme_Cluster_Break","hst":"Hangul_Syllable_Type","InCB":"Indic_Conjunct_Break","InPC":"Indic_Positional_Category","InSC":"Indic_Syllabic_Category","jg":"Joining_Group","jt":"Joining_Type","lb":"Line_Break","NFC_QC":"NFC_Quick_Check","NFD_QC":"NFD_Quick_Check","NFKC_QC":"NFKC_Quick_Check","NFKD_QC":"NFKD_Quick_Check","nt":"Numeric_Type","SB":"Sentence_Break","vo":"Vertical_Orientation","WB":"Word_Break","AHex":"ASCII_Hex_Digit","Alpha":"Alphabetic","Bidi_C":"Bidi_Control","Bidi_M":"Bidi_Mirrored","Cased":"Cased","CE":"Composition_Exclusion","CI":"Case_Ignorable","Comp_Ex":"Full_Composition_Exclusion","CWCF":"Changes_When_Casefolded","CWCM":"Changes_When_Casemapped","CWKCF":"Changes_When_NFKC_Casefolded","CWL":"Changes_When_Lowercased","CWT":"Changes_When_Titlecased","CWU":"Changes_When_Uppercased","Dash":"Dash","Dep":"Deprecated","DI":"Default_Ignorable_Code_Point","Dia":"Diacritic","EBase":"Emoji_Modifier_Base","EComp":"Emoji_Component","EMod":"Emoji_Modifier","Emoji":"Emoji","EPres":"Emoji_Presentation","Ext":"Extender","ExtPict":"Extended_Pictographic","Gr_Base":"Grapheme_Base","Gr_Ext":"Grapheme_Extend","Gr_Link":"Grapheme_Link","Hex":"Hex_Digit","Hyphen":"Hyphen","ID_Compat_Math_Continue":"ID_Compat_Math_Continue","ID_Compat_Math_Start":"ID_Compat_Math_Start","IDC":"ID_Continue","Ideo":"Ideographic","IDS":"ID_Start","IDSB":"IDS_Binary_Operator","IDST":"IDS_Trinary_Operator","IDSU":"IDS_Unary_Operator","Join_C":"Join_Control","LOE":"Logical_Order_Exception","Lower":"Lowercase","Math":"Math","NChar":"Noncharacter_Code_Point","OAlpha":"Other_Alphabetic","ODI":"Other_Default_Ignorable_Code_Point","OGr_Ext":"Other_Grapheme_Extend","OIDC":"Other_ID_Continue","OIDS":"Other_ID_Start","OLower":"Other_Lowercase","OMath":"Other_Math","OUpper":"Other_Uppercase","Pat_Syn":"Pattern_Syntax","Pat_WS":"Pattern_White_Space","PCM":"Prepended_Concatenation_Mark","QMark":"Quotation_Mark","Radical":"Radical","RI":"Regional_Indicator","SD":"Soft_Dotted","STerm":"Sentence_Terminal","Term":"Terminal_Punctuation","UIdeo":"Unified_Ideograph","Upper":"Uppercase","VS":"Variation_Selector","WSpace":"White_Space","space":"White_Space","XIDC":"XID_Continue","XIDS":"XID_Start","XO_NFC":"Expands_On_NFC","XO_NFD":"Expands_On_NFD","XO_NFKC":"Expands_On_NFKC","XO_NFKD":"Expands_On_NFKD"}

cli/generate/src/rules.rs

@@ -0,0 +1,493 @@
+use std::{collections::HashMap, fmt};
+
+use smallbitvec::SmallBitVec;
+
+use super::grammars::VariableType;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum SymbolType {
+    External,
+    End,
+    EndOfNonTerminalExtra,
+    Terminal,
+    NonTerminal,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum Associativity {
+    Left,
+    Right,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct Alias {
+    pub value: String,
+    pub is_named: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default)]
+pub enum Precedence {
+    #[default]
+    None,
+    Integer(i32),
+    Name(String),
+}
+
+pub type AliasMap = HashMap<Symbol, Alias>;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+pub struct MetadataParams {
+    pub precedence: Precedence,
+    pub dynamic_precedence: i32,
+    pub associativity: Option<Associativity>,
+    pub is_token: bool,
+    pub is_string: bool,
+    pub is_active: bool,
+    pub is_main_token: bool,
+    pub alias: Option<Alias>,
+    pub field_name: Option<String>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct Symbol {
+    pub kind: SymbolType,
+    pub index: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum Rule {
+    Blank,
+    String(String),
+    Pattern(String, String),
+    NamedSymbol(String),
+    Symbol(Symbol),
+    Choice(Vec<Rule>),
+    Metadata {
+        params: MetadataParams,
+        rule: Box<Rule>,
+    },
+    Repeat(Box<Rule>),
+    Seq(Vec<Rule>),
+}
+
+// Because tokens are represented as small (~400 max) unsigned integers,
+// sets of tokens can be efficiently represented as bit vectors with each
+// index corresponding to a token, and each value representing whether or not
+// the token is present in the set.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct TokenSet {
+    terminal_bits: SmallBitVec,
+    external_bits: SmallBitVec,
+    eof: bool,
+    end_of_nonterminal_extra: bool,
+}
+
+impl Rule {
+    pub fn field(name: String, content: Self) -> Self {
+        add_metadata(content, move |params| {
+            params.field_name = Some(name);
+        })
+    }
+
+    pub fn alias(content: Self, value: String, is_named: bool) -> Self {
+        add_metadata(content, move |params| {
+            params.alias = Some(Alias { value, is_named });
+        })
+    }
+
+    pub fn token(content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.is_token = true;
+        })
+    }
+
+    pub fn immediate_token(content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.is_token = true;
+            params.is_main_token = true;
+        })
+    }
+
+    pub fn prec(value: Precedence, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.precedence = value;
+        })
+    }
+
+    pub fn prec_left(value: Precedence, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.associativity = Some(Associativity::Left);
+            params.precedence = value;
+        })
+    }
+
+    pub fn prec_right(value: Precedence, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.associativity = Some(Associativity::Right);
+            params.precedence = value;
+        })
+    }
+
+    pub fn prec_dynamic(value: i32, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.dynamic_precedence = value;
+        })
+    }
+
+    pub fn repeat(rule: Self) -> Self {
+        Self::Repeat(Box::new(rule))
+    }
+
+    pub fn choice(rules: Vec<Self>) -> Self {
+        let mut elements = Vec::with_capacity(rules.len());
+        for rule in rules {
+            choice_helper(&mut elements, rule);
+        }
+        Self::Choice(elements)
+    }
+
+    pub const fn seq(rules: Vec<Self>) -> Self {
+        Self::Seq(rules)
+    }
+}
+
+impl Alias {
+    #[must_use]
+    pub const fn kind(&self) -> VariableType {
+        if self.is_named {
+            VariableType::Named
+        } else {
+            VariableType::Anonymous
+        }
+    }
+}
+
+impl Precedence {
+    #[must_use]
+    pub const fn is_none(&self) -> bool {
+        matches!(self, Self::None)
+    }
+}
+
+#[cfg(test)]
+impl Rule {
+    #[must_use]
+    pub const fn terminal(index: usize) -> Self {
+        Self::Symbol(Symbol::terminal(index))
+    }
+
+    #[must_use]
+    pub const fn non_terminal(index: usize) -> Self {
+        Self::Symbol(Symbol::non_terminal(index))
+    }
+
+    #[must_use]
+    pub const fn external(index: usize) -> Self {
+        Self::Symbol(Symbol::external(index))
+    }
+
+    #[must_use]
+    pub fn named(name: &'static str) -> Self {
+        Self::NamedSymbol(name.to_string())
+    }
+
+    #[must_use]
+    pub fn string(value: &'static str) -> Self {
+        Self::String(value.to_string())
+    }
+
+    #[must_use]
+    pub fn pattern(value: &'static str, flags: &'static str) -> Self {
+        Self::Pattern(value.to_string(), flags.to_string())
+    }
+}
+
+impl Symbol {
+    #[must_use]
+    pub fn is_terminal(&self) -> bool {
+        self.kind == SymbolType::Terminal
+    }
+
+    #[must_use]
+    pub fn is_non_terminal(&self) -> bool {
+        self.kind == SymbolType::NonTerminal
+    }
+
+    #[must_use]
+    pub fn is_external(&self) -> bool {
+        self.kind == SymbolType::External
+    }
+
+    #[must_use]
+    pub fn is_eof(&self) -> bool {
+        self.kind == SymbolType::End
+    }
+
+    #[must_use]
+    pub const fn non_terminal(index: usize) -> Self {
+        Self {
+            kind: SymbolType::NonTerminal,
+            index,
+        }
+    }
+
+    #[must_use]
+    pub const fn terminal(index: usize) -> Self {
+        Self {
+            kind: SymbolType::Terminal,
+            index,
+        }
+    }
+
+    #[must_use]
+    pub const fn external(index: usize) -> Self {
+        Self {
+            kind: SymbolType::External,
+            index,
+        }
+    }
+
+    #[must_use]
+    pub const fn end() -> Self {
+        Self {
+            kind: SymbolType::End,
+            index: 0,
+        }
+    }
+
+    #[must_use]
+    pub const fn end_of_nonterminal_extra() -> Self {
+        Self {
+            kind: SymbolType::EndOfNonTerminalExtra,
+            index: 0,
+        }
+    }
+}
+
+impl From<Symbol> for Rule {
+    #[must_use]
+    fn from(symbol: Symbol) -> Self {
+        Self::Symbol(symbol)
+    }
+}
+
+impl TokenSet {
+    #[must_use]
+    pub const fn new() -> Self {
+        Self {
+            terminal_bits: SmallBitVec::new(),
+            external_bits: SmallBitVec::new(),
+            eof: false,
+            end_of_nonterminal_extra: false,
+        }
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = Symbol> + '_ {
+        self.terminal_bits
+            .iter()
+            .enumerate()
+            .filter_map(|(i, value)| {
+                if value {
+                    Some(Symbol::terminal(i))
+                } else {
+                    None
+                }
+            })
+            .chain(
+                self.external_bits
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(i, value)| {
+                        if value {
+                            Some(Symbol::external(i))
+                        } else {
+                            None
+                        }
+                    }),
+            )
+            .chain(if self.eof { Some(Symbol::end()) } else { None })
+            .chain(if self.end_of_nonterminal_extra {
+                Some(Symbol::end_of_nonterminal_extra())
+            } else {
+                None
+            })
+    }
+
+    pub fn terminals(&self) -> impl Iterator<Item = Symbol> + '_ {
+        self.terminal_bits
+            .iter()
+            .enumerate()
+            .filter_map(|(i, value)| {
+                if value {
+                    Some(Symbol::terminal(i))
+                } else {
+                    None
+                }
+            })
+    }
+
+    pub fn contains(&self, symbol: &Symbol) -> bool {
+        match symbol.kind {
+            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
+            SymbolType::Terminal => self.terminal_bits.get(symbol.index).unwrap_or(false),
+            SymbolType::External => self.external_bits.get(symbol.index).unwrap_or(false),
+            SymbolType::End => self.eof,
+            SymbolType::EndOfNonTerminalExtra => self.end_of_nonterminal_extra,
+        }
+    }
+
+    pub fn contains_terminal(&self, index: usize) -> bool {
+        self.terminal_bits.get(index).unwrap_or(false)
+    }
+
+    pub fn insert(&mut self, other: Symbol) {
+        let vec = match other.kind {
+            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
+            SymbolType::Terminal => &mut self.terminal_bits,
+            SymbolType::External => &mut self.external_bits,
+            SymbolType::End => {
+                self.eof = true;
+                return;
+            }
+            SymbolType::EndOfNonTerminalExtra => {
+                self.end_of_nonterminal_extra = true;
+                return;
+            }
+        };
+        if other.index >= vec.len() {
+            vec.resize(other.index + 1, false);
+        }
+        vec.set(other.index, true);
+    }
+
+    pub fn remove(&mut self, other: &Symbol) -> bool {
+        let vec = match other.kind {
+            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
+            SymbolType::Terminal => &mut self.terminal_bits,
+            SymbolType::External => &mut self.external_bits,
+            SymbolType::End => {
+                return if self.eof {
+                    self.eof = false;
+                    true
+                } else {
+                    false
+                }
+            }
+            SymbolType::EndOfNonTerminalExtra => {
+                return if self.end_of_nonterminal_extra {
+                    self.end_of_nonterminal_extra = false;
+                    true
+                } else {
+                    false
+                };
+            }
+        };
+        if other.index < vec.len() && vec[other.index] {
+            vec.set(other.index, false);
+            return true;
+        }
+        false
+    }
+
+    pub fn is_empty(&self) -> bool {
+        !self.eof
+            && !self.end_of_nonterminal_extra
+            && !self.terminal_bits.iter().any(|a| a)
+            && !self.external_bits.iter().any(|a| a)
+    }
+
+    pub fn insert_all_terminals(&mut self, other: &Self) -> bool {
+        let mut result = false;
+        if other.terminal_bits.len() > self.terminal_bits.len() {
+            self.terminal_bits.resize(other.terminal_bits.len(), false);
+        }
+        for (i, element) in other.terminal_bits.iter().enumerate() {
+            if element {
+                result |= !self.terminal_bits[i];
+                self.terminal_bits.set(i, element);
+            }
+        }
+        result
+    }
+
+    fn insert_all_externals(&mut self, other: &Self) -> bool {
+        let mut result = false;
+        if other.external_bits.len() > self.external_bits.len() {
+            self.external_bits.resize(other.external_bits.len(), false);
+        }
+        for (i, element) in other.external_bits.iter().enumerate() {
+            if element {
+                result |= !self.external_bits[i];
+                self.external_bits.set(i, element);
+            }
+        }
+        result
+    }
+
+    pub fn insert_all(&mut self, other: &Self) -> bool {
+        let mut result = false;
+        if other.eof {
+            result |= !self.eof;
+            self.eof = true;
+        }
+        if other.end_of_nonterminal_extra {
+            result |= !self.end_of_nonterminal_extra;
+            self.end_of_nonterminal_extra = true;
+        }
+        result |= self.insert_all_terminals(other);
+        result |= self.insert_all_externals(other);
+        result
+    }
+}
+
+impl FromIterator<Symbol> for TokenSet {
+    fn from_iter<T: IntoIterator<Item = Symbol>>(iter: T) -> Self {
+        let mut result = Self::new();
+        for symbol in iter {
+            result.insert(symbol);
+        }
+        result
+    }
+}
+
+fn add_metadata<T: FnOnce(&mut MetadataParams)>(input: Rule, f: T) -> Rule {
+    match input {
+        Rule::Metadata { rule, mut params } if !params.is_token => {
+            f(&mut params);
+            Rule::Metadata { rule, params }
+        }
+        _ => {
+            let mut params = MetadataParams::default();
+            f(&mut params);
+            Rule::Metadata {
+                rule: Box::new(input),
+                params,
+            }
+        }
+    }
+}
+
+fn choice_helper(result: &mut Vec<Rule>, rule: Rule) {
+    match rule {
+        Rule::Choice(elements) => {
+            for element in elements {
+                choice_helper(result, element);
+            }
+        }
+        _ => {
+            if !result.contains(&rule) {
+                result.push(rule);
+            }
+        }
+    }
+}
+
+impl fmt::Display for Precedence {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Integer(i) => write!(f, "{i}"),
+            Self::Name(s) => write!(f, "'{s}'"),
+            Self::None => write!(f, "none"),
+        }
+    }
+}

cli/generate/src/tables.rs

@@ -0,0 +1,166 @@
+use std::collections::BTreeMap;
+
+use super::{
+    nfa::CharacterSet,
+    rules::{Alias, Symbol, TokenSet},
+};
+pub type ProductionInfoId = usize;
+pub type ParseStateId = usize;
+pub type LexStateId = usize;
+
+use std::hash::BuildHasherDefault;
+
+use indexmap::IndexMap;
+use rustc_hash::FxHasher;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum ParseAction {
+    Accept,
+    Shift {
+        state: ParseStateId,
+        is_repetition: bool,
+    },
+    ShiftExtra,
+    Recover,
+    Reduce {
+        symbol: Symbol,
+        child_count: usize,
+        dynamic_precedence: i32,
+        production_id: ProductionInfoId,
+    },
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum GotoAction {
+    Goto(ParseStateId),
+    ShiftExtra,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ParseTableEntry {
+    pub actions: Vec<ParseAction>,
+    pub reusable: bool,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ParseState {
+    pub id: ParseStateId,
+    pub terminal_entries: IndexMap<Symbol, ParseTableEntry, BuildHasherDefault<FxHasher>>,
+    pub nonterminal_entries: IndexMap<Symbol, GotoAction, BuildHasherDefault<FxHasher>>,
+    pub lex_state_id: usize,
+    pub external_lex_state_id: usize,
+    pub core_id: usize,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct FieldLocation {
+    pub index: usize,
+    pub inherited: bool,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct ProductionInfo {
+    pub alias_sequence: Vec<Option<Alias>>,
+    pub field_map: BTreeMap<String, Vec<FieldLocation>>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct ParseTable {
+    pub states: Vec<ParseState>,
+    pub symbols: Vec<Symbol>,
+    pub production_infos: Vec<ProductionInfo>,
+    pub max_aliased_production_length: usize,
+    pub external_lex_states: Vec<TokenSet>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct AdvanceAction {
+    pub state: LexStateId,
+    pub in_main_token: bool,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
+pub struct LexState {
+    pub accept_action: Option<Symbol>,
+    pub eof_action: Option<AdvanceAction>,
+    pub advance_actions: Vec<(CharacterSet, AdvanceAction)>,
+}
+
+#[derive(Debug, PartialEq, Eq, Default)]
+pub struct LexTable {
+    pub states: Vec<LexState>,
+}
+
+impl ParseTableEntry {
+    #[must_use]
+    pub const fn new() -> Self {
+        Self {
+            reusable: true,
+            actions: Vec::new(),
+        }
+    }
+}
+
+impl ParseState {
+    pub fn is_end_of_non_terminal_extra(&self) -> bool {
+        self.terminal_entries
+            .contains_key(&Symbol::end_of_nonterminal_extra())
+    }
+
+    pub fn referenced_states(&self) -> impl Iterator<Item = ParseStateId> + '_ {
+        self.terminal_entries
+            .iter()
+            .flat_map(|(_, entry)| {
+                entry.actions.iter().filter_map(|action| match action {
+                    ParseAction::Shift { state, .. } => Some(*state),
+                    _ => None,
+                })
+            })
+            .chain(self.nonterminal_entries.iter().filter_map(|(_, action)| {
+                if let GotoAction::Goto(state) = action {
+                    Some(*state)
+                } else {
+                    None
+                }
+            }))
+    }
+
+    pub fn update_referenced_states<F>(&mut self, mut f: F)
+    where
+        F: FnMut(usize, &Self) -> usize,
+    {
+        let mut updates = Vec::new();
+        for (symbol, entry) in &self.terminal_entries {
+            for (i, action) in entry.actions.iter().enumerate() {
+                if let ParseAction::Shift { state, .. } = action {
+                    let result = f(*state, self);
+                    if result != *state {
+                        updates.push((*symbol, i, result));
+                    }
+                }
+            }
+        }
+        for (symbol, action) in &self.nonterminal_entries {
+            if let GotoAction::Goto(other_state) = action {
+                let result = f(*other_state, self);
+                if result != *other_state {
+                    updates.push((*symbol, 0, result));
+                }
+            }
+        }
+        for (symbol, action_index, new_state) in updates {
+            if symbol.is_non_terminal() {
+                self.nonterminal_entries
+                    .insert(symbol, GotoAction::Goto(new_state));
+            } else {
+                let entry = self.terminal_entries.get_mut(&symbol).unwrap();
+                if let ParseAction::Shift { is_repetition, .. } = entry.actions[action_index] {
+                    entry.actions[action_index] = ParseAction::Shift {
+                        state: new_state,
+                        is_repetition,
+                    };
+                }
+            }
+        }
+    }
+}

cli/generate/src/templates/array.h

@@ -14,7 +14,6 @@ extern "C" {
 #include <string.h>
 
 #ifdef _MSC_VER
-#pragma warning(push)
 #pragma warning(disable : 4101)
 #elif defined(__GNUC__) || defined(__clang__)
 #pragma GCC diagnostic push
@@ -50,104 +49,69 @@ extern "C" {
 /// memory allocated for the array's contents.
 #define array_clear(self) ((self)->size = 0)
 
-#ifdef __cplusplus
-#define _array__cast(self, expr) (decltype((self)->contents))(expr)
-#else
-#define _array__cast(self, expr) (expr)
-#endif
-
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity)                 \
-  ((self)->contents = _array__cast(self, _array__reserve( \
-    (void *)(self)->contents, &(self)->capacity,          \
-    array_elem_size(self), new_capacity))                 \
-  )
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self)                           \
-  do {                                               \
-    if ((self)->contents) ts_free((self)->contents); \
-    (self)->contents = NULL;                         \
-    (self)->size = 0;                                \
-    (self)->capacity = 0;                            \
-  } while (0)
+#define array_delete(self) _array__delete((Array *)(self))
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                                 \
-  do {                                                            \
-    (self)->contents = _array__cast(self, _array__grow(           \
-      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
-      1, array_elem_size(self)                                    \
-    ));                                                           \
-   (self)->contents[(self)->size++] = (element);                  \
-  } while(0)
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count)                                               \
-  do {                                                                           \
-    if ((count) == 0) break;                                                     \
-    (self)->contents = _array__cast(self, _array__grow(                          \
-      (self)->contents, (self)->size, &(self)->capacity,                         \
-      count, array_elem_size(self)                                               \
-    ));                                                                          \
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count);                                                     \
+    (self)->size += (count); \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other) \
+#define array_push_all(self, other)                                       \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, other_contents)                 \
-  ((self)->contents = _array__cast(self, _array__splice(          \
-    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
-    array_elem_size(self), (self)->size, 0, count, other_contents \
-  )))
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents) \
-  ((self)->contents = _array__cast(self, _array__splice(              \
-    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
-    array_elem_size(self), _index, old_count, new_count, new_contents  \
-  )))
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element)                     \
-  ((self)->contents = _array__cast(self, _array__splice(        \
-    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
-    array_elem_size(self), _index, 0, 1, &(element)             \
-  )))
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
+  _array__erase((Array *)(self), array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other)                                   \
-  ((self)->contents = _array__cast(self, _array__assign(            \
-    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
-    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
-  )))
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
 
 /// Swap one array with another
-#define array_swap(self, other)                                     \
-  do {                                                              \
-    void *_array_swap_tmp = (void *)(self)->contents;               \
-    (self)->contents = (other)->contents;                           \
-    (other)->contents = _array__cast(other, _array_swap_tmp);       \
-    _array__swap(&(self)->size, &(self)->capacity,                  \
-                 &(other)->size, &(other)->capacity);               \
-  } while (0)
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -192,90 +156,82 @@ extern "C" {
 
 // Private
 
-// Pointers to individual `Array` fields (rather than the entire `Array` itself)
-// are passed to the various `_array__*` functions below to address strict aliasing
-// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
-//
-// The `Array` type itself was not altered as a solution in order to avoid breakage
-// with existing consumers (in particular, parsers with external scanners).
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(void* self_contents, uint32_t *size,
-                                size_t element_size, uint32_t index) {
-  assert(index < *size);
-  char *contents = (char *)self_contents;
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (*size - index - 1) * element_size);
-  (*size)--;
+          (self->size - index - 1) * element_size);
+  self->size--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void *_array__reserve(void *contents, uint32_t *capacity,
-                                  size_t element_size, uint32_t new_capacity) {
-  void *new_contents = contents;
-  if (new_capacity > *capacity) {
-    if (contents) {
-      new_contents = ts_realloc(contents, new_capacity * element_size);
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
     } else {
-      new_contents = ts_malloc(new_capacity * element_size);
+      self->contents = ts_malloc(new_capacity * element_size);
     }
-    *capacity = new_capacity;
+    self->capacity = new_capacity;
   }
-  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
-                                 const void *other_contents, uint32_t other_size, size_t element_size) {
-  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
-  *self_size = other_size;
-  memcpy(new_contents, other_contents, *self_size * element_size);
-  return new_contents;
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
 }
 
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
-                               uint32_t *other_size, uint32_t *other_capacity) {
-  uint32_t tmp_size = *self_size;
-  uint32_t tmp_capacity = *self_capacity;
-  *self_size = *other_size;
-  *self_capacity = *other_capacity;
-  *other_size = tmp_size;
-  *other_capacity = tmp_capacity;
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
-                               uint32_t count, size_t element_size) {
-  void *new_contents = contents;
-  uint32_t new_size = size + count;
-  if (new_size > *capacity) {
-    uint32_t new_capacity = *capacity * 2;
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
+    _array__reserve(self, element_size, new_capacity);
   }
-  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
-                                 size_t element_size,
+static inline void _array__splice(Array *self, size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = *size + new_count - old_count;
+  uint32_t new_size = self->size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  assert(old_end <= *size);
+  assert(old_end <= self->size);
 
-  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
+  _array__reserve(self, element_size, new_size);
 
-  char *contents = (char *)new_contents;
-  if (*size > old_end) {
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (*size - old_end) * element_size
+      (self->size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -293,9 +249,7 @@ static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t
       );
     }
   }
-  *size += new_count - old_count;
-
-  return new_contents;
+  self->size += new_count - old_count;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.
@@ -324,7 +278,7 @@ static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t
 #define _compare_int(a, b) ((int)*(a) - (int)(b))
 
 #ifdef _MSC_VER
-#pragma warning(pop)
+#pragma warning(default : 4101)
 #elif defined(__GNUC__) || defined(__clang__)
 #pragma GCC diagnostic pop
 #endif

cli/loader/Cargo.toml

@@ -5,40 +5,36 @@ description = "Locates, builds, and loads tree-sitter grammars at runtime"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
+readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
-documentation = "https://docs.rs/tree-sitter-loader"
 license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 
-[package.metadata.docs.rs]
-all-features = true
-rustdoc-args = [ "--cfg", "docsrs" ]
-
-[lib]
-path = "src/loader.rs"
-
-[lints]
-workspace = true
-
 [features]
-default = [ "tree-sitter-highlight", "tree-sitter-tags" ]
-wasm = [ "tree-sitter/wasm" ]
+wasm = ["tree-sitter/wasm"]
+# TODO: For backward compatibility these must be enabled by default,
+# consider removing for the next semver incompatible release
+default = ["tree-sitter-highlight", "tree-sitter-tags"]
 
 [dependencies]
+anyhow.workspace = true
 cc.workspace = true
-etcetera.workspace = true
+dirs.workspace = true
+fs4.workspace = true
+indoc.workspace = true
+lazy_static.workspace = true
 libloading.workspace = true
-log.workspace = true
 once_cell.workspace = true
+path-slash.workspace = true
 regex.workspace = true
 semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tempfile.workspace = true
-thiserror.workspace = true
+url.workspace = true
 
 tree-sitter = { workspace = true }
-tree-sitter-highlight = { optional = true, workspace = true }
-tree-sitter-tags = { optional = true, workspace = true }
+tree-sitter-highlight = { workspace = true, optional = true }
+tree-sitter-tags = { workspace = true, optional = true }

cli/loader/build.rs

@@ -7,4 +7,7 @@ fn main() {
         "cargo:rustc-env=BUILD_HOST={}",
         std::env::var("HOST").unwrap()
     );
+
+    let emscripten_version = std::fs::read_to_string("emscripten-version").unwrap();
+    println!("cargo:rustc-env=EMSCRIPTEN_VERSION={emscripten_version}");
 }

cli/loader/emscripten-version

@@ -0,0 +1 @@
+3.1.64

cli/npm/dsl.d.ts

@@ -3,22 +3,17 @@ type BlankRule = { type: 'BLANK' };
 type ChoiceRule = { type: 'CHOICE'; members: Rule[] };
 type FieldRule = { type: 'FIELD'; name: string; content: Rule };
 type ImmediateTokenRule = { type: 'IMMEDIATE_TOKEN'; content: Rule };
-type PatternRule = { type: 'PATTERN'; value: string; flags?: string };
-type PrecedenceValue = string | number;
+type PatternRule = { type: 'PATTERN'; value: string };
 type PrecDynamicRule = { type: 'PREC_DYNAMIC'; content: Rule; value: number };
-type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: PrecedenceValue };
-type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: PrecedenceValue };
-type PrecRule = { type: 'PREC'; content: Rule; value: PrecedenceValue };
+type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: number };
+type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number };
+type PrecRule = { type: 'PREC'; content: Rule; value: number };
 type Repeat1Rule = { type: 'REPEAT1'; content: Rule };
 type RepeatRule = { type: 'REPEAT'; content: Rule };
-type ReservedRule = { type: 'RESERVED'; content: Rule; context_name: string };
 type SeqRule = { type: 'SEQ'; members: Rule[] };
 type StringRule = { type: 'STRING'; value: string };
 type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
-type PrecedenceEntry = StringRule | SymbolRule<string>;
 type TokenRule = { type: 'TOKEN'; content: Rule };
-type EOFRule = { type: 'EOF' };
-
 
 type Rule =
   | AliasRule
@@ -33,20 +28,12 @@ type Rule =
   | PrecRule
   | Repeat1Rule
   | RepeatRule
-  | ReservedRule
   | SeqRule
   | StringRule
   | SymbolRule<string>
-  | TokenRule
-  | EOFRule;
+  | TokenRule;
 
-declare class RustRegex {
-  value: string;
-
-  constructor(pattern: string);
-}
-
-type RuleOrLiteral = Rule | RegExp | RustRegex | string;
+type RuleOrLiteral = Rule | RegExp | string;
 
 type GrammarSymbols<RuleName extends string> = {
   [name in RuleName]: SymbolRule<name>;
@@ -55,7 +42,7 @@ type GrammarSymbols<RuleName extends string> = {
 
 type RuleBuilder<RuleName extends string> = (
   $: GrammarSymbols<RuleName>,
-  previous?: Rule,
+  previous: Rule,
 ) => RuleOrLiteral;
 
 type RuleBuilders<
@@ -91,8 +78,8 @@ interface Grammar<
    */
   precedences?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: PrecedenceEntry[][],
-  ) => (string | PrecedenceEntry)[][],
+    previous: Rule[][],
+  ) => RuleOrLiteral[][],
 
   /**
    * An array of arrays of rule names. Each inner array represents a set of
@@ -106,8 +93,8 @@ interface Grammar<
    */
   conflicts?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: SymbolRule<string>[][],
-  ) => SymbolRule<string>[][];
+    previous: Rule[][],
+  ) => RuleOrLiteral[][];
 
   /**
    * An array of token names which can be returned by an _external scanner_.
@@ -118,7 +105,7 @@ interface Grammar<
    * @param $ grammar rules
    * @param previous array of externals from the base schema, if any
    *
-   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners
+   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners
    */
   externals?: (
     $: Record<string, SymbolRule<string>>,
@@ -132,11 +119,9 @@ interface Grammar<
    * specify extras: `$ => []` in your grammar.
    *
    *  @param $ grammar rules
-   *  @param previous array of extras from the base grammar
    */
   extras?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[],
   ) => RuleOrLiteral[];
 
   /**
@@ -149,8 +134,8 @@ interface Grammar<
    */
   inline?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: SymbolRule<string>[],
-  ) => SymbolRule<string>[];
+    previous: Rule[],
+  ) => RuleOrLiteral[];
 
   /**
    * A list of hidden rule names that should be considered supertypes in the
@@ -158,12 +143,12 @@ interface Grammar<
    *
    * @param $ grammar rules
    *
-   * @see https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
+   * @see https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
    */
   supertypes?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: SymbolRule<string>[],
-  ) => SymbolRule<string>[];
+    previous: Rule[],
+  ) => RuleOrLiteral[];
 
   /**
    * The name of a token that will match keywords for the purpose of the
@@ -171,49 +156,15 @@ interface Grammar<
    *
    * @param $ grammar rules
    *
-   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar#keyword-extraction
+   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#keyword-extraction
    */
-  word?: (
-    $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-  ) => SymbolRule<string>;
-
-
-  /**
-   * Mapping of names to reserved word sets. The first reserved word set is the
-   * global word set, meaning it applies to every rule in every parse state.
-   * The other word sets can be used with the `reserved` function. Each callback
-   * receives the base grammar's reserved word set of the same name as its second
-   * argument, or `undefined` if no matching set exists.
-   */
-  reserved?: Record<
-    string,
-    (
-      $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-      previous: Rule[] | undefined,
-    ) => RuleOrLiteral[]
-  >;
+  word?: ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral;
 }
 
-/**
- * Return type of grammar(). The runtime evaluates and normalizes the grammar
- * beneath a "grammar" key. Optional input fields become required output fields
- * with default values when not provided.
- */
 type GrammarSchema<RuleName extends string> = {
-  grammar: {
-    name: string;
-    /** Base grammar name when extending; undefined for root grammars. */
-    inherits: string | undefined;
-    rules: Record<RuleName, Rule>;
-    precedences: PrecedenceEntry[][];
-    conflicts: string[][];
-    externals: Rule[];
-    extras: Rule[];
-    inline: string[];
-    supertypes: string[];
-    word: string | undefined;
-    reserved: Record<string, Rule[]>;
-  };
+  [K in keyof Grammar<RuleName>]: K extends 'rules'
+  ? Record<RuleName, Rule>
+  : Grammar<RuleName>[K];
 };
 
 /**
@@ -292,7 +243,7 @@ declare function optional(rule: RuleOrLiteral): ChoiceRule;
  * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
  */
 declare const prec: {
-  (value: string | number, rule: RuleOrLiteral): PrecRule;
+  (value: String | number, rule: RuleOrLiteral): PrecRule;
 
   /**
    * Marks the given rule as left-associative (and optionally applies a
@@ -308,7 +259,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   left(rule: RuleOrLiteral): PrecLeftRule;
-  left(value: string | number, rule: RuleOrLiteral): PrecLeftRule;
+  left(value: String | number, rule: RuleOrLiteral): PrecLeftRule;
 
   /**
    * Marks the given rule as right-associative (and optionally applies a
@@ -324,7 +275,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   right(rule: RuleOrLiteral): PrecRightRule;
-  right(value: string | number, rule: RuleOrLiteral): PrecRightRule;
+  right(value: String | number, rule: RuleOrLiteral): PrecRightRule;
 
   /**
    * Marks the given rule with a numerical precedence which will be used to
@@ -341,7 +292,7 @@ declare const prec: {
    *
    * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
    */
-  dynamic(value: number, rule: RuleOrLiteral): PrecDynamicRule;
+  dynamic(value: String | number, rule: RuleOrLiteral): PrecDynamicRule;
 };
 
 /**
@@ -361,15 +312,6 @@ declare function repeat(rule: RuleOrLiteral): RepeatRule;
  */
 declare function repeat1(rule: RuleOrLiteral): Repeat1Rule;
 
-/**
- * Overrides the global reserved word set for a given rule. The word set name
- * should be defined in the `reserved` field in the grammar.
- *
- * @param wordset name of the reserved word set
- * @param rule rule that will use the reserved word set
- */
-declare function reserved(wordset: string, rule: RuleOrLiteral): ReservedRule;
-
 /**
  * Creates a rule that matches any number of other rules, one after another.
  * It is analogous to simply writing multiple symbols next to each other
@@ -388,7 +330,7 @@ declare function sym<Name extends string>(name: Name): SymbolRule<Name>;
 
 /**
  * Marks the given rule as producing only a single token. Tree-sitter's
- * default is to treat each string or RegExp literal in the grammar as a
+ * default is to treat each String or RegExp literal in the grammar as a
  * separate token. Each token is matched separately by the lexer and
  * returned as its own leaf node in the tree. The token function allows
  * you to express a complex rule using the DSL functions (rather
@@ -412,19 +354,6 @@ declare const token: {
   immediate(rule: RuleOrLiteral): ImmediateTokenRule;
 };
 
-/**
- * Matches the end of input. May only appear as the final symbol of a
- * (possibly nested) sequence; a production ending in `eof()` reduces only
- * when the lookahead is end-of-input, rather than shifting a token.
- *
- * Choice branches that continue past `eof()` are dropped as unreachable,
- * and `eof()` is not allowed inside `token()`.
- *
- * Useful when a rule should match either an explicit terminator (e.g. a
- * newline) or the end of the file.
- */
-declare function eof(): EOFRule;
-
 /**
  * Creates a new language grammar with the provided schema.
  *

cli/npm/install.js

@@ -6,8 +6,7 @@ const http = require('http');
 const https = require('https');
 const packageJSON = require('./package.json');
 
-https.globalAgent.keepAlive = false;
-
+// Look to a results table in https://github.com/tree-sitter/tree-sitter/issues/2196
 const matrix = {
   platform: {
     'darwin': {

cli/npm/package.json

@@ -1,33 +1,24 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.28.0",
-  "author": {
-    "name": "Max Brunsfeld",
-    "email": "maxbrunsfeld@gmail.com"
-  },
-  "maintainers": [
-    {
-      "name": "Amaan Qureshi",
-      "email": "amaanq12@gmail.com"
-    }
-  ],
+  "version": "0.24.4",
+  "author": "Max Brunsfeld",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
-    "directory": "crates/cli/npm"
+    "url": "https://github.com/tree-sitter/tree-sitter.git"
   },
   "description": "CLI for generating fast incremental parsers",
   "keywords": [
     "parser",
     "lexer"
   ],
+  "main": "lib/api/index.js",
   "engines": {
     "node": ">=12.0.0"
   },
   "scripts": {
     "install": "node install.js",
-    "prepack": "cp ../../../LICENSE ../README.md .",
+    "prepack": "cp ../../LICENSE ../README.md .",
     "postpack": "rm LICENSE README.md"
   },
   "bin": {

cli/src/fuzz/allocations.rs

@@ -2,21 +2,19 @@ use std::{
     collections::HashMap,
     os::raw::c_void,
     sync::{
-        Mutex,
         atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},
+        Mutex,
     },
 };
 
 #[ctor::ctor]
 unsafe fn initialize_allocation_recording() {
-    unsafe {
-        tree_sitter::set_allocator(Some(tree_sitter::Allocator {
-            malloc: ts_record_malloc,
-            calloc: ts_record_calloc,
-            realloc: ts_record_realloc,
-            free: ts_record_free,
-        }));
-    }
+    tree_sitter::set_allocator(
+        Some(ts_record_malloc),
+        Some(ts_record_calloc),
+        Some(ts_record_realloc),
+        Some(ts_record_free),
+    );
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -35,18 +33,14 @@ thread_local! {
     static RECORDER: AllocationRecorder = AllocationRecorder::default();
 }
 
-unsafe extern "C" {
+extern "C" {
     fn malloc(size: usize) -> *mut c_void;
     fn calloc(count: usize, size: usize) -> *mut c_void;
     fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
     fn free(ptr: *mut c_void);
 }
 
-pub fn record<T>(f: impl FnOnce() -> T) -> T {
-    record_checked(f).unwrap()
-}
-
-pub fn record_checked<T>(f: impl FnOnce() -> T) -> Result<T, String> {
+pub fn record<T>(f: impl FnOnce() -> T) -> Result<T, String> {
     RECORDER.with(|recorder| {
         recorder.enabled.store(true, SeqCst);
         recorder.allocation_count.store(0, SeqCst);
@@ -99,57 +93,30 @@ fn record_dealloc(ptr: *mut c_void) {
     });
 }
 
-/// # Safety
-///
-/// The caller must ensure that the returned pointer is eventually
-/// freed by calling `ts_record_free`.
-#[must_use]
-pub unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
-    unsafe {
-        let result = malloc(size);
+unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
+    let result = malloc(size);
+    record_alloc(result);
+    result
+}
+
+unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
+    let result = calloc(count, size);
+    record_alloc(result);
+    result
+}
+
+unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
+    let result = realloc(ptr, size);
+    if ptr.is_null() {
         record_alloc(result);
-        result
-    }
-}
-
-/// # Safety
-///
-/// The caller must ensure that the returned pointer is eventually
-/// freed by calling `ts_record_free`.
-#[must_use]
-pub unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
-    unsafe {
-        let result = calloc(count, size);
-        record_alloc(result);
-        result
-    }
-}
-
-/// # Safety
-///
-/// The caller must ensure that the returned pointer is eventually
-/// freed by calling `ts_record_free`.
-#[must_use]
-pub unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
-    unsafe {
-        let result = realloc(ptr, size);
-        if ptr.is_null() {
-            record_alloc(result);
-        } else if !core::ptr::eq(ptr, result) {
-            record_dealloc(ptr);
-            record_alloc(result);
-        }
-        result
-    }
-}
-
-/// # Safety
-///
-/// The caller must ensure that `ptr` was allocated by a previous call
-/// to `ts_record_malloc`, `ts_record_calloc`, or `ts_record_realloc`.
-pub unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
-    unsafe {
+    } else if ptr != result {
         record_dealloc(ptr);
-        free(ptr);
+        record_alloc(result);
     }
+    result
+}
+
+unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
+    record_dealloc(ptr);
+    free(ptr);
 }

cli/src/fuzz/corpus_test.rs

@@ -1,22 +1,10 @@
 use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
 
-use super::{LOG_ENABLED, LOG_GRAPH_ENABLED, scope_sequence::ScopeSequence};
+use super::{scope_sequence::ScopeSequence, LOG_ENABLED, LOG_GRAPH_ENABLED};
 use crate::util;
 
-struct SizeCheckFrame<'a> {
-    node: Node<'a>,
-    end_byte: usize,
-    end_point: Point,
-    child_count: u32,
-    child_index: u32,
-    last_child_end_byte: usize,
-    last_child_end_point: Point,
-    some_child_has_changes: bool,
-    actual_named_child_count: usize,
-}
-
-impl SizeCheckFrame<'_> {
-    fn new<'a>(node: Node<'a>, line_offsets: &[usize]) -> SizeCheckFrame<'a> {
+pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
+    fn check(node: Node, line_offsets: &[usize]) {
         let start_byte = node.start_byte();
         let end_byte = node.end_byte();
         let start_point = node.start_position();
@@ -30,21 +18,37 @@ impl SizeCheckFrame<'_> {
         );
         assert_eq!(end_byte, line_offsets[end_point.row] + end_point.column);
 
-        SizeCheckFrame {
-            node,
-            end_byte,
-            end_point,
-            child_count: node.child_count(),
-            child_index: 0,
-            last_child_end_byte: start_byte,
-            last_child_end_point: start_point,
-            some_child_has_changes: false,
-            actual_named_child_count: 0,
+        let mut last_child_end_byte = start_byte;
+        let mut last_child_end_point = start_point;
+        let mut some_child_has_changes = false;
+        let mut actual_named_child_count = 0;
+        for i in 0..node.child_count() {
+            let child = node.child(i).unwrap();
+            assert!(child.start_byte() >= last_child_end_byte);
+            assert!(child.start_position() >= last_child_end_point);
+            check(child, line_offsets);
+            if child.has_changes() {
+                some_child_has_changes = true;
+            }
+            if child.is_named() {
+                actual_named_child_count += 1;
+            }
+            last_child_end_byte = child.end_byte();
+            last_child_end_point = child.end_position();
+        }
+
+        assert_eq!(actual_named_child_count, node.named_child_count());
+
+        if node.child_count() > 0 {
+            assert!(end_byte >= last_child_end_byte);
+            assert!(end_point >= last_child_end_point);
+        }
+
+        if some_child_has_changes {
+            assert!(node.has_changes());
         }
     }
-}
 
-pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
     let mut line_offsets = vec![0];
     for (i, c) in input.iter().enumerate() {
         if *c == b'\n' {
@@ -52,41 +56,7 @@ pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
         }
     }
 
-    let mut stack: Vec<SizeCheckFrame> = vec![SizeCheckFrame::new(tree.root_node(), &line_offsets)];
-    while let Some(top) = stack.last_mut() {
-        if top.child_index < top.child_count {
-            let i = top.child_index;
-            let child = top.node.child(i).unwrap();
-
-            assert!(child.start_byte() >= top.last_child_end_byte);
-            assert!(child.start_position() >= top.last_child_end_point);
-            if child.has_changes() {
-                top.some_child_has_changes = true;
-            }
-            if child.is_named() {
-                top.actual_named_child_count += 1;
-            }
-            top.last_child_end_byte = child.end_byte();
-            top.last_child_end_point = child.end_position();
-            top.child_index += 1;
-
-            stack.push(SizeCheckFrame::new(child, &line_offsets));
-            continue;
-        }
-
-        let frame = stack.pop().unwrap();
-        assert_eq!(
-            frame.actual_named_child_count,
-            frame.node.named_child_count()
-        );
-        if frame.child_count > 0 {
-            assert!(frame.end_byte >= frame.last_child_end_byte);
-            assert!(frame.end_point >= frame.last_child_end_point);
-        }
-        if frame.some_child_has_changes {
-            assert!(frame.node.has_changes());
-        }
-    }
+    check(tree.root_node(), &line_offsets);
 }
 
 pub fn check_changed_ranges(old_tree: &Tree, new_tree: &Tree, input: &[u8]) -> Result<(), String> {