.dockerignore

@@ -0,0 +1,2 @@
+target
+.git

.editorconfig

@@ -10,9 +10,6 @@ insert_final_newline = true
 [*.rs]
 indent_size = 4
 
-[*.{zig,zon}]
-indent_size = 4
-
 [Makefile]
 indent_style = tab
 indent_size = 8

.envrc

@@ -1,4 +0,0 @@
-source_up_if_exists
-if [ -z "${IN_NIX_SHELL:-}" ]; then
-  use flake .
-fi

.gitattributes

@@ -3,4 +3,5 @@
 /lib/src/unicode/*.h linguist-vendored
 /lib/src/unicode/LICENSE linguist-vendored
 
+/cli/src/generate/prepare_grammar/*.json -diff
 Cargo.lock -diff

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,15 +1,7 @@
 name: Bug Report
 description: Report a problem
-type: Bug
+labels: [bug]
 body:
-  - type: checkboxes
-    attributes:
-      label: AI Policy
-      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
-      options:
-        - label: I have read the AI Policy and this issue complies with it.
-          required: true
-
   - type: textarea
     attributes:
       label: "Problem"

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,15 +1,7 @@
 name: Feature request
 description: Request an enhancement
-type: Feature
+labels: [enhancement]
 body:
-  - type: checkboxes
-    attributes:
-      label: AI Policy
-      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
-      options:
-        - label: I have read the AI Policy and this issue complies with it.
-          required: true
-
   - type: markdown
     attributes:
       value: |

.github/actions/cache/action.yml

@@ -10,20 +10,17 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
       id: cache
       with:
         path: |
           test/fixtures/grammars
           target/release/tree-sitter-*.wasm
         key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles(
-          'crates/generate/src/**',
+          'cli/generate/src/**',
           'lib/src/parser.h',
           'lib/src/array.h',
           'lib/src/alloc.h',
-          'lib/src/wasm-stdlib/external_scanner_stdlib.h',
-          'crates/loader/wasi-sdk-version',
-          'crates/loader/binaryen-version',
+          'xtask/src/*',
           'test/fixtures/grammars/*/**/src/*.c',
-          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}

.github/cliff.toml

@@ -43,16 +43,16 @@ commit_preprocessors = [
 ]
 # regex for parsing and grouping commits
 commit_parsers = [
-  { group = "<!-- 0 -->Breaking", message = "!:" },
-  { group = "<!-- 1 -->Features", message = "^feat" },
-  { group = "<!-- 2 -->Bug Fixes", message = "^fix" },
-  { group = "<!-- 3 -->Performance", message = "^perf" },
-  { group = "<!-- 4 -->Documentation", message = "^doc" },
-  { group = "<!-- 5 -->Refactor", message = "^refactor" },
-  { group = "<!-- 6 -->Testing", message = "^test" },
-  { group = "<!-- 7 -->Build System and CI", message = "^build" },
-  { group = "<!-- 7 -->Build System and CI", message = "^ci" },
-  { group = "<!-- 8 -->Other", message = ".*" },
+  { message = "!:", group = "<!-- 0 -->Breaking" },
+  { message = "^feat", group = "<!-- 1 -->Features" },
+  { message = "^fix", group = "<!-- 2 -->Bug Fixes" },
+  { message = "^perf", group = "<!-- 3 -->Performance" },
+  { message = "^doc", group = "<!-- 4 -->Documentation" },
+  { message = "^refactor", group = "<!-- 5 -->Refactor" },
+  { message = "^test", group = "<!-- 6 -->Testing" },
+  { message = "^build", group = "<!-- 7 -->Build System and CI" },
+  { message = "^ci", group = "<!-- 7 -->Build System and CI" },
+  { message = ".*", group = "<!-- 8 -->Other" },
 ]
 # filter out the commits that are not matched by commit parsers
 filter_commits = false

.github/dependabot.yml

@@ -4,8 +4,6 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    cooldown:
-      default-days: 3
     commit-message:
       prefix: "build(deps)"
     labels:
@@ -14,16 +12,10 @@ updates:
     groups:
       cargo:
         patterns: ["*"]
-    ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-major", "version-update:semver-minor"]
-
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
-    cooldown:
-      default-days: 3
     commit-message:
       prefix: "ci"
     labels:
@@ -32,22 +24,3 @@ updates:
     groups:
       actions:
         patterns: ["*"]
-
-  - package-ecosystem: "npm"
-    versioning-strategy: increase
-    directories:
-      - "/crates/npm"
-      - "/crates/eslint"
-      - "/lib/binding_web"
-    schedule:
-      interval: "weekly"
-    cooldown:
-      default-days: 3
-    commit-message:
-      prefix: "build(deps)"
-    labels:
-      - "dependencies"
-      - "npm"
-    groups:
-      npm:
-        patterns: ["*"]

.github/pull_request_template.md

@@ -1,6 +0,0 @@
-### AI Policy
-
-- [ ] I have read the [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy) and this PR complies with it.
-- [ ] If AI tools were used: I have disclosed the tool and extent of usage below.
-
-<!-- If you used AI tools, state which tool and how it was used. Delete this section if not applicable. -->

.github/scripts/close_spam.js

@@ -1,29 +0,0 @@
-module.exports = async ({ github, context }) => {
-  let target = context.payload.issue;
-  if (target) {
-    await github.rest.issues.update({
-      ...context.repo,
-      issue_number: target.number,
-      state: "closed",
-      state_reason: "not_planned",
-      title: "[spam]",
-      body: "",
-      type: null,
-    });
-  } else {
-    target = context.payload.pull_request;
-    await github.rest.pulls.update({
-      ...context.repo,
-      pull_number: target.number,
-      state: "closed",
-      title: "[spam]",
-      body: "",
-    });
-  }
-
-  await github.rest.issues.lock({
-    ...context.repo,
-    issue_number: target.number,
-    lock_reason: "spam",
-  });
-};

.github/scripts/cross.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+
+exec docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/scripts/make.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -eu
+
+tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
+
+if [[ $BUILD_CMD == cross ]]; then
+  cross.sh make CC="$CC" AR="$AR" "$@"
+else
+  exec make "$@"
+fi

.github/scripts/reviewers_remove.js

@@ -0,0 +1,16 @@
+module.exports = async ({ github, context }) => {
+  const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    pull_number: context.issue.number,
+  });
+
+  const reviewers = requestedReviewers.data.users.map((e) => e.login);
+
+  github.rest.pulls.removeRequestedReviewers({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    pull_number: context.issue.number,
+    reviewers: reviewers,
+  });
+};

.github/scripts/tree-sitter.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -eu
+
+tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
+
+if [[ $BUILD_CMD == cross ]]; then
+  cross.sh "$CROSS_RUNNER" "$tree_sitter" "$@"
+else
+  exec "$tree_sitter" "$@"
+fi

.github/scripts/wasm_stdlib.js

@@ -1,35 +0,0 @@
-module.exports = async ({ github, context, core }) => {
-  if (context.eventName !== 'pull_request') return;
-
-  const prNumber = context.payload.pull_request.number;
-  const owner = context.repo.owner;
-  const repo = context.repo.repo;
-
-  const files = await github.paginate(github.rest.pulls.listFiles, {
-    owner,
-    repo,
-    pull_number: prNumber,
-    per_page: 100
-  });
-
-  const changedFiles = files.map(file => file.filename);
-
-  const wasmStdLibSources = [
-    'lib/src/wasm-stdlib/external_scanner_allocator.c',
-    'lib/src/wasm-stdlib/imports.txt',
-    'lib/src/wasm-stdlib/libc.c',
-    'lib/src/wasm-stdlib/stdio.c'
-  ];
-  const dirChanged = changedFiles.some(file =>
-    wasmStdLibSources.includes(file) ||
-    file.startsWith('lib/src/wasm-stdlib/libc/ctype/') ||
-    file.startsWith('lib/src/wasm-stdlib/libc/string/')
-  );
-
-  if (!dirChanged) return;
-
-  const wasmStdLibHeader = 'lib/src/wasm-stdlib/external_scanner_stdlib.h';
-  const requiredChanged = changedFiles.includes(wasmStdLibHeader);
-
-  if (!requiredChanged) core.setFailed(`Changes detected in the Wasm stdlib sources but ${wasmStdLibHeader} was not modified.`);
-};

.github/workflows/backport.yml

@@ -14,20 +14,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-        with:
-          persist-credentials: true
-          ref: ${{ github.event.pull_request.base.ref }}
+        uses: actions/checkout@v4
 
       - name: Create app token
-        uses: actions/create-github-app-token@v3
+        uses: actions/create-github-app-token@v1
         id: app-token
         with:
           app-id: ${{ vars.BACKPORT_APP }}
           private-key: ${{ secrets.BACKPORT_KEY }}
 
       - name: Create backport PR
-        uses: korthout/backport-action@v4.6.0
+        uses: korthout/backport-action@v3
         with:
           pull_title: "${pull_title}"
           label_pattern: "^ci:backport ([^ ]+)$"

.github/workflows/bindgen.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1

.github/workflows/build.yml

@@ -1,5 +1,10 @@
 name: Build & Test
 
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+  CROSS_DEBUG: 1
+
 on:
   workflow_call:
     inputs:
@@ -26,46 +31,38 @@ jobs:
           - windows-x86
           - macos-arm64
           - macos-x64
-          - illumos-x64
-          - wasm32
 
         include:
           # When adding a new `target`:
           # 1. Define a new platform alias above
-          # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
-          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
-          - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
-          - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
-          - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
-          - { platform: linux-x86       , target: i686-unknown-linux-gnu        , os: ubuntu-24.04     }
-          - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu   , os: ubuntu-24.04     }
-          - { platform: windows-arm64   , target: aarch64-pc-windows-msvc       , os: windows-11-arm   }
-          - { platform: windows-x64     , target: x86_64-pc-windows-msvc        , os: windows-2025     }
-          - { platform: windows-x86     , target: i686-pc-windows-msvc          , os: windows-2025     }
-          - { platform: macos-arm64     , target: aarch64-apple-darwin          , os: macos-15         }
-          - { platform: macos-x64       , target: x86_64-apple-darwin           , os: macos-15-intel   }
-          - { platform: wasm32          , target: wasm32-unknown-unknown        , os: ubuntu-24.04     }
+          # 2. Add a new record to the matrix map in `cli/npm/install.js`
+          - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu   , os: ubuntu-latest    , use-cross: true }
+          - { platform: linux-arm         , target: arm-unknown-linux-gnueabi   , os: ubuntu-latest    , use-cross: true }
+          - { platform: linux-x64         , target: x86_64-unknown-linux-gnu    , os: ubuntu-22.04     , features: wasm  }
+          - { platform: linux-x86         , target: i686-unknown-linux-gnu      , os: ubuntu-latest    , use-cross: true }
+          - { platform: linux-powerpc64   , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest    , use-cross: true }
+          - { platform: windows-arm64     , target: aarch64-pc-windows-msvc     , os: windows-latest                     }
+          - { platform: windows-x64       , target: x86_64-pc-windows-msvc      , os: windows-latest   , features: wasm  }
+          - { platform: windows-x86       , target: i686-pc-windows-msvc        , os: windows-latest                     }
+          - { platform: macos-arm64       , target: aarch64-apple-darwin        , os: macos-latest     , features: wasm  }
+          - { platform: macos-x64         , target: x86_64-apple-darwin         , os: macos-13         , features: wasm  }
 
-          # illumos is not supported OOTB, it runs in a vm
-          - { platform: illumos-x64     , target: x86_64-unknown-illumos        , os: ubuntu-24.04     , vm: true , no-run: true }
+          # Cross compilers for C library
+          - { platform: linux-arm64       , cc: aarch64-linux-gnu-gcc           , ar: aarch64-linux-gnu-ar   }
+          - { platform: linux-arm         , cc: arm-linux-gnueabi-gcc           , ar: arm-linux-gnueabi-ar   }
+          - { platform: linux-x86         , cc: i686-linux-gnu-gcc              , ar: i686-linux-gnu-ar      }
+          - { platform: linux-powerpc64   , cc: powerpc64-linux-gnu-gcc         , ar: powerpc64-linux-gnu-ar }
 
-          # Extra features
-          - { platform: linux-arm64     , features: wasm }
-          - { platform: linux-x64       , features: wasm , run-wasm-test: true }
-          - { platform: macos-arm64     , features: wasm , run-wasm-test: true }
-          - { platform: macos-x64       , features: wasm }
+          # Prevent race condition (see #2041)
+          - { platform: windows-x64   , rust-test-threads: 1 }
+          - { platform: windows-x86   , rust-test-threads: 1 }
 
-          # Cross-compilation
-          - { platform: linux-arm       , cross: true }
-          - { platform: linux-x86       , cross: true }
-          - { platform: linux-powerpc64 , cross: true }
-
-          # Compile-only
-          - { platform: wasm32          , no-run: true }
+          # Can't natively run CLI on Github runner's host
+          - { platform: windows-arm64 , no-run: true }
 
     env:
-      CARGO_TERM_COLOR: always
-      RUSTFLAGS: -D warnings
+      BUILD_CMD: cargo
+      SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
 
     defaults:
       run:
@@ -73,186 +70,122 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v7.0.1
+      uses: actions/checkout@v4
 
-    - name: Set up cross-compilation
-      if: matrix.cross
-      run: |
-        for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do
-          camel_target=${target//-/_}; target_cc=${target/-unknown/}
-          printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
-          printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
-          printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
-        done >> $GITHUB_ENV
-        {
-          printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
-          printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
-        } >> $GITHUB_ENV
-
-    - name: Get emscripten version
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
-
-    - name: Cache Emscripten SDK
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      uses: actions/cache@v6
-      with:
-        path: emsdk
-        key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
+    - name: Read Emscripten version
+      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<cli/loader/emscripten-version)" >> $GITHUB_ENV
 
     - name: Install Emscripten
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      run: |
-        if [[ ! -d emsdk ]]; then
-          git clone --depth 1 https://github.com/emscripten-core/emsdk.git
-        fi
-        cd emsdk
-        ./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
-        ./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
-        echo "$PWD" >> "$GITHUB_PATH"
-        echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
-        echo "EMSDK=$PWD" >> "$GITHUB_ENV"
-        echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
-
-    - name: Set up Node.js
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
-      uses: actions/setup-node@v7.0.0
+      if: ${{ !matrix.no-run && !matrix.use-cross }}
+      uses: mymindstorm/setup-emsdk@v14
       with:
-        node-version: 24
-        cache: npm
-        cache-dependency-path: lib/binding_web/package-lock.json
+        version: ${{ env.EMSCRIPTEN_VERSION }}
 
     - name: Set up Rust
-      if: ${{ !matrix.vm }}
       uses: actions-rust-lang/setup-rust-toolchain@v1
       with:
         target: ${{ matrix.target }}
 
-    - name: Install Rust Wasm test target
-      if: matrix.run-wasm-test
-      run: rustup toolchain install nightly --profile minimal --component rust-src
-
-    - name: Install cross-compilation toolchain
-      if: matrix.cross
+    - name: Install cross
+      if: ${{ matrix.use-cross }}
       run: |
-        sudo apt-get update -qy
-        if [[ $PLATFORM == linux-arm ]]; then
-          sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user
-        elif [[ $PLATFORM == linux-x86 ]]; then
-          sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu
-        elif [[ $PLATFORM == linux-powerpc64 ]]; then
-          sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user
+        if [ ! -x "$(command -v cross)" ]; then
+          # TODO: Remove 'RUSTFLAGS=""' once https://github.com/cross-rs/cross/issues/1561 is resolved
+          RUSTFLAGS="" cargo install cross --git https://github.com/cross-rs/cross
         fi
-      env:
-        PLATFORM: ${{ matrix.platform }}
 
-    - name: Install MinGW and Clang (Windows x64 MSYS2)
-      if: matrix.platform == 'windows-x64'
-      uses: msys2/setup-msys2@v2
-      with:
-        install: |
-          mingw-w64-x86_64-toolchain
-          mingw-w64-x86_64-clang
-          mingw-w64-x86_64-make
-          mingw-w64-x86_64-cmake
-
-    - name: Build C library (Windows x64 MSYS2 CMake)
-      if: matrix.platform == 'windows-x64'
-      shell: msys2 {0}
+    - name: Configure cross
+      if: ${{ matrix.use-cross }}
       run: |
-        cmake -G Ninja -S . -B build/static \
-          -DBUILD_SHARED_LIBS=OFF \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-          -DTREE_SITTER_FEATURE_WASM=$WASM \
-          -DCMAKE_C_COMPILER=clang
-        cmake --build build/static
+        printf '%s\n' > Cross.toml \
+          '[target.${{ matrix.target }}]'                                   \
+          'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"'            \
+          '[build]'                                                         \
+          'pre-build = ['                                                   \
+          '  "dpkg --add-architecture $CROSS_DEB_ARCH",'                    \
+          '  "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",'  \
+          '  "apt-get update && apt-get -y install libssl-dev nodejs"'      \
+          ']'
+        cat - Cross.toml <<< 'Cross.toml:'
+        printf '%s\n' >> $GITHUB_ENV \
+          "CROSS_CONFIG=$PWD/Cross.toml" \
+          "CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge"
 
-        cmake -G Ninja -S . -B build/shared \
-          -DBUILD_SHARED_LIBS=ON \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-          -DTREE_SITTER_FEATURE_WASM=$WASM \
-          -DCMAKE_C_COMPILER=clang
-        cmake --build build/shared
-        rm -rf \
-          build/{static,shared} \
-          "${CMAKE_PREFIX_PATH}/artifacts" \
-          target/wasmtime-${WASMTIME_VERSION}
+    - name: Set up environment
       env:
-        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
+        RUST_TEST_THREADS: ${{ matrix.rust-test-threads }}
+        USE_CROSS: ${{ matrix.use-cross }}
+        TARGET: ${{ matrix.target }}
+        CC: ${{ matrix.cc }}
+        AR: ${{ matrix.ar }}
+      run: |
+        PATH="$PWD/.github/scripts:$PATH"
+        printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH
 
-    - name: Download wasmtime C API
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+        printf '%s\n' >> $GITHUB_ENV \
+          'TREE_SITTER=tree-sitter.sh' \
+          "TARGET=$TARGET" \
+          "ROOT=$PWD"
+
+        [[ -n $RUST_TEST_THREADS ]] && \
+          printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV
+
+        [[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV
+        [[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV
+
+        if [[ $USE_CROSS == true ]]; then
+          printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV
+          runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'")
+          [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV
+        fi
+
+    - name: Build wasmtime library
+      if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }}
       run: |
         mkdir -p target
         WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
                            jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        case '${{ matrix.target }}' in
-          x86_64-unknown-linux-gnu)  WT_TARGET=x86_64-linux ;;
-          aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
-          x86_64-apple-darwin)       WT_TARGET=x86_64-macos ;;
-          aarch64-apple-darwin)      WT_TARGET=aarch64-macos ;;
-        esac
-        curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
-          | tar xJf - -C target
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
+        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
+        cd target/wasmtime-${WASMTIME_VERSION}
+        cmake -S crates/c-api -B target/c-api \
+          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
+          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
+          -DWASMTIME_FEATURE_CRANELIFT=ON \
+          -DWASMTIME_TARGET='${{ matrix.target }}'
+        cmake --build target/c-api && cmake --install target/c-api
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
       env:
         WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
 
     - name: Build C library (make)
-      if: runner.os != 'Windows' && !matrix.vm
-      run: |
-        if [[ $PLATFORM == linux-arm ]]; then
-          CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar
-        elif [[ $PLATFORM == linux-x86 ]]; then
-          CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar
-        elif [[ $PLATFORM == linux-powerpc64 ]]; then
-          CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar
-        else
-          CC=gcc; AR=ar
-        fi
-        make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
+      if: ${{ runner.os != 'Windows' }}
+      run: make.sh -j CFLAGS="$CFLAGS"
       env:
-        PLATFORM: ${{ matrix.platform }}
-        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
 
     - name: Build C library (CMake)
-      if: "!matrix.cross && !matrix.vm"
+      if: ${{ !matrix.use-cross }}
       run: |
-        cmake -S . -B build/static \
+        cmake -S lib -B build/static \
           -DBUILD_SHARED_LIBS=OFF \
           -DCMAKE_BUILD_TYPE=Debug \
           -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
           -DTREE_SITTER_FEATURE_WASM=$WASM
         cmake --build build/static --verbose
 
-        cmake -S . -B build/shared \
+        cmake -S lib -B build/shared \
           -DBUILD_SHARED_LIBS=ON \
           -DCMAKE_BUILD_TYPE=Debug \
           -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
           -DTREE_SITTER_FEATURE_WASM=$WASM
         cmake --build build/shared --verbose
       env:
-        CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
-        WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
+        CC: ${{ contains(matrix.target, 'linux') && 'clang' || '' }}
+        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
 
-    - name: Build C library and Rust crate (illumos gmake)
-      if: matrix.platform == 'illumos-x64'
-      uses: vmactions/omnios-vm@v1.3.6
-      with:
-        release: r151056-build
-        copyback: false
-        prepare: |
-          pkg install -q build-essential || [ $? -eq 4 ]
-          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
-        run: |
-          . "$HOME/.cargo/env"
-          gmake -j
-          cargo build -p tree-sitter
-
-    - name: Build Wasm library
-      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+    - name: Build wasm library
+      # No reason to build on the same Github runner hosts many times
+      if: ${{ !matrix.no-run && !matrix.use-cross }}
       shell: bash
       run: |
         cd lib/binding_web
@@ -262,77 +195,64 @@ jobs:
         npm run build
         npm run build:debug
 
-    - name: Check no_std builds
-      if: inputs.run-test && !matrix.no-run
-      working-directory: lib
-      shell: bash
-      run: cargo check --no-default-features --target='${{ matrix.target }}'
-
     - name: Build target
-      if: "!inputs.run-test && !matrix.vm"
-      run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
-      env:
-        PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
+      run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }}
 
     - name: Cache fixtures
       id: cache
-      if: inputs.run-test && !matrix.no-run
+      if: ${{ !matrix.no-run && inputs.run-test }}
       uses: ./.github/actions/cache
 
     - name: Fetch fixtures
-      if: inputs.run-test && !matrix.no-run
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures
+      if: ${{ !matrix.no-run && inputs.run-test }}
+      run: $BUILD_CMD run -p xtask -- fetch-fixtures
 
     - name: Generate fixtures
-      if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true'
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
+      if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }}
+      run: $BUILD_CMD run -p xtask -- generate-fixtures
 
     - name: Generate Wasm fixtures
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
+      if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }}
+      run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm
 
     - name: Run main tests
-      if: inputs.run-test && !matrix.no-run
-      run: cargo test --workspace --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
+      if: ${{ !matrix.no-run && inputs.run-test }}
+      run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }}
 
-    - name: Run Wasm tests
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
+    - name: Run wasm tests
+      if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }}
+      run: $BUILD_CMD run -p xtask -- test-wasm
 
-    - name: Run Rust Wasm web test
-      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
-      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-rust-wasm-web
+    - name: Run benchmarks
+      # Cross-compiled benchmarks are pointless
+      if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }}
+      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
 
     - name: Upload CLI artifact
-      if: "!inputs.run-test && !matrix.no-run"
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v4
       with:
         name: tree-sitter.${{ matrix.platform }}
-        path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }}
         if-no-files-found: error
         retention-days: 7
 
     - name: Upload Wasm artifacts
-      if: "!inputs.run-test && matrix.platform == 'linux-x64'"
-      uses: actions/upload-artifact@v7
+      if: ${{ matrix.platform == 'linux-x64' }}
+      uses: actions/upload-artifact@v4
       with:
         name: tree-sitter.wasm
         path: |
-          lib/binding_web/web-tree-sitter.js
-          lib/binding_web/web-tree-sitter.js.map
-          lib/binding_web/web-tree-sitter.cjs
-          lib/binding_web/web-tree-sitter.cjs.map
-          lib/binding_web/web-tree-sitter.wasm
-          lib/binding_web/web-tree-sitter.wasm.map
-          lib/binding_web/debug/web-tree-sitter.cjs
-          lib/binding_web/debug/web-tree-sitter.cjs.map
-          lib/binding_web/debug/web-tree-sitter.js
-          lib/binding_web/debug/web-tree-sitter.js.map
-          lib/binding_web/debug/web-tree-sitter.wasm
-          lib/binding_web/debug/web-tree-sitter.wasm.map
-          lib/binding_web/lib/*.c
-          lib/binding_web/lib/*.h
-          lib/binding_web/lib/*.ts
-          lib/binding_web/src/*.ts
+          lib/binding_web/tree-sitter.js
+          lib/binding_web/tree-sitter.js.map
+          lib/binding_web/tree-sitter.cjs
+          lib/binding_web/tree-sitter.cjs.map
+          lib/binding_web/tree-sitter.wasm
+          lib/binding_web/tree-sitter.wasm.map
+          lib/binding_web/debug/tree-sitter.cjs
+          lib/binding_web/debug/tree-sitter.cjs.map
+          lib/binding_web/debug/tree-sitter.js
+          lib/binding_web/debug/tree-sitter.js.map
+          lib/binding_web/debug/tree-sitter.wasm
+          lib/binding_web/debug/tree-sitter.wasm.map
         if-no-files-found: error
         retention-days: 7

.github/workflows/ci.yml

@@ -1,14 +1,21 @@
 name: CI
 
 on:
-  push:
-    branches:
-      - 'master'
-      - 'release-[0-9]+.[0-9]+'
   pull_request:
-    branches:
-      - 'master'
-      - 'release-[0-9]+.[0-9]+'
+    paths-ignore:
+      - docs/**
+      - "**/README.md"
+      - CONTRIBUTING.md
+      - LICENSE
+      - cli/src/templates
+  push:
+    branches: [master]
+    paths-ignore:
+      - docs/**
+      - "**/README.md"
+      - CONTRIBUTING.md
+      - LICENSE
+      - cli/src/templates
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -19,33 +26,26 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: stable
+
+      - name: Set up nightly Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
           components: clippy, rustfmt
 
-      - name: Lint Rust files
-        run: make lint
-
-      - name: Install Taplo
-        uses: taiki-e/install-action@v2
-        with:
-          tool: taplo@0.10.0
-
-      - name: Lint TOML files
-        run: make lint-toml
-
-      - name: Lint web files
-        run: make lint-web
+      - name: Lint files
+        run: |
+          make lint
+          make lint-web
 
   sanitize:
     uses: ./.github/workflows/sanitize.yml
 
   build:
     uses: ./.github/workflows/build.yml
-
-  check-wasm-stdlib:
-    uses: ./.github/workflows/wasm_stdlib.yml

.github/workflows/crate_versions.yml

@@ -1,22 +0,0 @@
-name: Crate Versions Check
-
-on:
-  pull_request:
-    types: [labeled, opened, synchronize, reopened]
-  workflow_dispatch:
-
-jobs:
-  check-crates:
-    runs-on: ubuntu-latest
-    if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Check crates against crates.io
-        uses: katyo/publish-crates@v2
-        with:
-          dry-run: true

.github/workflows/docs.yml

@@ -3,7 +3,6 @@ on:
   push:
     branches: [master]
     paths: [docs/**]
-  workflow_dispatch:
 
 jobs:
   deploy-docs:
@@ -16,29 +15,35 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
 
       - name: Install mdbook
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
-          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.5.4 --jq "$jq_expr")
+          url=$(gh api repos/rust-lang/mdbook/releases/latest --jq "$jq_expr")
           mkdir mdbook
           curl -sSL "$url" | tar -xz -C mdbook
           printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
 
+      - name: Install mdbook-admonish
+        run: cargo install mdbook-admonish
+
       - name: Build Book
         run: mdbook build docs
 
       - name: Setup Pages
-        uses: actions/configure-pages@v6
+        uses: actions/configure-pages@v5
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v5
+        uses: actions/upload-pages-artifact@v3
         with:
           path: docs/book
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v5
+        uses: actions/deploy-pages@v4

.github/workflows/emscripten.yml

@@ -0,0 +1,30 @@
+name: Update Emscripten
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+permissions:
+  contents: write
+  pull-requests: read
+
+jobs:
+  update-emscripten:
+    if: github.actor == 'dependabot[bot]'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Run emscripten update xtask
+        run: |
+          git config --global user.name "dependabot[bot]"
+          git config --global user.email "49699333+dependabot[bot]@users.noreply.github.com"
+          cargo xtask upgrade-emscripten
+
+      - name: Push updated version
+        run: git push origin HEAD:$GITHUB_HEAD_REF

.github/workflows/nvim_ts.yml

@@ -3,11 +3,7 @@ name: nvim-treesitter parser tests
 on:
   pull_request:
     paths:
-      - 'lib/**'
-      - 'crates/cli/**'
-      - 'crates/config/**'
-      - 'crates/generate/**'
-      - 'crates/loader/**'
+      - 'cli/**'
       - '.github/workflows/nvim_ts.yml'
   workflow_dispatch:
 
@@ -17,7 +13,7 @@ concurrency:
 
 jobs:
   check_compilation:
-    timeout-minutes: 30
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
@@ -26,71 +22,45 @@ jobs:
     name: ${{ matrix.os }} - ${{ matrix.type }}
     runs-on: ${{ matrix.os }}
     env:
-      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim.exe' || 'nvim' }}
-      NVIM_TAG: stable
-      NVIM_DIR: neovim
+      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }}
       NVIM_TS_DIR: nvim-treesitter
     steps:
-      - uses: actions/checkout@v7.0.1
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - run: cargo build --profile optimize
+      - uses: actions/checkout@v4
 
-      - name: Clone Neovim
-        uses: actions/checkout@v7.0.1
-        with:
-          repository: neovim/neovim
-          ref: ${{ env.NVIM_TAG }}
-          path: ${{ env.NVIM_DIR }}
-
-      - if: runner.os != 'Windows'
-        name: Setup environment (Posix)
-        run: |
-          echo ${{ github.workspace }}/target/optimize >> "$GITHUB_PATH"
-          echo ${{ github.workspace }}/neovim/build/bin >> "$GITHUB_PATH"
-          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$GITHUB_ENV"
-
-      - if: runner.os == 'Windows'
-        name: Setup environment (why can't you just be normal?!)
-        run: |
-          ${{ env.NVIM_DIR }}/.github/scripts/env.ps1
-          echo ${{ github.workspace }}/target/optimize >> "$env:GITHUB_PATH"
-          echo ${{ github.workspace }}/neovim/build/bin >> "$env:GITHUB_PATH"
-          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$env:GITHUB_ENV"
-
-      - name: Build Neovim
-        working-directory: ${{ env.NVIM_DIR }}
-        run: |
-          cmake -S cmake.deps -B .deps -G Ninja -D CMAKE_BUILD_TYPE=Release -D TREESITTER_URL=https://github.com/tree-sitter/tree-sitter/archive/${{ github.event.pull_request.head.sha }}.tar.gz -D DEPS_IGNORE_SHA=TRUE
-          cmake --build .deps --config Release
-          cmake -B build -G Ninja -D CMAKE_BUILD_TYPE=Release
-          cmake --build build --config Release
-
-      - name: Clone nvim-treesitter
-        uses: actions/checkout@v7.0.1
+      - uses: actions/checkout@v4
         with:
           repository: nvim-treesitter/nvim-treesitter
           path: ${{ env.NVIM_TS_DIR }}
+          ref: main
+
+      - if: runner.os != 'Windows'
+        run: echo ${{ github.workspace }}/target/release >> $GITHUB_PATH
+
+      - if: runner.os == 'Windows'
+        run: echo ${{ github.workspace }}/target/release >> $env:GITHUB_PATH
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build --release
+      - uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Install and prepare Neovim
+        run: bash ./scripts/ci-install.sh
+        working-directory: ${{ env.NVIM_TS_DIR }}
 
       - if: matrix.type == 'generate'
         name: Generate and compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=10
+        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=2
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash
 
       - if: matrix.type == 'build'
         name: Compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10
+        run: $NVIM -l ./scripts/install-parsers.lua
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash
 
       - if: "!cancelled()"
-        name: Test parsers
-        run: $NVIM -l ./scripts/check-parsers.lua
-        working-directory: ${{ env.NVIM_TS_DIR }}
-        shell: bash
-
-      - if: "!cancelled()"
-        name: Test queries
+        name: Check query files
         run: $NVIM -l ./scripts/check-queries.lua
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash

.github/workflows/release.yml

@@ -1,14 +1,7 @@
 name: Release
 
 on:
-  schedule:
-    - cron: '5 5 * * *'
   workflow_dispatch:
-    inputs:
-      tag_name:
-        description: 'Tag name for release'
-        required: false
-        default: nightly
   push:
     tags:
       - v[0-9]+.[0-9]+.[0-9]+
@@ -24,26 +17,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     permissions:
-      id-token: write
-      attestations: write
       contents: write
     steps:
-      - if: github.event_name == 'workflow_dispatch'
-        env:
-          TAG_NAME: ${{ github.event.inputs.tag_name }}
-        run: echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_ENV
-
-      - if: github.event_name == 'schedule'
-        run: echo 'TAG_NAME=nightly' >> $GITHUB_ENV
-
-      - if: github.event_name == 'push'
-        run: echo "TAG_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
-
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 
@@ -53,93 +33,84 @@ jobs:
 
       - name: Prepare release artifacts
         run: |
-          mkdir -p target web
-          mv artifacts/tree-sitter.wasm/* web/
+          mkdir -p target
+          mv artifacts/tree-sitter.wasm/* target/
 
-          tar -czf target/web-tree-sitter.tar.gz -C web .
+          # Rename files
+          mv target/tree-sitter.js        target/web-tree-sitter.js
+          mv target/tree-sitter.js.map    target/web-tree-sitter.js.map
+          mv target/tree-sitter.cjs       target/web-tree-sitter.cjs
+          mv target/tree-sitter.cjs.map   target/web-tree-sitter.cjs.map
+          mv target/tree-sitter.wasm      target/web-tree-sitter.wasm
+          mv target/tree-sitter.wasm.map  target/web-tree-sitter.wasm.map
+
+          mv target/debug/tree-sitter.js        target/web-tree-sitter-debug.js
+          mv target/debug/tree-sitter.js.map    target/web-tree-sitter-debug.js.map
+          mv target/debug/tree-sitter.cjs       target/web-tree-sitter-debug.cjs
+          mv target/debug/tree-sitter.cjs.map   target/web-tree-sitter-debug.cjs.map
+          mv target/debug/tree-sitter.wasm      target/web-tree-sitter-debug.wasm
+          mv target/debug/tree-sitter.wasm.map  target/web-tree-sitter-debug.wasm.map
+          rm -rf target/debug
 
           rm -r artifacts/tree-sitter.wasm
-
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
-            chmod +x $exe
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
-            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
 
-      - name: Generate attestations
-        uses: actions/attest-build-provenance@v4
-        with:
-          subject-path: |
-            target/tree-sitter-*.gz
-            target/tree-sitter-cli-*.zip
-            target/web-tree-sitter.tar.gz
-
-      - if: env.TAG_NAME == 'nightly'
-        run: |
-          echo 'PRERELEASE=--prerelease' >> $GITHUB_ENV
-          gh release delete nightly --yes || true
-          git push https://${GITHUB_ACTOR}:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY} :nightly || true
-        env:
-          GH_TOKEN: ${{ github.token }}
-
       - name: Create release
         run: |-
-          gh release create ${{ env.TAG_NAME }} $PRERELEASE \
+          gh release create ${{ github.ref_name }} \
             target/tree-sitter-*.gz \
-            target/tree-sitter-cli-*.zip \
-            target/web-tree-sitter.tar.gz
+            target/web-tree-sitter.js \
+            target/web-tree-sitter.js.map \
+            target/web-tree-sitter.cjs \
+            target/web-tree-sitter.cjs.map \
+            target/web-tree-sitter.wasm \
+            target/web-tree-sitter.wasm.map \
+            target/web-tree-sitter-debug.js \
+            target/web-tree-sitter-debug.js.map \
+            target/web-tree-sitter-debug.cjs \
+            target/web-tree-sitter-debug.cjs.map \
+            target/web-tree-sitter-debug.wasm \
+            target/web-tree-sitter-debug.wasm.map
         env:
           GH_TOKEN: ${{ github.token }}
 
   crates_io:
     name: Publish packages to Crates.io
-    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
-    environment: crates
-    permissions:
-      id-token: write
-      contents: read
     needs: release
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Set up Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
 
-      - name: Set up registry token
-        id: auth
-        uses: rust-lang/crates-io-auth-action@v1
-
       - name: Publish crates to Crates.io
         uses: katyo/publish-crates@v2
         with:
-          registry-token: ${{ steps.auth.outputs.token }}
+          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
 
   npm:
     name: Publish packages to npmjs.com
-    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
-    environment: npm
-    permissions:
-      id-token: write
-      contents: read
     needs: release
     strategy:
       fail-fast: false
       matrix:
-        directory: [crates/cli/npm, lib/binding_web]
+        directory: [cli/npm, lib/binding_web]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Set up Node
-        uses: actions/setup-node@v7.0.0
+        uses: actions/setup-node@v4
         with:
-          node-version: 24
+          node-version: 20
           registry-url: https://registry.npmjs.org
 
       - name: Set up Rust
@@ -154,8 +125,9 @@ jobs:
           npm run build:debug
           CJS=true npm run build
           CJS=true npm run build:debug
-          npm run build:dts
 
       - name: Publish to npmjs.com
         working-directory: ${{ matrix.directory }}
         run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/response.yml

@@ -17,13 +17,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout script
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
         with:
           sparse-checkout: .github/scripts/close_unresponsive.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v9
+        uses: actions/github-script@v7
         with:
           script: |
             const script = require('./.github/scripts/close_unresponsive.js')
@@ -35,13 +35,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout script
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
         with:
           sparse-checkout: .github/scripts/remove_response_label.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v9
+        uses: actions/github-script@v7
         with:
           script: |
             const script = require('./.github/scripts/remove_response_label.js')

.github/workflows/reviewers_remove.yml

@@ -11,21 +11,15 @@ jobs:
   remove-reviewers:
     runs-on: ubuntu-latest
     steps:
-      - name: Remove reviewers
-        uses: actions/github-script@v9
+      - name: Checkout script
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts/reviewers_remove.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v7
         with:
           script: |
-            const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.issue.number,
-            });
-
-            const reviewers = requestedReviewers.data.users.map((e) => e.login);
-
-            github.rest.pulls.removeRequestedReviewers({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.issue.number,
-              reviewers: reviewers,
-            });
+            const script = require('./.github/scripts/reviewers_remove.js')
+            await script({github, context})

.github/workflows/sanitize.yml

@@ -15,7 +15,7 @@ jobs:
       TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Install UBSAN library
         run: sudo apt-get update -y && sudo apt-get install -y libubsan1

.github/workflows/spam.yml

@@ -1,29 +0,0 @@
-name: Close as spam
-
-on:
-  issues:
-    types: [labeled]
-  pull_request_target:
-    types: [labeled]
-
-permissions:
-  issues: write
-  pull-requests: write
-
-jobs:
-  spam:
-    runs-on: ubuntu-latest
-    if: github.event.label.name == 'spam'
-    steps:
-      - name: Checkout script
-        uses: actions/checkout@v7.0.1
-        with:
-          sparse-checkout: .github/scripts/close_spam.js
-          sparse-checkout-cone-mode: false
-
-      - name: Run script
-        uses: actions/github-script@v9
-        with:
-          script: |
-            const script = require('./.github/scripts/close_spam.js')
-            await script({github, context})

.github/workflows/wasm_exports.yml

@@ -1,24 +1,23 @@
-name: Check Wasm Exports
+name: Check WASM Exports
 
 on:
   pull_request:
     paths:
       - lib/include/tree_sitter/api.h
       - lib/binding_web/**
-      - xtask/src/**
   push:
     branches: [master]
     paths:
       - lib/include/tree_sitter/api.h
       - lib/binding_rust/bindings.rs
-      - CMakeLists.txt
+      - lib/CMakeLists.txt
 
 jobs:
   check-wasm-exports:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7.0.1
+        uses: actions/checkout@v4
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -31,11 +30,11 @@ jobs:
       - name: Build C library (make)
         run: make -j CFLAGS="$CFLAGS"
         env:
-          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
 
-      - name: Build Wasm Library
+      - name: Build WASM Library
         working-directory: lib/binding_web
         run: npm ci && npm run build:debug
 
-      - name: Check Wasm exports
+      - name: Check WASM exports
         run: cargo xtask check-wasm-exports

.github/workflows/wasm_stdlib.yml

@@ -1,19 +0,0 @@
-name: Check Wasm Stdlib build
-
-on:
-  workflow_call:
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7.0.1
-
-      - name: Check directory changes
-        uses: actions/github-script@v9
-        with:
-          script: |
-            const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
-            const script = require(scriptPath);
-            return script({ github, context, core });

.gitignore

@@ -1,12 +1,10 @@
 log*.html
-.direnv
 
 .idea
 *.xcodeproj
 .vscode
 .cache
 .zig-cache
-.direnv
 
 profile*
 fuzz-results
@@ -26,7 +24,6 @@ docs/assets/js/tree-sitter.js
 *.dylib
 *.so
 *.so.[0-9]*
-*.dll
 *.o
 *.obj
 *.exp
@@ -36,5 +33,3 @@ docs/assets/js/tree-sitter.js
 .build
 build
 zig-*
-
-/result

.taplo.toml

@@ -1,19 +0,0 @@
-[formatting]
-column_width = 100
-compact_arrays = false
-reorder_inline_tables = true
-reorder_keys = true
-
-[[rule]]
-include = [ "**/Cargo.toml" ]
-keys = [ "package" ]
-
-[rule.formatting]
-reorder_keys = false
-
-[[rule]]
-include = [ "**/Cargo.toml" ]
-keys = [ "profile" ]
-
-[rule.formatting]
-reorder_keys = false

Cargo.toml

@@ -1,35 +1,37 @@
 [workspace]
-default-members = [ "crates/cli" ]
+default-members = ["cli"]
 members = [
-  "crates/cli",
-  "crates/config",
-  "crates/generate",
-  "crates/highlight",
-  "crates/loader",
-  "crates/tags",
-  "crates/xtask",
-  "crates/language",
+  "cli",
+  "cli/config",
+  "cli/loader",
   "lib",
+  "lib/language",
+  "tags",
+  "highlight",
+  "xtask",
 ]
 resolver = "2"
 
 [workspace.package]
-authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>" ]
-categories = [ "command-line-utilities", "parsing" ]
-edition = "2024"
+version = "0.25.1"
+authors = [
+  "Max Brunsfeld <maxbrunsfeld@gmail.com>",
+  "Amaan Qureshi <amaanq12@gmail.com>",
+]
+edition = "2021"
+rust-version = "1.82"
 homepage = "https://tree-sitter.github.io/tree-sitter"
-keywords = [ "incremental", "parsing" ]
-license = "MIT"
 repository = "https://github.com/tree-sitter/tree-sitter"
-rust-version = "1.90"
-version = "0.28.0"
+license = "MIT"
+keywords = ["incremental", "parsing"]
+categories = ["command-line-utilities", "parsing"]
 
 [workspace.lints.clippy]
-cargo = { level = "warn", priority = -1 }
 dbg_macro = "deny"
-nursery = { level = "warn", priority = -1 }
-pedantic = { level = "warn", priority = -1 }
 todo = "deny"
+pedantic = { level = "warn", priority = -1 }
+nursery = { level = "warn", priority = -1 }
+cargo = { level = "warn", priority = -1 }
 
 # The lints below are a specific subset of the pedantic+nursery lints
 # that we explicitly allow in the tree-sitter codebase because they either:
@@ -38,31 +40,47 @@ todo = "deny"
 # 2. Are unnecessary, or
 # 3. Worsen the code
 
+branches_sharing_code = "allow"
+cast_lossless = "allow"
 cast_possible_truncation = "allow"
 cast_possible_wrap = "allow"
 cast_precision_loss = "allow"
 cast_sign_loss = "allow"
+checked_conversions = "allow"
+cognitive_complexity = "allow"
+collection_is_never_read = "allow"
+fallible_impl_from = "allow"
+fn_params_excessive_bools = "allow"
+inline_always = "allow"
 if_not_else = "allow"
 items_after_statements = "allow"
+match_wildcard_for_single_variants = "allow"
 missing_errors_doc = "allow"
 missing_panics_doc = "allow"
+module_name_repetitions = "allow"
 multiple_crate_versions = "allow"
 option_if_let_else = "allow"
+or_fun_call = "allow"
+range_plus_one = "allow"
+redundant_clone = "allow"
+redundant_closure_for_method_calls = "allow"
+ref_option = "allow"
 similar_names = "allow"
 string_lit_as_bytes = "allow"
 struct_excessive_bools = "allow"
+struct_field_names = "allow"
+transmute_undefined_repr = "allow"
 too_many_lines = "allow"
-tuple_array_conversions = "allow"
-
-[workspace.lints.rust]
-mismatched_lifetime_syntaxes = "allow"
+unnecessary_wraps = "allow"
+unused_self = "allow"
+used_underscore_items = "allow"
 
 [profile.optimize]
 inherits = "release"
-codegen-units = 1    # Maximum size reduction optimizations.
+strip = true         # Automatically strip symbols from the binary.
 lto = true           # Link-time optimization.
 opt-level = 3        # Optimization level 3.
-strip = true         # Automatically strip symbols from the binary.
+codegen-units = 1    # Maximum size reduction optimizations.
 
 [profile.size]
 inherits = "optimize"
@@ -70,69 +88,70 @@ opt-level = "s"       # Optimize for size.
 
 [profile.release-dev]
 inherits = "release"
-codegen-units = 256
+lto = false
 debug = true
 debug-assertions = true
-incremental = true
-lto = false
 overflow-checks = true
+incremental = true
+codegen-units = 256
 
 [workspace.dependencies]
 ansi_colours = "1.2.3"
-anstyle = "1.0.14"
-anyhow = "1.0.102"
-bstr = "1.12.1"
-cc = "1.2.63"
-clap = { features = [
+anstyle = "1.0.10"
+anyhow = "1.0.95"
+bstr = "1.11.3"
+cc = "1.2.10"
+clap = { version = "4.5.27", features = [
   "cargo",
   "derive",
   "env",
   "help",
-  "string",
   "unstable-styles",
-], version = "4.5.58" }
-clap_complete = "4.6.3"
-clap_complete_nushell = "4.5.10"
-crc32fast = "1.5.0"
-ctor = "0.6.3"
-ctrlc = { features = [ "termination" ], version = "3.5.2" }
-dialoguer = { features = [ "fuzzy-select" ], version = "0.12.0" }
-etcetera = "0.11.0"
-glob = "0.3.3"
-hashbrown = { default-features = false, version = "0.17.1" }
+] }
+clap_complete = "4.5.42"
+clap_complete_nushell = "4.5.5"
+ctor = "0.2.9"
+ctrlc = { version = "3.4.5", features = ["termination"] }
+dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
+etcetera = "0.8.0"
+filetime = "0.2.25"
+fs4 = "0.12.0"
+git2 = "0.20.0"
+glob = "0.3.2"
 heck = "0.5.0"
 html-escape = "0.2.13"
-indexmap = "2.13.0"
-indoc = "2.0.7"
-libloading = "0.9.0"
-log = { features = [ "std" ], version = "0.4.30" }
-memchr = "2.8.1"
-once_cell = "1.21.3"
+indexmap = "2.7.1"
+indoc = "2.0.5"
+libloading = "0.8.6"
+log = { version = "0.4.25", features = ["std"] }
+memchr = "2.7.4"
+once_cell = "1.20.2"
+path-slash = "0.2.1"
 pretty_assertions = "1.4.1"
-rand = "0.10.1"
-regex = "1.12.3"
-regex-syntax = "0.8.9"
-rustc-hash = "2.1.1"
-schemars = "1.2.1"
-semver = { features = [ "serde" ], version = "1.0.27" }
-serde = { features = [ "derive" ], version = "1.0.228" }
-serde_json = { features = [ "preserve_order" ], version = "1.0.150" }
+rand = "0.8.5"
+regex = "1.11.1"
+regex-syntax = "0.8.5"
+rustc-hash = "2.1.0"
+semver = { version = "1.0.25", features = ["serde"] }
+serde = { version = "1.0.217", features = ["derive"] }
+serde_derive = "1.0.217"
+serde_json = { version = "1.0.137", features = ["preserve_order"] }
 similar = "2.7.0"
+smallbitvec = "2.5.3"
 streaming-iterator = "0.1.9"
-tempfile = "3.25.0"
-thiserror = "2.0.18"
+tempfile = "3.15.0"
+thiserror = "2.0.11"
 tiny_http = "0.12.0"
-topological-sort = "0.2.2"
-unindent = "0.2.4"
+toml = "0.8.19"
+unindent = "0.2.3"
+url = { version = "2.5.4", features = ["serde"] }
 walkdir = "2.5.0"
-wasmparser = "0.244.0"
-webbrowser = "1.2.1"
+wasmparser = "0.224.0"
+webbrowser = "1.0.3"
 
-tree-sitter = { path = "./lib", version = "0.28.0" }
-tree-sitter-config = { path = "./crates/config", version = "0.28.0" }
-tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.28.0" }
-tree-sitter-highlight = { path = "./crates/highlight", version = "0.28.0" }
-tree-sitter-loader = { path = "./crates/loader", version = "0.28.0" }
-tree-sitter-tags = { path = "./crates/tags", version = "0.28.0" }
-
-tree-sitter-language = { path = "./crates/language", version = "0.1.8" }
+tree-sitter = { version = "0.25.1", path = "./lib" }
+tree-sitter-generate = { version = "0.25.1", path = "./cli/generate" }
+tree-sitter-loader = { version = "0.25.1", path = "./cli/loader" }
+tree-sitter-config = { version = "0.25.1", path = "./cli/config" }
+tree-sitter-highlight = { version = "0.25.1", path = "./highlight" }
+tree-sitter-tags = { version = "0.25.1", path = "./tags" }

Dockerfile

@@ -0,0 +1,10 @@
+FROM rust:1.76-buster
+
+WORKDIR /app
+
+RUN apt-get update
+RUN apt-get install -y nodejs
+
+COPY . .
+
+CMD cargo test --all-features

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Max Brunsfeld
+Copyright (c) 2018-2024 Max Brunsfeld
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,4 +1,8 @@
-VERSION := 0.28.0
+ifeq ($(OS),Windows_NT)
+$(error Windows is not supported)
+endif
+
+VERSION := 0.25.1
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -6,7 +10,6 @@ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 PREFIX ?= /usr/local
 INCLUDEDIR ?= $(PREFIX)/include
 LIBDIR ?= $(PREFIX)/lib
-BINDIR ?= $(PREFIX)/bin
 PCLIBDIR ?= $(LIBDIR)/pkgconfig
 
 # collect sources
@@ -22,9 +25,9 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -32,25 +35,20 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
 SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
 
 # OS-specific bits
-MACHINE := $(shell $(CC) -dumpmachine)
-
-ifneq ($(findstring darwin,$(MACHINE)),)
+ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),)
 	SOEXT = dylib
 	SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
 	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
 	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER)
-else ifneq ($(findstring mingw32,$(MACHINE)),)
-	SOEXT = dll
-	LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a
 else
 	SOEXT = so
 	SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
 	SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
 	LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER)
+endif
 ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
 	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
 endif
-endif
 
 all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc
 
@@ -63,10 +61,6 @@ ifneq ($(STRIP),)
 	$(STRIP) $@
 endif
 
-ifneq ($(findstring mingw32,$(MACHINE)),)
-libtree-sitter.dll.a: libtree-sitter.$(SOEXT)
-endif
-
 tree-sitter.pc: lib/tree-sitter.pc.in
 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
 		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
@@ -75,27 +69,17 @@ tree-sitter.pc: lib/tree-sitter.pc.in
 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
 
-shared: libtree-sitter.$(SOEXT)
-
-static: libtree-sitter.a
-
 clean:
-	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a
+	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT)
 
 install: all
 	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
 	install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h
 	install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
 	install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a
-ifneq ($(findstring mingw32,$(MACHINE)),)
-	install -d '$(DESTDIR)$(BINDIR)'
-	install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll
-	install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a
-else
 	install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER)
-	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR)
-	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT)
-endif
+	ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR)
+	ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT)
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \
@@ -104,9 +88,8 @@ uninstall:
 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \
 		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \
 		'$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
-	rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
 
-.PHONY: all shared static install uninstall clean
+.PHONY: all install uninstall clean
 
 
 ##### Dev targets #####
@@ -122,20 +105,16 @@ test-wasm:
 
 lint:
 	cargo update --workspace --locked --quiet
-	cargo fmt --all --check
-	cargo clippy --workspace --all-targets -- -D warnings
+	cargo check --workspace --all-targets
+	cargo +nightly fmt --all --check
+	cargo +nightly clippy --workspace --all-targets -- -D warnings
 
 lint-web:
 	npm --prefix lib/binding_web ci
 	npm --prefix lib/binding_web run lint
 
-lint-toml:
-	taplo check
-	taplo format --check --diff
-
 format:
-	cargo fmt --all
-	taplo format
+	cargo +nightly fmt --all
 
 changelog:
 	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)

Package.swift

@@ -0,0 +1,33 @@
+// swift-tools-version: 5.8
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "TreeSitter",
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "TreeSitter",
+            targets: ["TreeSitter"]),
+    ],
+    targets: [
+        .target(name: "TreeSitter",
+                path: "lib",
+                exclude: [
+                        "src/unicode/ICU_SHA",
+                        "src/unicode/README.md",
+                        "src/unicode/LICENSE",
+                        "src/wasm/stdlib-symbols.txt",
+                        "src/lib.c",
+                ],
+                sources: ["src"],
+                publicHeadersPath: "include",
+                cSettings: [
+                        .headerSearchPath("src"),
+                        .define("_POSIX_C_SOURCE", to: "200112L"),
+                        .define("_DEFAULT_SOURCE"),
+                ]),
+    ],
+    cLanguageStandard: .c11
+)

README.md

@@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca
 ## Links
 - [Documentation](https://tree-sitter.github.io)
 - [Rust binding](lib/binding_rust/README.md)
-- [Wasm binding](lib/binding_web/README.md)
-- [Command-line interface](crates/cli/README.md)
+- [WASM binding](lib/binding_web/README.md)
+- [Command-line interface](cli/README.md)
 
 [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord
 [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix

build.zig

@@ -1,147 +1,116 @@
 const std = @import("std");
 
 pub fn build(b: *std.Build) !void {
-    const target = b.standardTargetOptions(.{});
-    const optimize = b.standardOptimizeOption(.{});
+  const target = b.standardTargetOptions(.{});
+  const optimize = b.standardOptimizeOption(.{});
 
-    var threaded: std.Io.Threaded = .init(b.allocator, .{});
-    defer threaded.deinit();
-    const io = threaded.io();
+  const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
+  const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
+  const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
 
-    const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
-    const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
-    const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
+  const lib: *std.Build.Step.Compile = if (!shared) b.addStaticLibrary(.{
+    .name = "tree-sitter",
+    .target = target,
+    .optimize = optimize,
+    .link_libc = true,
+  }) else b.addSharedLibrary(.{
+    .name = "tree-sitter",
+    .pic = true,
+    .target = target,
+    .optimize = optimize,
+    .link_libc = true,
+  });
 
-    var tree_sitter = b.createModule(.{
-        .target = target,
-        .optimize = optimize,
-        .link_libc = true,
-        .pic = if (shared) true else null,
+  if (amalgamated) {
+    lib.addCSourceFile(.{
+      .file = b.path("lib/src/lib.c"),
+      .flags = &.{"-std=c11"},
     });
-    const lib: *std.Build.Step.Compile = b.addLibrary(.{
-        .name = "tree-sitter",
-        .linkage = if (shared) .dynamic else .static,
-        .root_module = tree_sitter,
+  } else {
+    lib.addCSourceFiles(.{
+      .root = b.path("lib/src"),
+      .files = try findSourceFiles(b),
+      .flags = &.{"-std=c11"},
     });
+  }
 
-    if (amalgamated) {
-        tree_sitter.addCSourceFile(.{
-            .file = b.path("lib/src/lib.c"),
-            .flags = &.{"-std=c11"},
-        });
-    } else {
-        const files = try findSourceFiles(b, io);
-        defer b.allocator.free(files);
-        tree_sitter.addCSourceFiles(.{
-            .root = b.path("lib/src"),
-            .files = files,
-            .flags = &.{"-std=c11"},
-        });
+  lib.addIncludePath(b.path("lib/include"));
+  lib.addIncludePath(b.path("lib/src"));
+  lib.addIncludePath(b.path("lib/src/wasm"));
+
+  lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
+  lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
+
+  if (wasm) {
+    if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
+      lib.root_module.addCMacro("TREE_SITTER_FEATURE_WASM", "");
+      lib.addSystemIncludePath(wasmtime.path("include"));
+      lib.addLibraryPath(wasmtime.path("lib"));
+      lib.linkSystemLibrary("wasmtime");
     }
+  }
 
-    tree_sitter.addIncludePath(b.path("lib/include"));
-    tree_sitter.addIncludePath(b.path("lib/src"));
-    tree_sitter.addIncludePath(b.path("lib/src/wasm"));
+  lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
 
-    tree_sitter.addCMacro("_POSIX_C_SOURCE", "200112L");
-    tree_sitter.addCMacro("_DEFAULT_SOURCE", "");
-    tree_sitter.addCMacro("_BSD_SOURCE", "");
-    tree_sitter.addCMacro("_DARWIN_C_SOURCE", "");
-
-    if (wasm) {
-        if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
-            tree_sitter.addCMacro("TREE_SITTER_FEATURE_WASM", "");
-            tree_sitter.addSystemIncludePath(wasmtime.path("include"));
-            tree_sitter.addLibraryPath(wasmtime.path("lib"));
-            if (shared) tree_sitter.linkSystemLibrary("wasmtime", .{});
-        }
-    }
-
-    lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
-
-    b.installArtifact(lib);
+  b.installArtifact(lib);
 }
 
-/// Get the name of the wasmtime dependency for this target.
-pub fn wasmtimeDep(target: std.Target) []const u8 {
-    const arch = target.cpu.arch;
-    const os = target.os.tag;
-    const abi = target.abi;
-    return @as(?[]const u8, switch (os) {
-        .linux => switch (arch) {
-            .x86_64 => switch (abi) {
-                .gnu => "wasmtime_c_api_x86_64_linux",
-                .musl => "wasmtime_c_api_x86_64_musl",
-                .android => "wasmtime_c_api_x86_64_android",
-                else => null,
-            },
-            .aarch64 => switch (abi) {
-                .gnu => "wasmtime_c_api_aarch64_linux",
-                .musl => "wasmtime_c_api_aarch64_musl",
-                .android => "wasmtime_c_api_aarch64_android",
-                else => null,
-            },
-            .x86 => switch (abi) {
-                .gnu => "wasmtime_c_api_i686_linux",
-                else => null,
-            },
-            .arm => switch (abi) {
-                .gnueabi => "wasmtime_c_api_armv7_linux",
-                else => null,
-            },
-            .s390x => switch (abi) {
-                .gnu => "wasmtime_c_api_s390x_linux",
-                else => null,
-            },
-            .riscv64 => switch (abi) {
-                .gnu => "wasmtime_c_api_riscv64gc_linux",
-                else => null,
-            },
-            else => null,
-        },
-        .windows => switch (arch) {
-            .x86_64 => switch (abi) {
-                .gnu => "wasmtime_c_api_x86_64_mingw",
-                .msvc => "wasmtime_c_api_x86_64_windows",
-                else => null,
-            },
-            .aarch64 => switch (abi) {
-                .msvc => "wasmtime_c_api_aarch64_windows",
-                else => null,
-            },
-            .x86 => switch (abi) {
-                .msvc => "wasmtime_c_api_i686_windows",
-                else => null,
-            },
-            else => null,
-        },
-        .macos => switch (arch) {
-            .x86_64 => "wasmtime_c_api_x86_64_macos",
-            .aarch64 => "wasmtime_c_api_aarch64_macos",
-            else => null,
-        },
-        else => null,
-    }) orelse std.debug.panic(
-        "Unsupported target for wasmtime: {s}-{s}-{s}",
-        .{ @tagName(arch), @tagName(os), @tagName(abi) },
-    );
+fn wasmtimeDep(target: std.Target) []const u8 {
+  const arch = target.cpu.arch;
+  const os = target.os.tag;
+  const abi = target.abi;
+  return switch (os) {
+    .linux => switch (arch) {
+      .x86_64 => switch (abi) {
+        .gnu => "wasmtime_c_api_x86_64_linux",
+        .musl => "wasmtime_c_api_x86_64_musl",
+        .android => "wasmtime_c_api_x86_64_android",
+        else => null
+      },
+      .aarch64 => switch (abi) {
+        .gnu => "wasmtime_c_api_aarch64_linux",
+        .android => "wasmtime_c_api_aarch64_android",
+        else => null
+      },
+      .s390x => "wasmtime_c_api_s390x_linux",
+      .riscv64 => "wasmtime_c_api_riscv64gc_linux",
+      else => null
+    },
+    .windows => switch (arch) {
+      .x86_64 => switch (abi) {
+        .gnu => "wasmtime_c_api_x86_64_mingw",
+        .msvc => "wasmtime_c_api_x86_64_windows",
+        else => null
+      },
+      else => null
+    },
+    .macos => switch (arch) {
+      .x86_64 => "wasmtime_c_api_x86_64_macos",
+      .aarch64 => "wasmtime_c_api_aarch64_macos",
+      else => null
+    },
+    else => null
+  } orelse std.debug.panic(
+    "Unsupported target for wasmtime: {s}-{s}-{s}",
+    .{ @tagName(arch), @tagName(os), @tagName(abi) }
+  );
 }
 
-fn findSourceFiles(b: *std.Build, io: std.Io) ![]const []const u8 {
-    var sources: std.ArrayListUnmanaged([]const u8) = .empty;
+fn findSourceFiles(b: *std.Build) ![]const []const u8 {
+  var sources = std.ArrayList([]const u8).init(b.allocator);
 
-    var dir = try b.build_root.handle.openDir(io, "lib/src", .{ .iterate = true });
-    var iter = dir.iterate();
-    defer dir.close(io);
+  var dir = try b.build_root.handle.openDir("lib/src", .{ .iterate = true });
+  var iter = dir.iterate();
+  defer dir.close();
 
-    while (try iter.next(io)) |entry| {
-        if (entry.kind != .file) continue;
-        const file = entry.name;
-        const ext = std.fs.path.extension(file);
-        if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
-            try sources.append(b.allocator, b.dupe(file));
-        }
+  while (try iter.next()) |entry| {
+    if (entry.kind != .file) continue;
+    const file = entry.name;
+    const ext = std.fs.path.extension(file);
+    if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
+      try sources.append(b.dupe(file));
     }
+  }
 
-    return sources.toOwnedSlice(b.allocator);
+  return sources.items;
 }

build.zig.zon

@@ -1,96 +1,69 @@
 .{
-    .name = .tree_sitter,
-    .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.28.0",
-    .minimum_zig_version = "0.16.0",
-    .paths = .{
-        "build.zig",
-        "build.zig.zon",
-        "lib/src",
-        "lib/include",
-        "README.md",
-        "LICENSE",
+  .name = "tree-sitter",
+  .version = "0.25.1",
+  .paths = .{
+    "build.zig",
+    "build.zig.zon",
+    "lib/src",
+    "lib/include",
+    "README.md",
+    "LICENSE",
+  },
+  .dependencies = .{
+    .wasmtime_c_api_aarch64_android = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-android-c-api.tar.xz",
+      .hash = "12204c77979ad8291c6e395d695a824fb053ffdfeb2cc21de95fffb09f77d77188d1",
+      .lazy = true,
     },
-    .dependencies = .{
-        .wasmtime_c_api_aarch64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAIp_mQVzQOITXcYcWxYLJkvB1W1SvLlrdiU2G7fj",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAMztsgU5Aj4oI3MRHXJVe5rW72op-kT_78I3kZVM",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AANZxOwT27sdrKxDDGGKsiwtcZlHy204xAWNIgDBH",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAJL1zQW9yxC98uc60lSuVUHhH77QHTto0zwIQnBj",
-            .lazy = true,
-        },
-        .wasmtime_c_api_aarch64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-windows-c-api.zip",
-            .hash = "N-V-__8AAHRCtQU93hJcRFOgVcof3IQpRV9stT2Pp54wpJc2",
-            .lazy = true,
-        },
-        .wasmtime_c_api_armv7_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-armv7-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAJaW6gT8QdULOU0jxX4a_DOCA5YD6cxWBC8IqhQF",
-            .lazy = true,
-        },
-        .wasmtime_c_api_i686_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-linux-c-api.tar.xz",
-            .hash = "N-V-__8AANguMgVX4XMhdOVkdj4yfFKXrG8RTgZDs3nQB8J8",
-            .lazy = true,
-        },
-        .wasmtime_c_api_i686_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-windows-c-api.zip",
-            .hash = "N-V-__8AANY9ggXg4rK2_1o3EIlrCq124l5RfykPv-DPforq",
-            .lazy = true,
-        },
-        .wasmtime_c_api_riscv64gc_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-riscv64gc-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAPDtCAdQ0dD9Rs-qWl-kPr2c7L3PVsUVjwy12Iz1",
-            .lazy = true,
-        },
-        .wasmtime_c_api_s390x_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-s390x-linux-c-api.tar.xz",
-            .hash = "N-V-__8AANA3BwY1ZOoGCWCR_tTY9G1vfIX128RGVxufN3ov",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_android = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-android-c-api.tar.xz",
-            .hash = "N-V-__8AAF4AIgY0ltjevj1ybGfvMU1ErPRnNve5X1TmvCru",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_linux = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-linux-c-api.tar.xz",
-            .hash = "N-V-__8AAIR0cAbjf3DkrTbu81Oq_zociz-0lCpb5DR0lIC9",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_macos = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-macos-c-api.tar.xz",
-            .hash = "N-V-__8AAFJ4lgRgCBnYdz8-Yfc4hLve45Hv-0RICAEuCp4s",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_mingw = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-mingw-c-api.zip",
-            .hash = "N-V-__8AAMxZxQZUpp1cU8J5zgLiMNq4e4dy0hcchlQFy03J",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_musl = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-musl-c-api.tar.xz",
-            .hash = "N-V-__8AAN5pWgZrZBt8VYWkN82WjyFe3DkGcE7uLn3jpt38",
-            .lazy = true,
-        },
-        .wasmtime_c_api_x86_64_windows = .{
-            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-windows-c-api.zip",
-            .hash = "N-V-__8AAEIJkgaVHFETgakbognNUpFELuV17vpjw6NjsWhQ",
-            .lazy = true,
-        },
+    .wasmtime_c_api_aarch64_linux = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-linux-c-api.tar.xz",
+      .hash = "12203a8e3d823490186fb1e230d54f575148713088e914926305ee5678790b731bba",
+      .lazy = true,
     },
+    .wasmtime_c_api_aarch64_macos = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-macos-c-api.tar.xz",
+      .hash = "122043e8b19079b855b12674b9e3d4a28dc5c399c43b62fbeb8bdf0fdb4ef2d1d38c",
+      .lazy = true,
+    },
+    .wasmtime_c_api_riscv64gc_linux = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-riscv64gc-linux-c-api.tar.xz",
+      .hash = "12209d07031cf33271bf4b0c63df407b535cd5d65c6402bd6f80d99de439d6feb89b",
+      .lazy = true,
+    },
+    .wasmtime_c_api_s390x_linux = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-s390x-linux-c-api.tar.xz",
+      .hash = "122033f7d9b04f429063d9b2d9ac75a7a00fce02c425e578208f54ddc40edaa1e355",
+      .lazy = true,
+    },
+    .wasmtime_c_api_x86_64_android = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-android-c-api.tar.xz",
+      .hash = "122093cb33df8e09e70b2d1dc09897a0388915b942918389b10bf23f9684bdb6f047",
+      .lazy = true,
+    },
+    .wasmtime_c_api_x86_64_linux = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-linux-c-api.tar.xz",
+      .hash = "12209210346e94bf6ef8e249fa5d3f1a84f95050ed19665ac8422a15b5f2246d83af",
+      .lazy = true,
+    },
+    .wasmtime_c_api_x86_64_macos = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-macos-c-api.tar.xz",
+      .hash = "12208f875dd3a89092485762f3b184707b3cccae85a84e2ffd38c138cc3a3fd90447",
+      .lazy = true,
+    },
+    .wasmtime_c_api_x86_64_mingw = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-mingw-c-api.zip",
+      .hash = "1220bea757df3a777b6ec6322fc498e4ece20d466eedc5e2a3610b338849553cd94d",
+      .lazy = true,
+    },
+    .wasmtime_c_api_x86_64_musl = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-musl-c-api.tar.xz",
+      .hash = "1220c9596e6a63edcc3234c0611d0cbac724bf30ac9a0fbaf402c7da649b278b1322",
+      .lazy = true,
+    },
+    .wasmtime_c_api_x86_64_windows = .{
+      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-windows-c-api.zip",
+      .hash = "1220440ccb01d72989cf1a47728897a35fc8dd31673cce598f2d62c58e2c3228b0ed",
+      .lazy = true,
+    },
+  }
 }

cli/Cargo.toml

@@ -0,0 +1,81 @@
+[package]
+name = "tree-sitter-cli"
+version.workspace = true
+description = "CLI tool for developing, testing, and using Tree-sitter parsers"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+readme = "README.md"
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+include = ["build.rs", "README.md", "benches/*", "src/**"]
+
+[lints]
+workspace = true
+
+[[bin]]
+name = "tree-sitter"
+path = "src/main.rs"
+doc = false
+
+[[bench]]
+name = "benchmark"
+harness = false
+
+[features]
+wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
+
+[dependencies]
+ansi_colours.workspace = true
+anstyle.workspace = true
+anyhow.workspace = true
+bstr.workspace = true
+clap.workspace = true
+clap_complete.workspace = true
+clap_complete_nushell.workspace = true
+ctor.workspace = true
+ctrlc.workspace = true
+dialoguer.workspace = true
+filetime.workspace = true
+glob.workspace = true
+heck.workspace = true
+html-escape.workspace = true
+indexmap.workspace = true
+indoc.workspace = true
+log.workspace = true
+memchr.workspace = true
+rand.workspace = true
+regex.workspace = true
+regex-syntax.workspace = true
+rustc-hash.workspace = true
+semver.workspace = true
+serde.workspace = true
+serde_derive.workspace = true
+serde_json.workspace = true
+similar.workspace = true
+smallbitvec.workspace = true
+streaming-iterator.workspace = true
+tiny_http.workspace = true
+url.workspace = true
+walkdir.workspace = true
+wasmparser.workspace = true
+webbrowser.workspace = true
+
+tree-sitter.workspace = true
+tree-sitter-generate.workspace = true
+tree-sitter-config.workspace = true
+tree-sitter-highlight.workspace = true
+tree-sitter-loader.workspace = true
+tree-sitter-tags.workspace = true
+
+[dev-dependencies]
+encoding_rs = "0.8.35"
+widestring = "1.1.0"
+tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
+
+tempfile.workspace = true
+pretty_assertions.workspace = true
+unindent.workspace = true

cli/README.md

@@ -7,22 +7,22 @@
 [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
 [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
 
-The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`,
-`Linux`, and `Windows`.
+The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`.
 
 ### Installation
 
-You can install the `tree-sitter-cli` with [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall):
+You can install the `tree-sitter-cli` with `cargo`:
 
-```sh
-cargo binstall tree-sitter-cli
-```
-
-or you can build it from source:
 ```sh
 cargo install --locked tree-sitter-cli
 ```
 
+or with `npm`:
+
+```sh
+npm install tree-sitter-cli
+```
+
 You can also download a pre-built binary for your platform from [the releases page].
 
 ### Dependencies
@@ -34,11 +34,9 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
 
 ### Commands
 
-* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current
-  working directory. See [the documentation] for more information.
+* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information.
 
-* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory.
-  See [the documentation] for more information.
+* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information.
 
 * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.
 

cli/benches/benchmark.rs

@@ -2,15 +2,13 @@ use std::{
     collections::BTreeMap,
     env, fs,
     path::{Path, PathBuf},
+    str,
     sync::LazyLock,
     time::Instant,
 };
 
 use anyhow::Context;
-use log::info;
 use tree_sitter::{Language, Parser, Query};
-#[cfg(feature = "wasm")]
-use tree_sitter::{WasmStore, wasmtime};
 use tree_sitter_loader::{CompileConfig, Loader};
 
 include!("../src/tests/helpers/dirs.rs");
@@ -20,18 +18,14 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
 static EXAMPLE_FILTER: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
 static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
-    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
+    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
+        .map(|s| s.parse::<usize>().unwrap())
+        .unwrap_or(5)
 });
-static WASM: LazyLock<bool> = LazyLock::new(|| env::var_os("TREE_SITTER_BENCHMARK_WASM").is_some());
 static TEST_LOADER: LazyLock<Loader> =
     LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
-#[cfg(feature = "wasm")]
-static WASM_ENGINE: LazyLock<wasmtime::Engine> = LazyLock::new(Default::default);
 
-#[expect(
-    clippy::type_complexity,
-    reason = "complex map type reflects benchmark data structure"
-)]
+#[allow(clippy::type_complexity)]
 static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
     BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>,
 > = LazyLock::new(|| {
@@ -43,14 +37,22 @@ static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
             if let Ok(example_files) = fs::read_dir(dir.join("examples")) {
                 example_paths.extend(example_files.filter_map(|p| {
                     let p = p.unwrap().path();
-                    if p.is_file() { Some(p) } else { None }
+                    if p.is_file() {
+                        Some(p)
+                    } else {
+                        None
+                    }
                 }));
             }
 
             if let Ok(query_files) = fs::read_dir(dir.join("queries")) {
                 query_paths.extend(query_files.filter_map(|p| {
                     let p = p.unwrap().path();
-                    if p.is_file() { Some(p) } else { None }
+                    if p.is_file() {
+                        Some(p)
+                    } else {
+                        None
+                    }
                 }));
             }
         } else {
@@ -69,8 +71,6 @@ static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
 });
 
 fn main() {
-    tree_sitter_cli::logger::init();
-
     let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR
         .values()
         .flat_map(|(e, q)| {
@@ -81,7 +81,7 @@ fn main() {
         .max()
         .unwrap_or(0);
 
-    info!("Benchmarking with {} repetitions", *REPETITION_COUNT);
+    eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT);
 
     let mut parser = Parser::new();
     let mut all_normal_speeds = Vec::new();
@@ -92,42 +92,38 @@ fn main() {
     {
         let language_name = language_path.file_name().unwrap().to_str().unwrap();
 
-        if let Some(filter) = LANGUAGE_FILTER.as_ref()
-            && language_name != filter.as_str()
-        {
-            continue;
+        if let Some(filter) = LANGUAGE_FILTER.as_ref() {
+            if language_name != filter.as_str() {
+                continue;
+            }
         }
 
-        info!("\nLanguage: {language_name}");
-        let language = if *WASM {
-            get_wasm_language(language_name, &mut parser)
-        } else {
-            get_language(language_path)
-        };
+        eprintln!("\nLanguage: {language_name}");
+        let language = get_language(language_path);
         parser.set_language(&language).unwrap();
 
-        info!("  Constructing Queries");
+        eprintln!("  Constructing Queries");
         for path in query_paths {
-            if let Some(filter) = EXAMPLE_FILTER.as_ref()
-                && !path.to_str().unwrap().contains(filter.as_str())
-            {
-                continue;
+            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
+                if !path.to_str().unwrap().contains(filter.as_str()) {
+                    continue;
+                }
             }
 
             parse(path, max_path_length, |source| {
                 Query::new(&language, str::from_utf8(source).unwrap())
-                    .with_context(|| format!("Query file path: {}", path.display()))
+                    .with_context(|| format!("Query file path: {path:?}"))
                     .expect("Failed to parse query");
             });
         }
 
-        info!("  Parsing Valid Code:");
+        eprintln!("  Parsing Valid Code:");
         let mut normal_speeds = Vec::new();
         for example_path in example_paths {
-            if let Some(filter) = EXAMPLE_FILTER.as_ref()
-                && !example_path.to_str().unwrap().contains(filter.as_str())
-            {
-                continue;
+            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
+                if !example_path.to_str().unwrap().contains(filter.as_str()) {
+                    continue;
+                }
             }
 
             normal_speeds.push(parse(example_path, max_path_length, |code| {
@@ -135,17 +131,17 @@ fn main() {
             }));
         }
 
-        info!("  Parsing Invalid Code (mismatched languages):");
+        eprintln!("  Parsing Invalid Code (mismatched languages):");
         let mut error_speeds = Vec::new();
         for (other_language_path, (example_paths, _)) in
             EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
         {
             if other_language_path != language_path {
                 for example_path in example_paths {
-                    if let Some(filter) = EXAMPLE_FILTER.as_ref()
-                        && !example_path.to_str().unwrap().contains(filter.as_str())
-                    {
-                        continue;
+                    if let Some(filter) = EXAMPLE_FILTER.as_ref() {
+                        if !example_path.to_str().unwrap().contains(filter.as_str()) {
+                            continue;
+                        }
                     }
 
                     error_speeds.push(parse(example_path, max_path_length, |code| {
@@ -156,30 +152,30 @@ fn main() {
         }
 
         if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) {
-            info!("  Average Speed (normal): {average_normal} bytes/ms");
-            info!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+            eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
+            eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
         }
 
         if let Some((average_error, worst_error)) = aggregate(&error_speeds) {
-            info!("  Average Speed (errors): {average_error} bytes/ms");
-            info!("  Worst Speed (errors):   {worst_error} bytes/ms");
+            eprintln!("  Average Speed (errors): {average_error} bytes/ms");
+            eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
         }
 
         all_normal_speeds.extend(normal_speeds);
         all_error_speeds.extend(error_speeds);
     }
 
-    info!("\n  Overall");
+    eprintln!("\n  Overall");
     if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) {
-        info!("  Average Speed (normal): {average_normal} bytes/ms");
-        info!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+        eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
+        eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
     }
 
     if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) {
-        info!("  Average Speed (errors): {average_error} bytes/ms");
-        info!("  Worst Speed (errors):   {worst_error} bytes/ms");
+        eprintln!("  Average Speed (errors): {average_error} bytes/ms");
+        eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
     }
-    info!("");
+    eprintln!();
 }
 
 fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
@@ -198,8 +194,14 @@ fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
 }
 
 fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize {
+    eprint!(
+        "    {:width$}\t",
+        path.file_name().unwrap().to_str().unwrap(),
+        width = max_path_length
+    );
+
     let source_code = fs::read(path)
-        .with_context(|| format!("Failed to read {}", path.display()))
+        .with_context(|| format!("Failed to read {path:?}"))
         .unwrap();
     let time = Instant::now();
     for _ in 0..*REPETITION_COUNT {
@@ -208,9 +210,8 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
     let duration = time.elapsed() / (*REPETITION_COUNT as u32);
     let duration_ns = duration.as_nanos();
     let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns;
-    info!(
-        "    {:max_path_length$}\ttime {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
-        path.file_name().unwrap().to_str().unwrap(),
+    eprintln!(
+        "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
         (duration_ns as f64) / 1e6,
     );
     speed as usize
@@ -220,35 +221,6 @@ fn get_language(path: &Path) -> Language {
     let src_path = GRAMMARS_DIR.join(path).join("src");
     TEST_LOADER
         .load_language_at_path(CompileConfig::new(&src_path, None, None))
-        .with_context(|| format!("Failed to load language at path {}", src_path.display()))
+        .with_context(|| format!("Failed to load language at path {src_path:?}"))
         .unwrap()
 }
-
-#[cfg(feature = "wasm")]
-fn get_wasm_language(language_name: &str, parser: &mut Parser) -> Language {
-    let wasm_language_name = language_name.replace('-', "_");
-    let wasm_path = ROOT_DIR
-        .join("target")
-        .join("release")
-        .join(format!("tree-sitter-{language_name}.wasm"));
-    let wasm = fs::read(&wasm_path)
-        .with_context(|| {
-            format!(
-                "Failed to read {}. Generate Wasm fixtures with `cargo xtask generate-fixtures --wasm`",
-                wasm_path.display()
-            )
-        })
-        .unwrap();
-    let mut store = WasmStore::new(&WASM_ENGINE).expect("Failed to create Wasm store");
-    let language = store
-        .load_language(&wasm_language_name, &wasm)
-        .with_context(|| format!("Failed to load Wasm language at {}", wasm_path.display()))
-        .unwrap();
-    parser.set_wasm_store(store).unwrap();
-    language
-}
-
-#[cfg(not(feature = "wasm"))]
-fn get_wasm_language(_language_name: &str, _parser: &mut Parser) -> Language {
-    panic!("Wasm benchmarking requires the `wasm` feature");
-}

cli/build.rs

@@ -52,14 +52,16 @@ fn main() {
 
 fn web_playground_files_present() -> bool {
     let paths = [
-        "../../docs/src/assets/js/playground.js",
-        "../../lib/binding_web/web-tree-sitter.js",
-        "../../lib/binding_web/web-tree-sitter.wasm",
+        "../docs/src/assets/js/playground.js",
+        "../lib/binding_web/tree-sitter.js",
+        "../lib/binding_web/tree-sitter.wasm",
     ];
 
     paths.iter().all(|p| Path::new(p).exists())
 }
 
+// When updating this function, don't forget to also update generate/build.rs which has a
+// near-identical function.
 fn read_git_sha() -> Option<String> {
     let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
 

cli/config/Cargo.toml

@@ -5,22 +5,18 @@ description = "User configuration of tree-sitter's command line programs"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
+readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
-documentation = "https://docs.rs/tree-sitter-config"
 license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 
-[lib]
-path = "src/tree_sitter_config.rs"
-
 [lints]
 workspace = true
 
 [dependencies]
+anyhow.workspace = true
 etcetera.workspace = true
-log.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-thiserror.workspace = true

cli/config/src/lib.rs

@@ -1,64 +1,11 @@
-#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
+#![doc = include_str!("../README.md")]
 
-use std::{
-    env, fs,
-    path::{Path, PathBuf},
-};
+use std::{env, fs, path::PathBuf};
 
+use anyhow::{Context, Result};
 use etcetera::BaseStrategy as _;
-use log::warn;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use thiserror::Error;
-
-pub type ConfigResult<T> = Result<T, ConfigError>;
-
-#[derive(Debug, Error)]
-pub enum ConfigError {
-    #[error("Bad JSON config {0} -- {1}")]
-    ConfigRead(String, serde_json::Error),
-    #[error(transparent)]
-    HomeDir(#[from] etcetera::HomeDirError),
-    #[error(transparent)]
-    IO(IoError),
-    #[error(transparent)]
-    Serialization(#[from] serde_json::Error),
-}
-
-#[derive(Debug, Error)]
-pub struct IoError {
-    pub error: std::io::Error,
-    pub path: Option<PathBuf>,
-}
-
-impl PartialEq for IoError {
-    fn eq(&self, other: &Self) -> bool {
-        self.path == other.path
-            && self.error.kind() == other.error.kind()
-            && self.error.raw_os_error() == other.error.raw_os_error()
-    }
-}
-
-impl Eq for IoError {}
-
-impl IoError {
-    fn new(error: std::io::Error, path: Option<&Path>) -> Self {
-        Self {
-            error,
-            path: path.map(Path::to_path_buf),
-        }
-    }
-}
-
-impl std::fmt::Display for IoError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.error)?;
-        if let Some(ref path) = self.path {
-            write!(f, " ({})", path.display())?;
-        }
-        Ok(())
-    }
-}
 
 /// Holds the contents of tree-sitter's configuration file.
 ///
@@ -75,7 +22,7 @@ pub struct Config {
 }
 
 impl Config {
-    pub fn find_config_file() -> ConfigResult<Option<PathBuf>> {
+    pub fn find_config_file() -> Result<Option<PathBuf>> {
         if let Ok(path) = env::var("TREE_SITTER_DIR") {
             let mut path = PathBuf::from(path);
             path.push("config.json");
@@ -98,14 +45,10 @@ impl Config {
                 .join("tree-sitter")
                 .join("config.json");
             if legacy_apple_path.is_file() {
-                let xdg_dir = xdg_path.parent().unwrap();
-                fs::create_dir_all(xdg_dir)
-                    .map_err(|e| ConfigError::IO(IoError::new(e, Some(xdg_dir))))?;
-                fs::rename(&legacy_apple_path, &xdg_path).map_err(|e| {
-                    ConfigError::IO(IoError::new(e, Some(legacy_apple_path.as_path())))
-                })?;
-                warn!(
-                    "Your config.json file has been automatically migrated from \"{}\" to \"{}\"",
+                fs::create_dir_all(xdg_path.parent().unwrap())?;
+                fs::rename(&legacy_apple_path, &xdg_path)?;
+                println!(
+                    "Warning: your config.json file has been automatically migrated from \"{}\" to \"{}\"",
                     legacy_apple_path.display(),
                     xdg_path.display()
                 );
@@ -123,7 +66,7 @@ impl Config {
         Ok(None)
     }
 
-    fn xdg_config_file() -> ConfigResult<PathBuf> {
+    fn xdg_config_file() -> Result<PathBuf> {
         let xdg_path = etcetera::choose_base_strategy()?
             .config_dir()
             .join("tree-sitter")
@@ -140,7 +83,7 @@ impl Config {
     ///     [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy)
     ///   - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
     ///     its configuration
-    pub fn load(path: Option<PathBuf>) -> ConfigResult<Self> {
+    pub fn load(path: Option<PathBuf>) -> Result<Self> {
         let location = if let Some(path) = path {
             path
         } else if let Some(path) = Self::find_config_file()? {
@@ -150,9 +93,9 @@ impl Config {
         };
 
         let content = fs::read_to_string(&location)
-            .map_err(|e| ConfigError::IO(IoError::new(e, Some(location.as_path()))))?;
+            .with_context(|| format!("Failed to read {}", location.to_string_lossy()))?;
         let config = serde_json::from_str(&content)
-            .map_err(|e| ConfigError::ConfigRead(location.to_string_lossy().to_string(), e))?;
+            .with_context(|| format!("Bad JSON config {}", location.to_string_lossy()))?;
         Ok(Self { location, config })
     }
 
@@ -162,7 +105,7 @@ impl Config {
     /// disk.
     ///
     /// (Note that this is typically only done by the `tree-sitter init-config` command.)
-    pub fn initial() -> ConfigResult<Self> {
+    pub fn initial() -> Result<Self> {
         let location = if let Ok(path) = env::var("TREE_SITTER_DIR") {
             let mut path = PathBuf::from(path);
             path.push("config.json");
@@ -175,20 +118,17 @@ impl Config {
     }
 
     /// Saves this configuration to the file that it was originally loaded from.
-    pub fn save(&self) -> ConfigResult<()> {
+    pub fn save(&self) -> Result<()> {
         let json = serde_json::to_string_pretty(&self.config)?;
-        let config_dir = self.location.parent().unwrap();
-        fs::create_dir_all(config_dir)
-            .map_err(|e| ConfigError::IO(IoError::new(e, Some(config_dir))))?;
-        fs::write(&self.location, json)
-            .map_err(|e| ConfigError::IO(IoError::new(e, Some(self.location.as_path()))))?;
+        fs::create_dir_all(self.location.parent().unwrap())?;
+        fs::write(&self.location, json)?;
         Ok(())
     }
 
     /// Parses a component-specific configuration from the configuration file.  The type `C` must
     /// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON
     /// object, and must only include the fields relevant to that component.
-    pub fn get<C>(&self) -> ConfigResult<C>
+    pub fn get<C>(&self) -> Result<C>
     where
         C: for<'de> Deserialize<'de>,
     {
@@ -199,7 +139,7 @@ impl Config {
     /// Adds a component-specific configuration to the configuration file.  The type `C` must be
     /// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and
     /// must only include the fields relevant to that component.
-    pub fn add<C>(&mut self, config: C) -> ConfigResult<()>
+    pub fn add<C>(&mut self, config: C) -> Result<()>
     where
         C: Serialize,
     {

cli/eslint/package.json

@@ -4,8 +4,7 @@
   "description": "Eslint configuration for Tree-sitter grammar files",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
-    "directory": "crates/cli/eslint"
+    "url": "git+https://github.com/tree-sitter/tree-sitter.git"
   },
   "license": "MIT",
   "author": "Amaan Qureshi <amaanq12@gmail.com>",
@@ -17,13 +16,9 @@
     "tree-sitter"
   ],
   "dependencies": {
-    "eslint-plugin-jsdoc": "^62.7.0"
+    "eslint-plugin-jsdoc": "^50.2.4"
   },
   "peerDependencies": {
     "eslint": ">= 9"
-  },
-  "scripts": {
-    "prepack": "cp ../../../LICENSE .",
-    "postpack": "rm LICENSE"
   }
 }

cli/generate/Cargo.toml

@@ -5,40 +5,30 @@ description = "Library for generating C source code from a tree-sitter grammar"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
+readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
-documentation = "https://docs.rs/tree-sitter-generate"
 license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 
-[lib]
-path = "src/generate.rs"
-
 [lints]
 workspace = true
 
-[features]
-default = [ "qjs-rt" ]
-load = [ "dep:semver" ]
-qjs-rt = [ "load", "rquickjs", "pathdiff" ]
-
 [dependencies]
-bitflags = "2.11.1"
-dunce = "1.0.5"
-hashbrown.workspace = true
+anyhow.workspace = true
+heck.workspace = true
 indexmap.workspace = true
+indoc.workspace = true
 log.workspace = true
-pathdiff = { optional = true, version = "0.2.3" }
 regex.workspace = true
 regex-syntax.workspace = true
-rquickjs = { features = [ "bindgen", "loader", "macro", "phf" ], optional = true, version = "0.13" }
 rustc-hash.workspace = true
-semver = { optional = true, workspace = true }
+semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+smallbitvec.workspace = true
 thiserror.workspace = true
-topological-sort.workspace = true
+url.workspace = true
 
-[dev-dependencies]
-tempfile.workspace = true
+tree-sitter.workspace = true

cli/generate/build.rs

@@ -0,0 +1,32 @@
+use std::{env, path::PathBuf, process::Command};
+
+fn main() {
+    if let Some(git_sha) = read_git_sha() {
+        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
+    }
+}
+
+// This is copied from the build.rs in parent directory. This should be updated if the
+// parent build.rs gets fixes.
+fn read_git_sha() -> Option<String> {
+    let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+
+    if !crate_path
+        .parent()?
+        .parent()
+        .is_some_and(|p| p.join(".git").exists())
+    {
+        return None;
+    }
+
+    Command::new("git")
+        .args(["rev-parse", "HEAD"])
+        .current_dir(crate_path)
+        .output()
+        .map_or(None, |output| {
+            if !output.status.success() {
+                return None;
+            }
+            Some(String::from_utf8_lossy(&output.stdout).to_string())
+        })
+}

cli/generate/src/build_tables/build_lex_table.rs

@@ -1,11 +1,9 @@
 use std::{
-    collections::{VecDeque, hash_map::Entry},
+    collections::{hash_map::Entry, HashMap, VecDeque},
     mem,
 };
 
-use rustc_hash::FxHashMap;
-
-use log::debug;
+use log::info;
 
 use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap};
 use crate::{
@@ -13,7 +11,7 @@ use crate::{
     grammars::{LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor},
     rules::{Symbol, TokenSet},
-    tables::{AdvanceAction, LexState, LexStateId, LexTable, ParseStateId, ParseTable},
+    tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable},
 };
 
 pub const LARGE_CHARACTER_RANGE_COUNT: usize = 8;
@@ -49,7 +47,7 @@ pub fn build_lex_table(
             .chain(state.reserved_words.iter())
             .filter_map(|token| {
                 if token.is_terminal() {
-                    if keywords.contains(token) {
+                    if keywords.contains(&token) {
                         syntax_grammar.word_token
                     } else {
                         Some(token)
@@ -67,17 +65,18 @@ pub fn build_lex_table(
             if merge_token_set(
                 &mut entry.0,
                 &tokens,
+                lexical_grammar,
                 token_conflict_map,
                 coincident_token_index,
             ) {
                 did_merge = true;
-                entry.1.push(i as u32);
+                entry.1.push(i);
                 break;
             }
         }
 
         if !did_merge {
-            parse_state_ids_by_token_set.push((tokens, vec![i as u32]));
+            parse_state_ids_by_token_set.push((tokens, vec![i]));
         }
     }
 
@@ -85,7 +84,7 @@ pub fn build_lex_table(
     for (tokens, parse_state_ids) in parse_state_ids_by_token_set {
         let lex_state_id = builder.add_state_for_tokens(&tokens);
         for id in parse_state_ids {
-            parse_table.states[id as usize].lex_state_id = lex_state_id;
+            parse_table.states[id].lex_state_id = lex_state_id;
         }
     }
 
@@ -131,7 +130,7 @@ pub fn build_lex_table(
 }
 
 struct QueueEntry {
-    state_id: LexStateId,
+    state_id: usize,
     nfa_states: Vec<u32>,
     eof_valid: bool,
 }
@@ -141,7 +140,7 @@ struct LexTableBuilder<'a> {
     cursor: NfaCursor<'a>,
     table: LexTable,
     state_queue: VecDeque<QueueEntry>,
-    state_ids_by_nfa_state_set: FxHashMap<(Vec<u32>, bool), LexStateId>,
+    state_ids_by_nfa_state_set: HashMap<(Vec<u32>, bool), usize>,
 }
 
 impl<'a> LexTableBuilder<'a> {
@@ -151,7 +150,7 @@ impl<'a> LexTableBuilder<'a> {
             cursor: NfaCursor::new(&lexical_grammar.nfa, vec![]),
             table: LexTable::default(),
             state_queue: VecDeque::new(),
-            state_ids_by_nfa_state_set: FxHashMap::default(),
+            state_ids_by_nfa_state_set: HashMap::new(),
         }
     }
 
@@ -161,13 +160,13 @@ impl<'a> LexTableBuilder<'a> {
         self.state_ids_by_nfa_state_set.clear();
     }
 
-    fn add_state_for_tokens(&mut self, tokens: &TokenSet) -> LexStateId {
+    fn add_state_for_tokens(&mut self, tokens: &TokenSet) -> usize {
         let mut eof_valid = false;
         let nfa_states = tokens
             .iter()
             .filter_map(|token| {
                 if token.is_terminal() {
-                    Some(self.lexical_grammar.variables[token.index as usize].start_state)
+                    Some(self.lexical_grammar.variables[token.index].start_state)
                 } else {
                     eof_valid = true;
                     None
@@ -177,11 +176,11 @@ impl<'a> LexTableBuilder<'a> {
         let (state_id, is_new) = self.add_state(nfa_states, eof_valid);
 
         if is_new {
-            debug!(
+            info!(
                 "entry point state: {state_id}, tokens: {:?}",
                 tokens
                     .iter()
-                    .map(|t| &self.lexical_grammar.variables[t.index as usize].name)
+                    .map(|t| &self.lexical_grammar.variables[t.index].name)
                     .collect::<Vec<_>>()
             );
         }
@@ -197,7 +196,7 @@ impl<'a> LexTableBuilder<'a> {
         state_id
     }
 
-    fn add_state(&mut self, nfa_states: Vec<u32>, eof_valid: bool) -> (LexStateId, bool) {
+    fn add_state(&mut self, nfa_states: Vec<u32>, eof_valid: bool) -> (usize, bool) {
         self.cursor.reset(nfa_states);
         match self
             .state_ids_by_nfa_state_set
@@ -205,7 +204,7 @@ impl<'a> LexTableBuilder<'a> {
         {
             Entry::Occupied(o) => (*o.get(), false),
             Entry::Vacant(v) => {
-                let state_id = self.table.states.len() as u32;
+                let state_id = self.table.states.len();
                 self.table.states.push(LexState::default());
                 self.state_queue.push_back(QueueEntry {
                     state_id,
@@ -218,52 +217,53 @@ impl<'a> LexTableBuilder<'a> {
         }
     }
 
-    fn populate_state(&mut self, state_id: LexStateId, nfa_states: Vec<u32>, eof_valid: bool) {
+    fn populate_state(&mut self, state_id: usize, nfa_states: Vec<u32>, eof_valid: bool) {
         self.cursor.force_reset(nfa_states);
 
         // The EOF state is represented as an empty list of NFA states.
         let mut completion = None;
         for (id, prec) in self.cursor.completions() {
-            if let Some((prev_id, prev_precedence)) = completion
-                && TokenConflictMap::prefer_token(
+            if let Some((prev_id, prev_precedence)) = completion {
+                if TokenConflictMap::prefer_token(
                     self.lexical_grammar,
                     (prev_precedence, prev_id),
                     (prec, id),
-                )
-            {
-                continue;
+                ) {
+                    continue;
+                }
             }
             completion = Some((id, prec));
         }
 
-        let (transitions, has_sep) = self.cursor.transitions_and_any_sep();
+        let transitions = self.cursor.transitions();
+        let has_sep = self.cursor.transition_chars().any(|(_, sep)| sep);
 
         // If EOF is a valid lookahead token, add a transition predicated on the null
         // character that leads to the empty set of NFA states.
         if eof_valid {
             let (next_state_id, _) = self.add_state(Vec::new(), false);
-            self.table.states[state_id as usize].eof_action = Some(AdvanceAction {
+            self.table.states[state_id].eof_action = Some(AdvanceAction {
                 state: next_state_id,
                 in_main_token: true,
             });
         }
 
         for transition in transitions {
-            if let Some((completed_id, completed_precedence)) = completion
-                && !TokenConflictMap::prefer_transition(
+            if let Some((completed_id, completed_precedence)) = completion {
+                if !TokenConflictMap::prefer_transition(
                     self.lexical_grammar,
                     &transition,
                     completed_id,
                     completed_precedence,
                     has_sep,
-                )
-            {
-                continue;
+                ) {
+                    continue;
+                }
             }
 
             let (next_state_id, _) =
                 self.add_state(transition.states, eof_valid && transition.is_separator);
-            self.table.states[state_id as usize].advance_actions.push((
+            self.table.states[state_id].advance_actions.push((
                 transition.characters,
                 AdvanceAction {
                     state: next_state_id,
@@ -273,78 +273,41 @@ impl<'a> LexTableBuilder<'a> {
         }
 
         if let Some((complete_id, _)) = completion {
-            self.table.states[state_id as usize].accept_action =
-                Some(Symbol::terminal(complete_id));
+            self.table.states[state_id].accept_action = Some(Symbol::terminal(complete_id));
         } else if self.cursor.state_ids.is_empty() {
-            self.table.states[state_id as usize].accept_action = Some(Symbol::end());
+            self.table.states[state_id].accept_action = Some(Symbol::end());
         }
     }
 }
 
-fn check_token_conflicts(
-    i: usize,
-    set_without_terminal: &TokenSet,
-    token_conflict_map: &TokenConflictMap,
-    coincident_token_index: &CoincidentTokenIndex,
-) -> bool {
-    let wpr = token_conflict_map.row_words;
-    let row_start = i * wpr;
-    let set_bits = set_without_terminal.terminal_bits_words();
-
-    // Does terminal i conflict with or match-prefix any terminal in the set?
-    let conflict_row = &token_conflict_map.conflict_or_prefix_bits[row_start..row_start + wpr];
-    for (&c, &s) in conflict_row.iter().zip(set_bits) {
-        if c & s != 0 {
-            return true;
-        }
-    }
-
-    // Does terminal i overlap (in either direction) with any non-coincident terminal in the set?
-    let overlap_row = &token_conflict_map.overlap_either_bits[row_start..row_start + wpr];
-    let coincident_row = &coincident_token_index.row_bits[row_start..row_start + wpr];
-    for ((&o, &s), &c) in overlap_row.iter().zip(set_bits).zip(coincident_row) {
-        if o & s & !c != 0 {
-            return true;
-        }
-    }
-
-    false
-}
-
 fn merge_token_set(
     tokens: &mut TokenSet,
     other: &TokenSet,
+    lexical_grammar: &LexicalGrammar,
     token_conflict_map: &TokenConflictMap,
     coincident_token_index: &CoincidentTokenIndex,
 ) -> bool {
-    if tokens
-        .terminals()
-        .filter(|terminal| !other.contains_terminal(terminal.index as usize))
-        .any(|terminal| {
-            check_token_conflicts(
-                terminal.index as usize,
-                other,
-                token_conflict_map,
-                coincident_token_index,
-            )
-        })
-    {
-        return false;
-    }
+    for i in 0..lexical_grammar.variables.len() {
+        let symbol = Symbol::terminal(i);
+        let set_without_terminal = match (tokens.contains_terminal(i), other.contains_terminal(i)) {
+            (true, false) => other,
+            (false, true) => tokens,
+            _ => continue,
+        };
 
-    if other
-        .terminals()
-        .filter(|terminal| !tokens.contains_terminal(terminal.index as usize))
-        .any(|terminal| {
-            check_token_conflicts(
-                terminal.index as usize,
-                tokens,
-                token_conflict_map,
-                coincident_token_index,
-            )
-        })
-    {
-        return false;
+        for existing_token in set_without_terminal.terminals() {
+            if token_conflict_map.does_conflict(i, existing_token.index)
+                || token_conflict_map.does_match_prefix(i, existing_token.index)
+            {
+                return false;
+            }
+            if !coincident_token_index.contains(symbol, existing_token)
+                && (token_conflict_map.does_overlap(existing_token.index, i)
+                    || token_conflict_map.does_overlap(i, existing_token.index))
+            {
+                return false;
+            }
+        }
     }
 
     tokens.insert_all(other);
@@ -354,7 +317,7 @@ fn merge_token_set(
 fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
     // Initially group the states by their accept action and their
     // valid lookahead characters.
-    let mut state_ids_by_signature = FxHashMap::default();
+    let mut state_ids_by_signature = HashMap::new();
     for (i, state) in table.states.iter().enumerate() {
         let signature = (
             i == 0,
@@ -369,7 +332,7 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         state_ids_by_signature
             .entry(signature)
             .or_insert(Vec::new())
-            .push(i as u32);
+            .push(i);
     }
     let mut state_ids_by_group_id = state_ids_by_signature
         .into_iter()
@@ -382,10 +345,10 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
         .unwrap();
     state_ids_by_group_id.swap(error_group_index, 0);
 
-    let mut group_ids_by_state_id = vec![0u32; table.states.len()];
+    let mut group_ids_by_state_id = vec![0; table.states.len()];
     for (group_id, state_ids) in state_ids_by_group_id.iter().enumerate() {
         for state_id in state_ids {
-            group_ids_by_state_id[*state_id as usize] = group_id as u32;
+            group_ids_by_state_id[*state_id] = group_id;
         }
     }
 
@@ -400,35 +363,30 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) {
     let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
     for state_ids in &state_ids_by_group_id {
         let mut new_state = LexState::default();
-        mem::swap(&mut new_state, &mut table.states[state_ids[0] as usize]);
+        mem::swap(&mut new_state, &mut table.states[state_ids[0]]);
 
         for (_, advance_action) in &mut new_state.advance_actions {
-            advance_action.state = group_ids_by_state_id[advance_action.state as usize];
+            advance_action.state = group_ids_by_state_id[advance_action.state];
         }
         if let Some(eof_action) = &mut new_state.eof_action {
-            eof_action.state = group_ids_by_state_id[eof_action.state as usize];
+            eof_action.state = group_ids_by_state_id[eof_action.state];
         }
         new_states.push(new_state);
     }
 
     for state in &mut parse_table.states {
-        state.lex_state_id = group_ids_by_state_id[state.lex_state_id as usize];
+        state.lex_state_id = group_ids_by_state_id[state.lex_state_id];
     }
 
     table.states = new_states;
 }
 
-fn lex_states_differ(
-    left: &LexState,
-    right: &LexState,
-    group_ids_by_state_id: &[LexStateId],
-) -> bool {
+fn lex_states_differ(left: &LexState, right: &LexState, group_ids_by_state_id: &[usize]) -> bool {
     left.advance_actions
         .iter()
         .zip(right.advance_actions.iter())
         .any(|(left, right)| {
-            group_ids_by_state_id[left.1.state as usize]
-                != group_ids_by_state_id[right.1.state as usize]
+            group_ids_by_state_id[left.1.state] != group_ids_by_state_id[right.1.state]
         })
 }
 
@@ -438,9 +396,9 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
     old_ids_by_new_id[1..].sort_by_key(|id| &table.states[*id]);
 
     // Get the inverse mapping
-    let mut new_ids_by_old_id = vec![0u32; old_ids_by_new_id.len()];
+    let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
     for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
-        new_ids_by_old_id[*old_id] = id as u32;
+        new_ids_by_old_id[*old_id] = id;
     }
 
     // Reorder the parse states and update their references to reflect
@@ -451,10 +409,10 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
             let mut state = LexState::default();
             mem::swap(&mut state, &mut table.states[*old_id]);
             for (_, advance_action) in &mut state.advance_actions {
-                advance_action.state = new_ids_by_old_id[advance_action.state as usize];
+                advance_action.state = new_ids_by_old_id[advance_action.state];
             }
             if let Some(eof_action) = &mut state.eof_action {
-                eof_action.state = new_ids_by_old_id[eof_action.state as usize];
+                eof_action.state = new_ids_by_old_id[eof_action.state];
             }
             state
         })
@@ -462,6 +420,6 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) {
 
     // Update the parse table's lex state references
     for state in &mut parse_table.states {
-        state.lex_state_id = new_ids_by_old_id[state.lex_state_id as usize];
+        state.lex_state_id = new_ids_by_old_id[state.lex_state_id];
     }
 }

cli/generate/src/build_tables/build_parse_table.rs

@@ -1,12 +1,12 @@
 use std::{
     cmp::Ordering,
-    collections::{BTreeMap, BTreeSet, VecDeque},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
     hash::BuildHasherDefault,
 };
 
-use indexmap::{IndexMap, map::Entry};
-use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
-use serde::{Deserialize, Serialize};
+use indexmap::{map::Entry, IndexMap};
+use rustc_hash::FxHasher;
+use serde::Serialize;
 use thiserror::Error;
 
 use super::{
@@ -14,15 +14,12 @@ use super::{
     item_set_builder::ParseItemSetBuilder,
 };
 use crate::{
-    Diagnostic,
-    build_tables::item::{LookaheadSetPool, START_PRODUCTION_ID, prec_display},
     grammars::{LexicalGrammar, PrecedenceEntry, ReservedWordSetId, SyntaxGrammar, VariableType},
     node_types::VariableInfo,
     rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
-    strpool::StrPool,
     tables::{
-        ActionList, ActionListPool, FieldLocation, GotoAction, ParseAction, ParseState,
-        ParseStateId, ParseTable, ParseTableEntry, ProductionInfo, ProductionInfoId,
+        FieldLocation, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable,
+        ParseTableEntry, ProductionInfo, ProductionInfoId,
     },
 };
 
@@ -31,19 +28,7 @@ use crate::{
 type SymbolSequence = Vec<Symbol>;
 
 type AuxiliarySymbolSequence = Vec<AuxiliarySymbolInfo>;
-
-pub struct ParseStateInfo<'a> {
-    pub preceding_symbols_by_id: Vec<SymbolSequence>,
-    item_sets_by_ids: IndexMap<ParseItemSet<'a>, ParseStateId, BuildHasherDefault<FxHasher>>,
-    pub lookaheads: LookaheadSetPool,
-}
-
-impl<'a> ParseStateInfo<'a> {
-    #[must_use]
-    pub fn item_set(&self, id: ParseStateId) -> &ParseItemSet<'a> {
-        self.item_sets_by_ids.get_index(id as usize).unwrap().0
-    }
-}
+pub type ParseStateInfo<'a> = (SymbolSequence, ParseItemSet<'a>);
 
 #[derive(Clone, PartialEq)]
 struct AuxiliarySymbolInfo {
@@ -70,20 +55,18 @@ struct ParseTableBuilder<'a> {
     syntax_grammar: &'a SyntaxGrammar,
     lexical_grammar: &'a LexicalGrammar,
     variable_info: &'a [VariableInfo],
-    core_ids_by_core: FxHashMap<ParseItemSetCore<'a>, u32>,
+    core_ids_by_core: HashMap<ParseItemSetCore<'a>, usize>,
     state_ids_by_item_set: IndexMap<ParseItemSet<'a>, ParseStateId, BuildHasherDefault<FxHasher>>,
-    preceding_symbols_by_id: Vec<SymbolSequence>,
-    production_info_ids_by_prod_id: Vec<Option<ProductionInfoId>>,
+    parse_state_info_by_id: Vec<ParseStateInfo<'a>>,
     parse_state_queue: VecDeque<ParseStateQueueEntry>,
-    non_terminal_extra_states: Vec<(Symbol, ParseStateId)>,
-    actual_conflicts: FxHashSet<Vec<Symbol>>,
-    parse_table: ParseTable<ParseTableEntry>,
-    str_pool: &'a StrPool,
+    non_terminal_extra_states: Vec<(Symbol, usize)>,
+    actual_conflicts: HashSet<Vec<Symbol>>,
+    parse_table: ParseTable,
 }
 
 pub type BuildTableResult<T> = Result<T, ParseTableBuilderError>;
 
-#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Error, Serialize)]
 pub enum ParseTableBuilderError {
     #[error("Unresolved conflict for symbol sequence:\n\n{0}")]
     Conflict(#[from] ConflictError),
@@ -93,11 +76,9 @@ pub enum ParseTableBuilderError {
         "The non-terminal rule `{0}` is used in a non-terminal `extra` rule, which is not allowed."
     )]
     ImproperNonTerminalExtra(String),
-    #[error("State count `{0}` exceeds the max value {max}.", max=u16::MAX)]
-    StateCount(usize),
 }
 
-#[derive(Default, Debug, Serialize, Error, Deserialize, PartialEq, Eq)]
+#[derive(Default, Debug, Serialize)]
 pub struct ConflictError {
     pub symbol_sequence: Vec<String>,
     pub conflicting_lookahead: String,
@@ -105,7 +86,7 @@ pub struct ConflictError {
     pub possible_resolutions: Vec<Resolution>,
 }
 
-#[derive(Default, Debug, Serialize, Error, Deserialize, PartialEq, Eq)]
+#[derive(Default, Debug, Serialize)]
 pub struct Interpretation {
     pub preceding_symbols: Vec<String>,
     pub variable_name: String,
@@ -115,17 +96,16 @@ pub struct Interpretation {
     pub conflicting_lookahead: String,
     pub precedence: Option<String>,
     pub associativity: Option<String>,
-    pub requires_eof_lookahead: bool,
 }
 
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Serialize)]
 pub enum Resolution {
     Precedence { symbols: Vec<String> },
     Associativity { symbols: Vec<String> },
     AddConflict { symbols: Vec<String> },
 }
 
-#[derive(Debug, Serialize, Deserialize, Error, PartialEq, Eq)]
+#[derive(Debug, Serialize)]
 pub struct AmbiguousExtraError {
     pub parent_symbols: Vec<String>,
 }
@@ -143,21 +123,16 @@ impl std::fmt::Display for ConflictError {
             .iter()
             .map(|i| {
                 let line = i.to_string();
-                let mut annotations = Vec::new();
-                if let (Some(precedence), Some(associativity)) = (&i.precedence, &i.associativity) {
-                    annotations.push(format!(
+                let prec_line = if let (Some(precedence), Some(associativity)) =
+                    (&i.precedence, &i.associativity)
+                {
+                    Some(format!(
                         "(precedence: {precedence}, associativity: {associativity})",
-                    ));
-                } else if let Some(precedence) = &i.precedence {
-                    annotations.push(format!("(precedence: {precedence})"));
-                }
-                if i.requires_eof_lookahead {
-                    annotations.push("(reduces only at end of input)".to_string());
-                }
-                let prec_line = if annotations.is_empty() {
-                    None
+                    ))
                 } else {
-                    Some(annotations.join("  "))
+                    i.precedence
+                        .as_ref()
+                        .map(|precedence| format!("(precedence: {precedence})"))
                 };
 
                 (line, prec_line)
@@ -260,13 +235,15 @@ impl std::fmt::Display for AmbiguousExtraError {
     }
 }
 
+impl std::error::Error for ConflictError {}
+impl std::error::Error for AmbiguousExtraError {}
+
 impl<'a> ParseTableBuilder<'a> {
     fn new(
         syntax_grammar: &'a SyntaxGrammar,
         lexical_grammar: &'a LexicalGrammar,
         item_set_builder: ParseItemSetBuilder<'a>,
         variable_info: &'a [VariableInfo],
-        str_pool: &'a StrPool,
     ) -> Self {
         Self {
             syntax_grammar,
@@ -275,27 +252,21 @@ impl<'a> ParseTableBuilder<'a> {
             variable_info,
             non_terminal_extra_states: Vec::new(),
             state_ids_by_item_set: IndexMap::default(),
-            core_ids_by_core: FxHashMap::default(),
-            preceding_symbols_by_id: Vec::new(),
-            production_info_ids_by_prod_id: vec![None; syntax_grammar.productions.len()],
+            core_ids_by_core: HashMap::new(),
+            parse_state_info_by_id: Vec::new(),
             parse_state_queue: VecDeque::new(),
             actual_conflicts: syntax_grammar.expected_conflicts.iter().cloned().collect(),
             parse_table: ParseTable {
                 states: Vec::new(),
-                action_lists: ActionListPool::default(),
                 symbols: Vec::new(),
                 external_lex_states: Vec::new(),
                 production_infos: Vec::new(),
                 max_aliased_production_length: 1,
             },
-            str_pool,
         }
     }
 
-    fn build(
-        mut self,
-        diagnostics: &mut Vec<Diagnostic>,
-    ) -> BuildTableResult<(ParseTable<ParseTableEntry>, ParseStateInfo<'a>)> {
+    fn build(mut self) -> BuildTableResult<(ParseTable, Vec<ParseStateInfo<'a>>)> {
         // Ensure that the empty alias sequence has index 0.
         self.parse_table
             .production_infos
@@ -305,14 +276,13 @@ impl<'a> ParseTableBuilder<'a> {
         self.add_parse_state(&Vec::new(), &Vec::new(), ParseItemSet::default());
 
         // Add the starting state at index 1.
-        let end_lookaheads = self.item_set_builder.lookaheads.singleton(Symbol::end());
         self.add_parse_state(
             &Vec::new(),
             &Vec::new(),
             ParseItemSet {
                 entries: vec![ParseItemSetEntry {
-                    item: ParseItem::start(self.item_set_builder.key_map),
-                    lookaheads: end_lookaheads,
+                    item: ParseItem::start(),
+                    lookaheads: std::iter::once(Symbol::end()).collect(),
                     following_reserved_word_set: ReservedWordSetId::default(),
                 }],
             },
@@ -326,61 +296,42 @@ impl<'a> ParseTableBuilder<'a> {
             .iter()
             .filter(|s| s.is_non_terminal())
         {
-            for prod_id in self
-                .syntax_grammar
-                .variable_prod_ids(extra_non_terminal.index as usize)
-            {
-                let production = self.syntax_grammar.production(prod_id);
-                let entry = non_terminal_extra_item_sets_by_first_terminal
+            let variable = &self.syntax_grammar.variables[extra_non_terminal.index];
+            for production in &variable.productions {
+                non_terminal_extra_item_sets_by_first_terminal
                     .entry(production.first_symbol().unwrap())
                     .or_insert_with(ParseItemSet::default)
                     .insert(ParseItem {
-                        variable_index: extra_non_terminal.index,
-                        prod_id,
+                        variable_index: extra_non_terminal.index as u32,
+                        production,
                         step_index: 1,
-                        keys: self.item_set_builder.key_map.keys_for(prod_id),
                         has_preceding_inherited_fields: false,
-                    });
-                entry.lookaheads = self
-                    .item_set_builder
+                    })
                     .lookaheads
-                    .insert(entry.lookaheads, Symbol::end_of_nonterminal_extra());
+                    .insert(Symbol::end_of_nonterminal_extra());
             }
         }
 
-        let non_terminal_sets_len = non_terminal_extra_item_sets_by_first_terminal.len();
-        self.non_terminal_extra_states
-            .reserve(non_terminal_sets_len);
-        self.preceding_symbols_by_id.reserve(non_terminal_sets_len);
-        self.parse_table.states.reserve(non_terminal_sets_len);
-        self.parse_state_queue.reserve(non_terminal_sets_len);
         // Add a state for each starting terminal of a non-terminal extra rule.
         for (terminal, item_set) in non_terminal_extra_item_sets_by_first_terminal {
             if terminal.is_non_terminal() {
                 Err(ParseTableBuilderError::ImproperNonTerminalExtra(
-                    self.symbol_name(terminal),
+                    self.symbol_name(&terminal),
                 ))?;
             }
 
-            // Add the parse state, and *then* push the terminal and the state id into the
-            // list of nonterminal extra states
-            let state_id = self.add_parse_state(&Vec::new(), &Vec::new(), item_set);
-            self.non_terminal_extra_states.push((terminal, state_id));
+            self.non_terminal_extra_states
+                .push((terminal, self.parse_table.states.len()));
+            self.add_parse_state(&Vec::new(), &Vec::new(), item_set);
         }
 
         while let Some(entry) = self.parse_state_queue.pop_front() {
-            // The dedup-map key is each state's kernel (the GOTO result, pre closure).
-            // Two states are identical iff their kernels match.
-            let kernel = self
-                .state_ids_by_item_set
-                .get_index(entry.state_id as usize)
-                // Invariant: `state_id` is the map's insertion index
-                .unwrap()
-                .0;
-            let item_set = self.item_set_builder.transitive_closure(kernel);
+            let item_set = self
+                .item_set_builder
+                .transitive_closure(&self.parse_state_info_by_id[entry.state_id].1);
 
             self.add_actions(
-                self.preceding_symbols_by_id[entry.state_id as usize].clone(),
+                self.parse_state_info_by_id[entry.state_id].0.clone(),
                 entry.preceding_auxiliary_symbols,
                 entry.state_id,
                 &item_set,
@@ -388,23 +339,20 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         if !self.actual_conflicts.is_empty() {
-            let mut conflicts = self
-                .actual_conflicts
-                .iter()
-                .map(|conf| conf.iter().map(|&s| self.symbol_name(s)).collect())
-                .collect::<Vec<_>>();
-            conflicts.sort_unstable();
-            diagnostics.push(Diagnostic::UnnecessaryConflicts(conflicts));
+            println!("Warning: unnecessary conflicts");
+            for conflict in &self.actual_conflicts {
+                println!(
+                    "  {}",
+                    conflict
+                        .iter()
+                        .map(|symbol| format!("`{}`", self.symbol_name(symbol)))
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                );
+            }
         }
 
-        Ok((
-            self.parse_table,
-            ParseStateInfo {
-                preceding_symbols_by_id: self.preceding_symbols_by_id,
-                item_sets_by_ids: self.state_ids_by_item_set,
-                lookaheads: self.item_set_builder.lookaheads,
-            },
-        ))
+        Ok((self.parse_table, self.parse_state_info_by_id))
     }
 
     fn add_parse_state(
@@ -422,11 +370,12 @@ impl<'a> ParseTableBuilder<'a> {
             // parse states to populate.
             Entry::Vacant(v) => {
                 let core = v.key().core();
-                let core_count = self.core_ids_by_core.len() as u32;
+                let core_count = self.core_ids_by_core.len();
                 let core_id = *self.core_ids_by_core.entry(core).or_insert(core_count);
 
-                let state_id = self.parse_table.states.len() as u32;
-                self.preceding_symbols_by_id.push(preceding_symbols.clone());
+                let state_id = self.parse_table.states.len();
+                self.parse_state_info_by_id
+                    .push((preceding_symbols.clone(), v.key().clone()));
 
                 self.parse_table.states.push(ParseState {
                     id: state_id,
@@ -436,7 +385,6 @@ impl<'a> ParseTableBuilder<'a> {
                     nonterminal_entries: IndexMap::default(),
                     reserved_words: TokenSet::default(),
                     core_id,
-                    has_eof_gated_reduce: false,
                 });
                 self.parse_state_queue.push_back(ParseStateQueueEntry {
                     state_id,
@@ -458,11 +406,7 @@ impl<'a> ParseTableBuilder<'a> {
         let mut terminal_successors = BTreeMap::new();
         let mut non_terminal_successors = BTreeMap::new();
         let mut lookaheads_with_conflicts = TokenSet::new();
-        let mut reduction_infos = FxHashMap::<Symbol, ReductionInfo>::default();
-
-        // `get_auxiliary_node_info` scans every entry in `item_set`, and the same auxiliary
-        // symbol typically appears across many entries in a state. Memoize per symbol.
-        let mut aux_node_info = FxHashMap::<Symbol, AuxiliarySymbolInfo>::default();
+        let mut reduction_infos = HashMap::<Symbol, ReductionInfo>::new();
 
         // Each item in the item set contributes to either or a Shift action or a Reduce
         // action in this state.
@@ -475,23 +419,17 @@ impl<'a> ParseTableBuilder<'a> {
             // If the item is unfinished, then this state has a transition for the item's
             // next symbol. Advance the item to its next step and insert the resulting
             // item into the successor item set.
-            if let Some(next_symbol) = item.symbol(self.syntax_grammar) {
+            if let Some(next_symbol) = item.symbol() {
                 let mut successor = item.successor();
                 let successor_set = if next_symbol.is_non_terminal() {
-                    let variable = &self.syntax_grammar.variables[next_symbol.index as usize];
+                    let variable = &self.syntax_grammar.variables[next_symbol.index];
 
                     // Keep track of where auxiliary non-terminals (repeat symbols) are
                     // used within visible symbols. This information may be needed later
                     // for conflict resolution.
                     if variable.is_auxiliary() {
-                        preceding_auxiliary_symbols.push(
-                            aux_node_info
-                                .entry(next_symbol)
-                                .or_insert_with(|| {
-                                    self.get_auxiliary_node_info(item_set, next_symbol)
-                                })
-                                .clone(),
-                        );
+                        preceding_auxiliary_symbols
+                            .push(self.get_auxiliary_node_info(item_set, next_symbol));
                     }
 
                     // For most parse items, the symbols associated with the preceding children
@@ -504,9 +442,7 @@ impl<'a> ParseTableBuilder<'a> {
                     // of its preceding children need to be taken into account when comparing
                     // it with other items.
                     if variable.is_hidden()
-                        && !self.variable_info[next_symbol.index as usize]
-                            .fields
-                            .is_empty()
+                        && !self.variable_info[next_symbol.index].fields.is_empty()
                     {
                         successor.has_preceding_inherited_fields = true;
                     }
@@ -520,10 +456,7 @@ impl<'a> ParseTableBuilder<'a> {
                         .or_insert_with(ParseItemSet::default)
                 };
                 let successor_entry = successor_set.insert(successor);
-                successor_entry.lookaheads = self
-                    .item_set_builder
-                    .lookaheads
-                    .union(successor_entry.lookaheads, *lookaheads);
+                successor_entry.lookaheads.insert_all(lookaheads);
                 successor_entry.following_reserved_word_set = successor_entry
                     .following_reserved_word_set
                     .max(*reserved_lookaheads);
@@ -535,37 +468,18 @@ impl<'a> ParseTableBuilder<'a> {
                 let action = if item.is_augmented() {
                     ParseAction::Accept
                 } else {
-                    // These values are narrowed to u16 to reduce the size of
-                    // ParseAction. No real-world grammar approaches these limits.
-                    debug_assert!(
-                        u16::try_from(item.step_index).is_ok(),
-                        "production step count exceeds u16::MAX"
-                    );
-                    let production_id = self.get_production_id(item);
-                    debug_assert!(
-                        u16::try_from(production_id).is_ok(),
-                        "production info id exceeds u16::MAX"
-                    );
                     ParseAction::Reduce {
                         symbol,
-                        child_count: item.step_index as u16,
-                        dynamic_precedence: item.production(self.syntax_grammar).dynamic_precedence,
-                        production_id: production_id as u16,
+                        child_count: item.step_index as usize,
+                        dynamic_precedence: item.production.dynamic_precedence,
+                        production_id: self.get_production_id(item),
                     }
                 };
 
-                let precedence = item.precedence(self.syntax_grammar);
-                let associativity = item.associativity(self.syntax_grammar);
-                if item.production(self.syntax_grammar).requires_eof_lookahead {
-                    self.parse_table.states[state_id as usize].has_eof_gated_reduce = true;
-                }
-                for lookahead in self.item_set_builder.lookaheads.get(*lookaheads).iter() {
-                    if item.production(self.syntax_grammar).requires_eof_lookahead
-                        && lookahead != Symbol::end()
-                    {
-                        continue;
-                    }
-                    let table_entry = self.parse_table.states[state_id as usize]
+                let precedence = item.precedence();
+                let associativity = item.associativity();
+                for lookahead in lookaheads.iter() {
+                    let table_entry = self.parse_table.states[state_id]
                         .terminal_entries
                         .entry(lookahead)
                         .or_insert_with(ParseTableEntry::new);
@@ -581,30 +495,24 @@ impl<'a> ParseTableBuilder<'a> {
                             self.syntax_grammar,
                             precedence,
                             &[symbol],
-                            reduction_info.precedence,
+                            &reduction_info.precedence,
                             &reduction_info.symbols,
                         ) {
                             Ordering::Greater => {
                                 table_entry.actions.clear();
                                 table_entry.actions.push(action);
-                                lookaheads_with_conflicts.remove(lookahead);
+                                lookaheads_with_conflicts.remove(&lookahead);
                                 *reduction_info = ReductionInfo::default();
                             }
-                            // Two items that reduce identically build the same tree, so
-                            // there is nothing for the user to resolve. Precedence is
-                            // still compared first, because an item that only repeats an
-                            // existing action can still outrank it and clear the entry.
                             Ordering::Equal => {
-                                if !table_entry.actions.contains(&action) {
-                                    table_entry.actions.push(action);
-                                    lookaheads_with_conflicts.insert(lookahead);
-                                }
+                                table_entry.actions.push(action);
+                                lookaheads_with_conflicts.insert(lookahead);
                             }
                             Ordering::Less => continue,
                         }
                     }
 
-                    reduction_info.precedence = precedence;
+                    reduction_info.precedence.clone_from(precedence);
                     if let Err(i) = reduction_info.symbols.binary_search(&symbol) {
                         reduction_info.symbols.insert(i, symbol);
                     }
@@ -631,13 +539,13 @@ impl<'a> ParseTableBuilder<'a> {
             );
             preceding_symbols.pop();
 
-            let entry = self.parse_table.states[state_id as usize]
+            let entry = self.parse_table.states[state_id]
                 .terminal_entries
                 .entry(symbol);
-            if let Entry::Occupied(e) = &entry
-                && !e.get().actions.is_empty()
-            {
-                lookaheads_with_conflicts.insert(symbol);
+            if let Entry::Occupied(e) = &entry {
+                if !e.get().actions.is_empty() {
+                    lookaheads_with_conflicts.insert(symbol);
+                }
             }
 
             entry
@@ -657,7 +565,7 @@ impl<'a> ParseTableBuilder<'a> {
                 next_item_set,
             );
             preceding_symbols.pop();
-            self.parse_table.states[state_id as usize]
+            self.parse_table.states[state_id]
                 .nonterminal_entries
                 .insert(symbol, GotoAction::Goto(next_state_id));
         }
@@ -679,7 +587,7 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         // Add actions for the grammar's `extra` symbols.
-        let state = &mut self.parse_table.states[state_id as usize];
+        let state = &mut self.parse_table.states[state_id];
         let is_end_of_non_terminal_extra = state.is_end_of_non_terminal_extra();
 
         // If this state represents the end of a non-terminal extra rule, then make sure that
@@ -697,13 +605,13 @@ impl<'a> ParseTableBuilder<'a> {
                             None
                         }
                     })
-                    .collect::<FxHashSet<_>>();
+                    .collect::<HashSet<_>>();
                 let parent_symbol_names = parent_symbols
                     .iter()
                     .map(|&variable_index| {
-                        self.str_pool
-                            .resolve(self.syntax_grammar.variables[variable_index as usize].name)
-                            .to_string()
+                        self.syntax_grammar.variables[variable_index as usize]
+                            .name
+                            .clone()
                     })
                     .collect::<Vec<_>>();
 
@@ -720,10 +628,10 @@ impl<'a> ParseTableBuilder<'a> {
                     .entry(*terminal)
                     .or_insert(ParseTableEntry {
                         reusable: true,
-                        actions: ActionList::One(ParseAction::Shift {
+                        actions: vec![ParseAction::Shift {
                             state: *state_id,
                             is_repetition: false,
-                        }),
+                        }],
                     });
             }
 
@@ -741,7 +649,7 @@ impl<'a> ParseTableBuilder<'a> {
                         .entry(*extra_token)
                         .or_insert(ParseTableEntry {
                             reusable: true,
-                            actions: ActionList::One(ParseAction::ShiftExtra),
+                            actions: vec![ParseAction::ShiftExtra],
                         });
                 }
             }
@@ -752,18 +660,13 @@ impl<'a> ParseTableBuilder<'a> {
                 .entries
                 .iter()
                 .filter_map(|entry| {
-                    if let Some(next_step) = entry.item.step(self.syntax_grammar) {
-                        if next_step.symbol() == keyword_capture_token {
-                            Some(ReservedWordSetId(u32::from(next_step.reserved)))
+                    if let Some(next_step) = entry.item.step() {
+                        if next_step.symbol == keyword_capture_token {
+                            Some(next_step.reserved_word_set_id)
                         } else {
                             None
                         }
-                    } else if self
-                        .item_set_builder
-                        .lookaheads
-                        .get(entry.lookaheads)
-                        .contains(keyword_capture_token)
-                    {
+                    } else if entry.lookaheads.contains(&keyword_capture_token) {
                         Some(entry.following_reserved_word_set)
                     } else {
                         None
@@ -772,7 +675,7 @@ impl<'a> ParseTableBuilder<'a> {
                 .max();
             if let Some(reserved_word_set_id) = reserved_word_set_id {
                 state.reserved_words =
-                    self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0 as usize].clone();
+                    self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0].clone();
             }
         }
 
@@ -788,7 +691,7 @@ impl<'a> ParseTableBuilder<'a> {
         conflicting_lookahead: Symbol,
         reduction_info: &ReductionInfo,
     ) -> BuildTableResult<()> {
-        let entry = self.parse_table.states[state_id as usize]
+        let entry = self.parse_table.states[state_id]
             .terminal_entries
             .get_mut(&conflicting_lookahead)
             .unwrap();
@@ -800,37 +703,32 @@ impl<'a> ParseTableBuilder<'a> {
         // REDUCE-REDUCE conflicts where all actions have the *same*
         // precedence, and there can still be SHIFT/REDUCE conflicts.
         let mut considered_associativity = false;
-        let mut shift_precedence = Vec::<(Precedence, Symbol)>::new();
+        let mut shift_precedence = Vec::<(&Precedence, Symbol)>::new();
         let mut conflicting_items = BTreeSet::new();
         for ParseItemSetEntry {
             item, lookaheads, ..
         } in &item_set.entries
         {
-            if let Some(step) = item.step(self.syntax_grammar) {
+            if let Some(step) = item.step() {
                 if item.step_index > 0
                     && self
                         .item_set_builder
-                        .first_set(step.symbol())
-                        .contains(conflicting_lookahead)
+                        .first_set(&step.symbol)
+                        .contains(&conflicting_lookahead)
                 {
                     if item.variable_index != u32::MAX {
                         conflicting_items.insert(item);
                     }
 
                     let p = (
-                        item.precedence(self.syntax_grammar),
+                        item.precedence(),
                         Symbol::non_terminal(item.variable_index as usize),
                     );
                     if let Err(i) = shift_precedence.binary_search(&p) {
                         shift_precedence.insert(i, p);
                     }
                 }
-            } else if self
-                .item_set_builder
-                .lookaheads
-                .get(*lookaheads)
-                .contains(conflicting_lookahead)
-                && item.variable_index != u32::MAX
+            } else if lookaheads.contains(&conflicting_lookahead) && item.variable_index != u32::MAX
             {
                 conflicting_items.insert(item);
             }
@@ -855,51 +753,30 @@ impl<'a> ParseTableBuilder<'a> {
 
             // If the SHIFT action has higher precedence, remove all the REDUCE actions.
             let mut shift_is_less = false;
-            let mut shift_is_equal = false;
             let mut shift_is_more = false;
             for p in shift_precedence {
                 match Self::compare_precedence(
                     self.syntax_grammar,
                     p.0,
                     &[p.1],
-                    reduction_info.precedence,
+                    &reduction_info.precedence,
                     &reduction_info.symbols,
                 ) {
                     Ordering::Greater => shift_is_more = true,
                     Ordering::Less => shift_is_less = true,
-                    Ordering::Equal => shift_is_equal = true,
+                    Ordering::Equal => {}
                 }
             }
 
             if shift_is_more && !shift_is_less {
-                entry.actions.keep_last();
+                entry.actions.drain(0..entry.actions.len() - 1);
             }
             // If the REDUCE actions have higher precedence, remove the SHIFT action.
             else if shift_is_less && !shift_is_more {
-                // Exception: if one SHIFT interpretation ties the REDUCE actions in
-                // precedence while another has lower precedence, and the REDUCE
-                // actions are purely right associative, honor that right
-                // associativity by shifting rather than reducing. The
-                // lower-precedence interpretation coexists with the tying one, so on
-                // its own it must not force a REDUCE that would flip the tie to left
-                // associative.
-                if shift_is_equal
-                    && matches!(
-                        (
-                            reduction_info.has_left_assoc,
-                            reduction_info.has_non_assoc,
-                            reduction_info.has_right_assoc,
-                        ),
-                        (false, false, true)
-                    )
-                {
-                    entry.actions.keep_last();
-                } else {
-                    entry.actions.pop();
-                    conflicting_items.retain(|item| item.is_done());
-                }
+                entry.actions.pop();
+                conflicting_items.retain(|item| item.is_done());
             }
-            // If the SHIFT and REDUCE actions have the same precedence, consider
+            // If the SHIFT and REDUCE actions have the same predence, consider
             // the REDUCE actions' associativity.
             else if !shift_is_less && !shift_is_more {
                 considered_associativity = true;
@@ -916,7 +793,7 @@ impl<'a> ParseTableBuilder<'a> {
                         conflicting_items.retain(|item| item.is_done());
                     }
                     (false, false, true) => {
-                        entry.actions.keep_last();
+                        entry.actions.drain(0..entry.actions.len() - 1);
                     }
                     _ => {}
                 }
@@ -924,7 +801,7 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         // If all of the actions but one have been eliminated, then there's no problem.
-        let entry = self.parse_table.states[state_id as usize]
+        let entry = self.parse_table.states[state_id]
             .terminal_entries
             .get_mut(&conflicting_lookahead)
             .unwrap();
@@ -936,7 +813,7 @@ impl<'a> ParseTableBuilder<'a> {
         let mut actual_conflict = Vec::new();
         for item in &conflicting_items {
             let symbol = Symbol::non_terminal(item.variable_index as usize);
-            if self.syntax_grammar.variables[symbol.index as usize].is_auxiliary() {
+            if self.syntax_grammar.variables[symbol.index].is_auxiliary() {
                 actual_conflict.extend(
                     preceding_auxiliary_symbols
                         .iter()
@@ -969,12 +846,12 @@ impl<'a> ParseTableBuilder<'a> {
         }
 
         let mut conflict_error = ConflictError::default();
-        for &symbol in preceding_symbols {
+        for symbol in preceding_symbols {
             conflict_error
                 .symbol_sequence
-                .push(self.symbol_name(symbol));
+                .push(self.symbol_name(symbol).to_string());
         }
-        conflict_error.conflicting_lookahead = self.symbol_name(conflicting_lookahead);
+        conflict_error.conflicting_lookahead = self.symbol_name(&conflicting_lookahead).to_string();
 
         let interpretations = conflicting_items
             .iter()
@@ -982,32 +859,26 @@ impl<'a> ParseTableBuilder<'a> {
                 let preceding_symbols = preceding_symbols
                     .iter()
                     .take(preceding_symbols.len() - item.step_index as usize)
-                    .map(|&symbol| self.symbol_name(symbol))
+                    .map(|symbol| self.symbol_name(symbol).to_string())
                     .collect::<Vec<_>>();
 
-                let variable_name = self
-                    .str_pool
-                    .resolve(self.syntax_grammar.variables[item.variable_index as usize].name)
-                    .to_string();
+                let variable_name = self.syntax_grammar.variables[item.variable_index as usize]
+                    .name
+                    .clone();
 
                 let production_step_symbols = item
-                    .production(self.syntax_grammar)
+                    .production
                     .steps
                     .iter()
-                    .map(|step| self.symbol_name(step.symbol()))
+                    .map(|step| self.symbol_name(&step.symbol).to_string())
                     .collect::<Vec<_>>();
 
-                let precedence = match item.precedence(self.syntax_grammar) {
+                let precedence = match item.precedence() {
                     Precedence::None => None,
-                    _ => Some(prec_display(
-                        item.precedence(self.syntax_grammar),
-                        self.str_pool,
-                    )),
+                    _ => Some(item.precedence().to_string()),
                 };
 
-                let associativity = item
-                    .associativity(self.syntax_grammar)
-                    .map(|assoc| format!("{assoc:?}"));
+                let associativity = item.associativity().map(|assoc| format!("{assoc:?}"));
 
                 Interpretation {
                     preceding_symbols,
@@ -1015,12 +886,9 @@ impl<'a> ParseTableBuilder<'a> {
                     production_step_symbols,
                     step_index: item.step_index,
                     done: item.is_done(),
-                    conflicting_lookahead: self.symbol_name(conflicting_lookahead),
+                    conflicting_lookahead: self.symbol_name(&conflicting_lookahead).to_string(),
                     precedence,
                     associativity,
-                    requires_eof_lookahead: item
-                        .production(self.syntax_grammar)
-                        .requires_eof_lookahead,
                 }
             })
             .collect::<Vec<_>>();
@@ -1040,13 +908,13 @@ impl<'a> ParseTableBuilder<'a> {
 
         let get_rule_names = |items: &[&ParseItem]| -> Vec<String> {
             let mut last_rule_id = None;
-            let mut result = Vec::with_capacity(items.len());
+            let mut result = Vec::new();
             for item in items {
                 if last_rule_id == Some(item.variable_index) {
                     continue;
                 }
                 last_rule_id = Some(item.variable_index);
-                result.push(self.symbol_name(Symbol::non_terminal(item.variable_index as usize)));
+                result.push(self.symbol_name(&Symbol::non_terminal(item.variable_index as usize)));
             }
 
             result
@@ -1061,7 +929,7 @@ impl<'a> ParseTableBuilder<'a> {
             }
 
             for item in &reduce_items {
-                let name = self.symbol_name(Symbol::non_terminal(item.variable_index as usize));
+                let name = self.symbol_name(&Symbol::non_terminal(item.variable_index as usize));
                 conflict_error
                     .possible_resolutions
                     .push(Resolution::Precedence {
@@ -1082,7 +950,7 @@ impl<'a> ParseTableBuilder<'a> {
             .push(Resolution::AddConflict {
                 symbols: actual_conflict
                     .iter()
-                    .map(|&s| self.symbol_name(s))
+                    .map(|s| self.symbol_name(s))
                     .collect(),
             });
 
@@ -1093,33 +961,33 @@ impl<'a> ParseTableBuilder<'a> {
 
     fn compare_precedence(
         grammar: &SyntaxGrammar,
-        left: Precedence,
+        left: &Precedence,
         left_symbols: &[Symbol],
-        right: Precedence,
+        right: &Precedence,
         right_symbols: &[Symbol],
     ) -> Ordering {
         let precedence_entry_matches =
-            |entry: &PrecedenceEntry, precedence: Precedence, symbols: &[Symbol]| -> bool {
+            |entry: &PrecedenceEntry, precedence: &Precedence, symbols: &[Symbol]| -> bool {
                 match entry {
                     PrecedenceEntry::Name(n) => {
                         if let Precedence::Name(p) = precedence {
-                            *n == p
+                            n == p
                         } else {
                             false
                         }
                     }
                     PrecedenceEntry::Symbol(n) => symbols
                         .iter()
-                        .any(|s| &grammar.variables[s.index as usize].name == n),
+                        .any(|s| &grammar.variables[s.index].name == n),
                 }
             };
 
         match (left, right) {
             // Integer precedences can be compared to other integer precedences,
             // and to the default precedence, which is zero.
-            (Precedence::Integer(l), Precedence::Integer(r)) if l != 0 || r != 0 => l.cmp(&r),
-            (Precedence::Integer(l), Precedence::None) if l != 0 => l.cmp(&0),
-            (Precedence::None, Precedence::Integer(r)) if r != 0 => 0.cmp(&r),
+            (Precedence::Integer(l), Precedence::Integer(r)) if *l != 0 || *r != 0 => l.cmp(r),
+            (Precedence::Integer(l), Precedence::None) if *l != 0 => l.cmp(&0),
+            (Precedence::None, Precedence::Integer(r)) if *r != 0 => 0.cmp(r),
 
             // Named precedences can be compared to other named precedences.
             _ => grammar
@@ -1159,7 +1027,7 @@ impl<'a> ParseTableBuilder<'a> {
             .iter()
             .filter_map(|ParseItemSetEntry { item, .. }| {
                 let variable_index = item.variable_index as usize;
-                if item.symbol(self.syntax_grammar) == Some(symbol)
+                if item.symbol() == Some(symbol)
                     && !self.syntax_grammar.variables[variable_index].is_auxiliary()
                 {
                     Some(Symbol::non_terminal(variable_index))
@@ -1175,46 +1043,37 @@ impl<'a> ParseTableBuilder<'a> {
     }
 
     fn get_production_id(&mut self, item: &ParseItem) -> ProductionInfoId {
-        debug_assert_ne!(item.prod_id, START_PRODUCTION_ID);
-        if let Some(id) = self.production_info_ids_by_prod_id[item.prod_id as usize] {
-            return id;
-        }
         let mut production_info = ProductionInfo {
             alias_sequence: Vec::new(),
             field_map: BTreeMap::new(),
         };
 
-        for (i, step) in item
-            .production(self.syntax_grammar)
-            .steps
-            .iter()
-            .enumerate()
-        {
-            production_info.alias_sequence.push(step.alias());
-            if let Some(field_name) = step.field() {
+        for (i, step) in item.production.steps.iter().enumerate() {
+            production_info.alias_sequence.push(step.alias.clone());
+            if let Some(field_name) = &step.field_name {
                 production_info
                     .field_map
-                    .entry(field_name)
+                    .entry(field_name.clone())
                     .or_default()
                     .push(FieldLocation {
-                        index: i as u32,
+                        index: i,
                         inherited: false,
                     });
             }
 
-            if step.symbol().kind == SymbolType::NonTerminal
-                && !self.syntax_grammar.variables[step.symbol().index as usize]
+            if step.symbol.kind == SymbolType::NonTerminal
+                && !self.syntax_grammar.variables[step.symbol.index]
                     .kind
                     .is_visible()
             {
-                let info = &self.variable_info[step.symbol().index as usize];
-                for &field_name in info.fields.keys() {
+                let info = &self.variable_info[step.symbol.index];
+                for field_name in info.fields.keys() {
                     production_info
                         .field_map
-                        .entry(field_name)
+                        .entry(field_name.clone())
                         .or_default()
                         .push(FieldLocation {
-                            index: i as u32,
+                            index: i,
                             inherited: true,
                         });
                 }
@@ -1225,14 +1084,11 @@ impl<'a> ParseTableBuilder<'a> {
             production_info.alias_sequence.pop();
         }
 
-        if item.production(self.syntax_grammar).steps.len()
-            > self.parse_table.max_aliased_production_length
-        {
-            self.parse_table.max_aliased_production_length =
-                item.production(self.syntax_grammar).steps.len();
+        if item.production.steps.len() > self.parse_table.max_aliased_production_length {
+            self.parse_table.max_aliased_production_length = item.production.steps.len();
         }
 
-        let id = if let Some(index) = self
+        if let Some(index) = self
             .parse_table
             .production_infos
             .iter()
@@ -1242,28 +1098,22 @@ impl<'a> ParseTableBuilder<'a> {
         } else {
             self.parse_table.production_infos.push(production_info);
             self.parse_table.production_infos.len() - 1
-        } as ProductionInfoId;
-        self.production_info_ids_by_prod_id[item.prod_id as usize] = Some(id);
-        id
+        }
     }
 
-    fn symbol_name(&self, symbol: Symbol) -> String {
+    fn symbol_name(&self, symbol: &Symbol) -> String {
         match symbol.kind {
             SymbolType::End | SymbolType::EndOfNonTerminalExtra => "EOF".to_string(),
-            SymbolType::External => self
-                .str_pool
-                .resolve(self.syntax_grammar.external_tokens[symbol.index as usize].name)
-                .to_string(),
-            SymbolType::NonTerminal => self
-                .str_pool
-                .resolve(self.syntax_grammar.variables[symbol.index as usize].name)
-                .to_string(),
+            SymbolType::External => self.syntax_grammar.external_tokens[symbol.index]
+                .name
+                .clone(),
+            SymbolType::NonTerminal => self.syntax_grammar.variables[symbol.index].name.clone(),
             SymbolType::Terminal => {
-                let variable = &self.lexical_grammar.variables[symbol.index as usize];
+                let variable = &self.lexical_grammar.variables[symbol.index];
                 if variable.kind == VariableType::Named {
-                    self.str_pool.resolve(variable.name).to_string()
+                    variable.name.clone()
                 } else {
-                    format!("'{}'", self.str_pool.resolve(variable.name))
+                    format!("'{}'", variable.name)
                 }
             }
         }
@@ -1275,15 +1125,12 @@ pub fn build_parse_table<'a>(
     lexical_grammar: &'a LexicalGrammar,
     item_set_builder: ParseItemSetBuilder<'a>,
     variable_info: &'a [VariableInfo],
-    str_pool: &'a StrPool,
-    diagnostics: &mut Vec<Diagnostic>,
-) -> BuildTableResult<(ParseTable<ParseTableEntry>, ParseStateInfo<'a>)> {
+) -> BuildTableResult<(ParseTable, Vec<ParseStateInfo<'a>>)> {
     ParseTableBuilder::new(
         syntax_grammar,
         lexical_grammar,
         item_set_builder,
         variable_info,
-        str_pool,
     )
-    .build(diagnostics)
+    .build()
 }

cli/generate/src/build_tables/coincident_tokens.rs

@@ -0,0 +1,79 @@
+use std::fmt;
+
+use crate::{
+    grammars::LexicalGrammar,
+    rules::Symbol,
+    tables::{ParseStateId, ParseTable},
+};
+
+pub struct CoincidentTokenIndex<'a> {
+    entries: Vec<Vec<ParseStateId>>,
+    grammar: &'a LexicalGrammar,
+    n: usize,
+}
+
+impl<'a> CoincidentTokenIndex<'a> {
+    pub fn new(table: &ParseTable, lexical_grammar: &'a LexicalGrammar) -> Self {
+        let n = lexical_grammar.variables.len();
+        let mut result = Self {
+            n,
+            grammar: lexical_grammar,
+            entries: vec![Vec::new(); n * n],
+        };
+        for (i, state) in table.states.iter().enumerate() {
+            for symbol in state.terminal_entries.keys() {
+                if symbol.is_terminal() {
+                    for other_symbol in state.terminal_entries.keys() {
+                        if other_symbol.is_terminal() {
+                            let index = result.index(symbol.index, other_symbol.index);
+                            if result.entries[index].last().copied() != Some(i) {
+                                result.entries[index].push(i);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        result
+    }
+
+    pub fn states_with(&self, a: Symbol, b: Symbol) -> &[ParseStateId] {
+        &self.entries[self.index(a.index, b.index)]
+    }
+
+    pub fn contains(&self, a: Symbol, b: Symbol) -> bool {
+        !self.entries[self.index(a.index, b.index)].is_empty()
+    }
+
+    #[must_use]
+    const fn index(&self, a: usize, b: usize) -> usize {
+        if a < b {
+            a * self.n + b
+        } else {
+            b * self.n + a
+        }
+    }
+}
+
+impl fmt::Debug for CoincidentTokenIndex<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "CoincidentTokenIndex {{")?;
+
+        writeln!(f, "  entries: {{")?;
+        for i in 0..self.n {
+            writeln!(f, "    {}: {{", self.grammar.variables[i].name)?;
+            for j in 0..self.n {
+                writeln!(
+                    f,
+                    "      {}: {:?},",
+                    self.grammar.variables[j].name,
+                    self.entries[self.index(i, j)].len()
+                )?;
+            }
+            writeln!(f, "    }},")?;
+        }
+        write!(f, "  }},")?;
+        write!(f, "}}")?;
+        Ok(())
+    }
+}

cli/generate/src/build_tables/item.rs

@@ -0,0 +1,462 @@
+use std::{
+    cmp::Ordering,
+    fmt,
+    hash::{Hash, Hasher},
+    sync::LazyLock,
+};
+
+use crate::{
+    grammars::{
+        LexicalGrammar, Production, ProductionStep, ReservedWordSetId, SyntaxGrammar,
+        NO_RESERVED_WORDS,
+    },
+    rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
+};
+
+static START_PRODUCTION: LazyLock<Production> = LazyLock::new(|| Production {
+    dynamic_precedence: 0,
+    steps: vec![ProductionStep {
+        symbol: Symbol {
+            index: 0,
+            kind: SymbolType::NonTerminal,
+        },
+        precedence: Precedence::None,
+        associativity: None,
+        alias: None,
+        field_name: None,
+        reserved_word_set_id: NO_RESERVED_WORDS,
+    }],
+});
+
+/// A [`ParseItem`] represents an in-progress match of a single production in a grammar.
+#[derive(Clone, Copy, Debug)]
+pub struct ParseItem<'a> {
+    /// The index of the parent rule within the grammar.
+    pub variable_index: u32,
+    /// The number of symbols that have already been matched.
+    pub step_index: u32,
+    /// The production being matched.
+    pub production: &'a Production,
+    /// A boolean indicating whether any of the already-matched children were
+    /// hidden nodes and had fields. Ordinarily, a parse item's behavior is not
+    /// affected by the symbols of its preceding children; it only needs to
+    /// keep track of their fields and aliases.
+    ///
+    /// Take for example these two items:
+    ///   X -> a b • c
+    ///   X -> a g • c
+    ///
+    /// They can be considered equivalent, for the purposes of parse table
+    /// generation, because they entail the same actions. But if this flag is
+    /// true, then the item's set of inherited fields may depend on the specific
+    /// symbols of its preceding children.
+    pub has_preceding_inherited_fields: bool,
+}
+
+/// A [`ParseItemSet`] represents a set of in-progress matches of productions in a
+/// grammar, and for each in-progress match, a set of "lookaheads" - tokens that
+/// are allowed to *follow* the in-progress rule. This object corresponds directly
+/// to a state in the final parse table.
+#[derive(Clone, Debug, PartialEq, Eq, Default)]
+pub struct ParseItemSet<'a> {
+    pub entries: Vec<ParseItemSetEntry<'a>>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ParseItemSetEntry<'a> {
+    pub item: ParseItem<'a>,
+    pub lookaheads: TokenSet,
+    pub following_reserved_word_set: ReservedWordSetId,
+}
+
+/// A [`ParseItemSetCore`] is like a [`ParseItemSet`], but without the lookahead
+/// information. Parse states with the same core are candidates for merging.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ParseItemSetCore<'a> {
+    pub entries: Vec<ParseItem<'a>>,
+}
+
+pub struct ParseItemDisplay<'a>(
+    pub &'a ParseItem<'a>,
+    pub &'a SyntaxGrammar,
+    pub &'a LexicalGrammar,
+);
+
+pub struct TokenSetDisplay<'a>(
+    pub &'a TokenSet,
+    pub &'a SyntaxGrammar,
+    pub &'a LexicalGrammar,
+);
+
+pub struct ParseItemSetDisplay<'a>(
+    pub &'a ParseItemSet<'a>,
+    pub &'a SyntaxGrammar,
+    pub &'a LexicalGrammar,
+);
+
+impl<'a> ParseItem<'a> {
+    pub fn start() -> Self {
+        ParseItem {
+            variable_index: u32::MAX,
+            production: &START_PRODUCTION,
+            step_index: 0,
+            has_preceding_inherited_fields: false,
+        }
+    }
+
+    pub fn step(&self) -> Option<&'a ProductionStep> {
+        self.production.steps.get(self.step_index as usize)
+    }
+
+    pub fn symbol(&self) -> Option<Symbol> {
+        self.step().map(|step| step.symbol)
+    }
+
+    pub fn associativity(&self) -> Option<Associativity> {
+        self.prev_step().and_then(|step| step.associativity)
+    }
+
+    pub fn precedence(&self) -> &Precedence {
+        self.prev_step()
+            .map_or(&Precedence::None, |step| &step.precedence)
+    }
+
+    pub fn prev_step(&self) -> Option<&'a ProductionStep> {
+        if self.step_index > 0 {
+            Some(&self.production.steps[self.step_index as usize - 1])
+        } else {
+            None
+        }
+    }
+
+    #[must_use]
+    pub fn is_done(&self) -> bool {
+        self.step_index as usize == self.production.steps.len()
+    }
+
+    #[must_use]
+    pub const fn is_augmented(&self) -> bool {
+        self.variable_index == u32::MAX
+    }
+
+    /// Create an item like this one, but advanced by one step.
+    #[must_use]
+    pub const fn successor(&self) -> Self {
+        ParseItem {
+            variable_index: self.variable_index,
+            production: self.production,
+            step_index: self.step_index + 1,
+            has_preceding_inherited_fields: self.has_preceding_inherited_fields,
+        }
+    }
+
+    /// Create an item identical to this one, but with a different production.
+    /// This is used when dynamically "inlining" certain symbols in a production.
+    pub const fn substitute_production(&self, production: &'a Production) -> Self {
+        let mut result = *self;
+        result.production = production;
+        result
+    }
+}
+
+impl<'a> ParseItemSet<'a> {
+    pub fn insert(&mut self, item: ParseItem<'a>) -> &mut ParseItemSetEntry<'a> {
+        match self.entries.binary_search_by(|e| e.item.cmp(&item)) {
+            Err(i) => {
+                self.entries.insert(
+                    i,
+                    ParseItemSetEntry {
+                        item,
+                        lookaheads: TokenSet::new(),
+                        following_reserved_word_set: ReservedWordSetId::default(),
+                    },
+                );
+                &mut self.entries[i]
+            }
+            Ok(i) => &mut self.entries[i],
+        }
+    }
+
+    pub fn core(&self) -> ParseItemSetCore<'a> {
+        ParseItemSetCore {
+            entries: self.entries.iter().map(|e| e.item).collect(),
+        }
+    }
+}
+
+impl fmt::Display for ParseItemDisplay<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        if self.0.is_augmented() {
+            write!(f, "START →")?;
+        } else {
+            write!(
+                f,
+                "{} →",
+                self.1.variables[self.0.variable_index as usize].name
+            )?;
+        }
+
+        for (i, step) in self.0.production.steps.iter().enumerate() {
+            if i == self.0.step_index as usize {
+                write!(f, " •")?;
+                if !step.precedence.is_none()
+                    || step.associativity.is_some()
+                    || step.reserved_word_set_id != ReservedWordSetId::default()
+                {
+                    write!(f, " (")?;
+                    if step.precedence.is_none() {
+                        write!(f, " {}", step.precedence)?;
+                    }
+                    if let Some(associativity) = step.associativity {
+                        write!(f, " {associativity:?}")?;
+                    }
+                    if step.reserved_word_set_id != ReservedWordSetId::default() {
+                        write!(f, "reserved: {}", step.reserved_word_set_id)?;
+                    }
+                    write!(f, " )")?;
+                }
+            }
+
+            write!(f, " ")?;
+            if step.symbol.is_terminal() {
+                if let Some(variable) = self.2.variables.get(step.symbol.index) {
+                    write!(f, "{}", variable.name)?;
+                } else {
+                    write!(f, "terminal-{}", step.symbol.index)?;
+                }
+            } else if step.symbol.is_external() {
+                write!(f, "{}", self.1.external_tokens[step.symbol.index].name)?;
+            } else {
+                write!(f, "{}", self.1.variables[step.symbol.index].name)?;
+            }
+
+            if let Some(alias) = &step.alias {
+                write!(f, "@{}", alias.value)?;
+            }
+        }
+
+        if self.0.is_done() {
+            write!(f, " •")?;
+            if let Some(step) = self.0.production.steps.last() {
+                if let Some(associativity) = step.associativity {
+                    if step.precedence.is_none() {
+                        write!(f, " ({associativity:?})")?;
+                    } else {
+                        write!(f, " ({} {associativity:?})", step.precedence)?;
+                    }
+                } else if !step.precedence.is_none() {
+                    write!(f, " ({})", step.precedence)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+const fn escape_invisible(c: char) -> Option<&'static str> {
+    Some(match c {
+        '\n' => "\\n",
+        '\r' => "\\r",
+        '\t' => "\\t",
+        '\0' => "\\0",
+        '\\' => "\\\\",
+        '\x0b' => "\\v",
+        '\x0c' => "\\f",
+        _ => return None,
+    })
+}
+
+fn display_variable_name(source: &str) -> String {
+    source
+        .chars()
+        .fold(String::with_capacity(source.len()), |mut acc, c| {
+            if let Some(esc) = escape_invisible(c) {
+                acc.push_str(esc);
+            } else {
+                acc.push(c);
+            }
+            acc
+        })
+}
+
+impl fmt::Display for TokenSetDisplay<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "[")?;
+        for (i, symbol) in self.0.iter().enumerate() {
+            if i > 0 {
+                write!(f, ", ")?;
+            }
+
+            if symbol.is_terminal() {
+                if let Some(variable) = self.2.variables.get(symbol.index) {
+                    write!(f, "{}", display_variable_name(&variable.name))?;
+                } else {
+                    write!(f, "terminal-{}", symbol.index)?;
+                }
+            } else if symbol.is_external() {
+                write!(f, "{}", self.1.external_tokens[symbol.index].name)?;
+            } else {
+                write!(f, "{}", self.1.variables[symbol.index].name)?;
+            }
+        }
+        write!(f, "]")?;
+        Ok(())
+    }
+}
+
+impl fmt::Display for ParseItemSetDisplay<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        for entry in &self.0.entries {
+            write!(
+                f,
+                "{}\t{}",
+                ParseItemDisplay(&entry.item, self.1, self.2),
+                TokenSetDisplay(&entry.lookaheads, self.1, self.2),
+            )?;
+            if entry.following_reserved_word_set != ReservedWordSetId::default() {
+                write!(
+                    f,
+                    "\treserved word set: {}",
+                    entry.following_reserved_word_set
+                )?;
+            }
+            writeln!(f)?;
+        }
+        Ok(())
+    }
+}
+
+impl Hash for ParseItem<'_> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_u32(self.variable_index);
+        hasher.write_u32(self.step_index);
+        hasher.write_i32(self.production.dynamic_precedence);
+        hasher.write_usize(self.production.steps.len());
+        hasher.write_i32(i32::from(self.has_preceding_inherited_fields));
+        self.precedence().hash(hasher);
+        self.associativity().hash(hasher);
+
+        // The already-matched children don't play any role in the parse state for
+        // this item, unless any of the following are true:
+        //   * the children have fields
+        //   * the children have aliases
+        //   * the children are hidden and represent rules that have fields.
+        // See the docs for `has_preceding_inherited_fields`.
+        for step in &self.production.steps[0..self.step_index as usize] {
+            step.alias.hash(hasher);
+            step.field_name.hash(hasher);
+            if self.has_preceding_inherited_fields {
+                step.symbol.hash(hasher);
+            }
+        }
+        for step in &self.production.steps[self.step_index as usize..] {
+            step.hash(hasher);
+        }
+    }
+}
+
+impl PartialEq for ParseItem<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        if self.variable_index != other.variable_index
+            || self.step_index != other.step_index
+            || self.production.dynamic_precedence != other.production.dynamic_precedence
+            || self.production.steps.len() != other.production.steps.len()
+            || self.precedence() != other.precedence()
+            || self.associativity() != other.associativity()
+            || self.has_preceding_inherited_fields != other.has_preceding_inherited_fields
+        {
+            return false;
+        }
+
+        for (i, step) in self.production.steps.iter().enumerate() {
+            // See the previous comment (in the `Hash::hash` impl) regarding comparisons
+            // of parse items' already-completed steps.
+            if i < self.step_index as usize {
+                if step.alias != other.production.steps[i].alias {
+                    return false;
+                }
+                if step.field_name != other.production.steps[i].field_name {
+                    return false;
+                }
+                if self.has_preceding_inherited_fields
+                    && step.symbol != other.production.steps[i].symbol
+                {
+                    return false;
+                }
+            } else if *step != other.production.steps[i] {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+impl Ord for ParseItem<'_> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.step_index
+            .cmp(&other.step_index)
+            .then_with(|| self.variable_index.cmp(&other.variable_index))
+            .then_with(|| {
+                self.production
+                    .dynamic_precedence
+                    .cmp(&other.production.dynamic_precedence)
+            })
+            .then_with(|| {
+                self.production
+                    .steps
+                    .len()
+                    .cmp(&other.production.steps.len())
+            })
+            .then_with(|| self.precedence().cmp(other.precedence()))
+            .then_with(|| self.associativity().cmp(&other.associativity()))
+            .then_with(|| {
+                for (i, step) in self.production.steps.iter().enumerate() {
+                    // See the previous comment (in the `Hash::hash` impl) regarding comparisons
+                    // of parse items' already-completed steps.
+                    let o = if i < self.step_index as usize {
+                        step.alias
+                            .cmp(&other.production.steps[i].alias)
+                            .then_with(|| {
+                                step.field_name.cmp(&other.production.steps[i].field_name)
+                            })
+                    } else {
+                        step.cmp(&other.production.steps[i])
+                    };
+                    if o != Ordering::Equal {
+                        return o;
+                    }
+                }
+                Ordering::Equal
+            })
+    }
+}
+
+impl PartialOrd for ParseItem<'_> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Eq for ParseItem<'_> {}
+
+impl Hash for ParseItemSet<'_> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_usize(self.entries.len());
+        for entry in &self.entries {
+            entry.item.hash(hasher);
+            entry.lookaheads.hash(hasher);
+            entry.following_reserved_word_set.hash(hasher);
+        }
+    }
+}
+
+impl Hash for ParseItemSetCore<'_> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_usize(self.entries.len());
+        for item in &self.entries {
+            item.hash(hasher);
+        }
+    }
+}

cli/generate/src/build_tables/item_set_builder.rs

@@ -0,0 +1,397 @@
+use std::{
+    collections::{HashMap, HashSet},
+    fmt,
+};
+
+use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, ParseItemSetEntry, TokenSetDisplay};
+use crate::{
+    grammars::{InlinedProductionMap, LexicalGrammar, ReservedWordSetId, SyntaxGrammar},
+    rules::{Symbol, SymbolType, TokenSet},
+};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct TransitiveClosureAddition<'a> {
+    item: ParseItem<'a>,
+    info: FollowSetInfo,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+struct FollowSetInfo {
+    lookaheads: TokenSet,
+    reserved_lookaheads: ReservedWordSetId,
+    propagates_lookaheads: bool,
+}
+
+pub struct ParseItemSetBuilder<'a> {
+    syntax_grammar: &'a SyntaxGrammar,
+    lexical_grammar: &'a LexicalGrammar,
+    first_sets: HashMap<Symbol, TokenSet>,
+    reserved_first_sets: HashMap<Symbol, ReservedWordSetId>,
+    last_sets: HashMap<Symbol, TokenSet>,
+    inlines: &'a InlinedProductionMap,
+    transitive_closure_additions: Vec<Vec<TransitiveClosureAddition<'a>>>,
+}
+
+fn find_or_push<T: Eq>(vector: &mut Vec<T>, value: T) {
+    if !vector.contains(&value) {
+        vector.push(value);
+    }
+}
+
+impl<'a> ParseItemSetBuilder<'a> {
+    pub fn new(
+        syntax_grammar: &'a SyntaxGrammar,
+        lexical_grammar: &'a LexicalGrammar,
+        inlines: &'a InlinedProductionMap,
+    ) -> Self {
+        let mut result = Self {
+            syntax_grammar,
+            lexical_grammar,
+            first_sets: HashMap::new(),
+            reserved_first_sets: HashMap::new(),
+            last_sets: HashMap::new(),
+            inlines,
+            transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()],
+        };
+
+        // For each grammar symbol, populate the FIRST and LAST sets: the set of
+        // terminals that appear at the beginning and end that symbol's productions,
+        // respectively.
+        // For a terminal symbol, the FIRST and LAST sets just consist of the
+        // terminal itself.
+        for i in 0..lexical_grammar.variables.len() {
+            let symbol = Symbol::terminal(i);
+            let mut set = TokenSet::new();
+            set.insert(symbol);
+            result.first_sets.insert(symbol, set.clone());
+            result.last_sets.insert(symbol, set);
+            result
+                .reserved_first_sets
+                .insert(symbol, ReservedWordSetId::default());
+        }
+
+        for i in 0..syntax_grammar.external_tokens.len() {
+            let symbol = Symbol::external(i);
+            let mut set = TokenSet::new();
+            set.insert(symbol);
+            result.first_sets.insert(symbol, set.clone());
+            result.last_sets.insert(symbol, set);
+            result
+                .reserved_first_sets
+                .insert(symbol, ReservedWordSetId::default());
+        }
+
+        // The FIRST set of a non-terminal `i` is the union of the the FIRST sets
+        // of all the symbols that appear at the beginnings of i's productions. Some
+        // of these symbols may themselves be non-terminals, so this is a recursive
+        // definition.
+        //
+        // Rather than computing these sets using recursion, we use an explicit stack
+        // called `symbols_to_process`.
+        let mut symbols_to_process = Vec::new();
+        let mut processed_non_terminals = HashSet::new();
+        for i in 0..syntax_grammar.variables.len() {
+            let symbol = Symbol::non_terminal(i);
+            let first_set = result.first_sets.entry(symbol).or_default();
+            let reserved_first_set = result.reserved_first_sets.entry(symbol).or_default();
+
+            processed_non_terminals.clear();
+            symbols_to_process.clear();
+            symbols_to_process.push(symbol);
+            while let Some(sym) = symbols_to_process.pop() {
+                for production in &syntax_grammar.variables[sym.index].productions {
+                    if let Some(step) = production.steps.first() {
+                        if step.symbol.is_terminal() || step.symbol.is_external() {
+                            first_set.insert(step.symbol);
+                        } else if processed_non_terminals.insert(step.symbol) {
+                            symbols_to_process.push(step.symbol);
+                        }
+                        *reserved_first_set = (*reserved_first_set).max(step.reserved_word_set_id);
+                    }
+                }
+            }
+
+            // The LAST set is defined in a similar way to the FIRST set.
+            let last_set = result.last_sets.entry(symbol).or_default();
+            processed_non_terminals.clear();
+            symbols_to_process.clear();
+            symbols_to_process.push(symbol);
+            while let Some(sym) = symbols_to_process.pop() {
+                for production in &syntax_grammar.variables[sym.index].productions {
+                    if let Some(step) = production.steps.last() {
+                        if step.symbol.is_terminal() || step.symbol.is_external() {
+                            last_set.insert(step.symbol);
+                        } else if processed_non_terminals.insert(step.symbol) {
+                            symbols_to_process.push(step.symbol);
+                        }
+                    }
+                }
+            }
+        }
+
+        // To compute an item set's transitive closure, we find each item in the set
+        // whose next symbol is a non-terminal, and we add new items to the set for
+        // each of that symbol's productions. These productions might themselves begin
+        // with non-terminals, so the process continues recursively. In this process,
+        // the total set of entries that get added depends only on two things:
+        //
+        //   * the non-terminal symbol that occurs next in each item
+        //
+        //   * the set of terminals that can follow that non-terminal symbol in the item
+        //
+        // So we can avoid a lot of duplicated recursive work by precomputing, for each
+        // non-terminal symbol `i`, a final list of *additions* that must be made to an
+        // item set when symbol `i` occurs as the next symbol in one if its core items.
+        // The structure of a precomputed *addition* is as follows:
+        //
+        //   * `item` - the new item that must be added as part of the expansion of the symbol `i`.
+        //
+        //   * `lookaheads` - the set of possible lookahead tokens that can always come after `item`
+        //     in an expansion of symbol `i`.
+        //
+        //   * `reserved_lookaheads` - the set of reserved lookahead lookahead tokens that can
+        //     always come after `item` in the expansion of symbol `i`.
+        //
+        //   * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the
+        //     *end* of the expansion of symbol `i`, so that i's own current lookahead tokens can
+        //     occur after `item`.
+        //
+        // Rather than computing these additions recursively, we use an explicit stack.
+        let empty_lookaheads = TokenSet::new();
+        let mut stack = Vec::new();
+        let mut follow_set_info_by_non_terminal = HashMap::<usize, FollowSetInfo>::new();
+        for i in 0..syntax_grammar.variables.len() {
+            // First, build up a map whose keys are all of the non-terminals that can
+            // appear at the beginning of non-terminal `i`, and whose values store
+            // information about the tokens that can follow those non-terminals.
+            stack.clear();
+            stack.push((i, &empty_lookaheads, ReservedWordSetId::default(), true));
+            follow_set_info_by_non_terminal.clear();
+            while let Some((sym_ix, lookaheads, reserved_word_set_id, propagates_lookaheads)) =
+                stack.pop()
+            {
+                let mut did_add = false;
+                let info = follow_set_info_by_non_terminal.entry(sym_ix).or_default();
+                did_add |= info.lookaheads.insert_all(lookaheads);
+                if reserved_word_set_id > info.reserved_lookaheads {
+                    info.reserved_lookaheads = reserved_word_set_id;
+                    did_add = true;
+                }
+                did_add |= propagates_lookaheads && !info.propagates_lookaheads;
+                info.propagates_lookaheads |= propagates_lookaheads;
+                if !did_add {
+                    continue;
+                }
+
+                for production in &syntax_grammar.variables[sym_ix].productions {
+                    if let Some(symbol) = production.first_symbol() {
+                        if symbol.is_non_terminal() {
+                            if let Some(next_step) = production.steps.get(1) {
+                                stack.push((
+                                    symbol.index,
+                                    &result.first_sets[&next_step.symbol],
+                                    result.reserved_first_sets[&next_step.symbol],
+                                    false,
+                                ));
+                            } else {
+                                stack.push((
+                                    symbol.index,
+                                    lookaheads,
+                                    reserved_word_set_id,
+                                    propagates_lookaheads,
+                                ));
+                            }
+                        }
+                    }
+                }
+            }
+
+            // Store all of those non-terminals' productions, along with their associated
+            // lookahead info, as *additions* associated with non-terminal `i`.
+            let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
+            for (&variable_index, follow_set_info) in &follow_set_info_by_non_terminal {
+                let variable = &syntax_grammar.variables[variable_index];
+                let non_terminal = Symbol::non_terminal(variable_index);
+                let variable_index = variable_index as u32;
+                if syntax_grammar.variables_to_inline.contains(&non_terminal) {
+                    continue;
+                }
+                for production in &variable.productions {
+                    let item = ParseItem {
+                        variable_index,
+                        production,
+                        step_index: 0,
+                        has_preceding_inherited_fields: false,
+                    };
+
+                    if let Some(inlined_productions) =
+                        inlines.inlined_productions(item.production, item.step_index)
+                    {
+                        for production in inlined_productions {
+                            find_or_push(
+                                additions_for_non_terminal,
+                                TransitiveClosureAddition {
+                                    item: item.substitute_production(production),
+                                    info: follow_set_info.clone(),
+                                },
+                            );
+                        }
+                    } else {
+                        find_or_push(
+                            additions_for_non_terminal,
+                            TransitiveClosureAddition {
+                                item,
+                                info: follow_set_info.clone(),
+                            },
+                        );
+                    }
+                }
+            }
+        }
+
+        result
+    }
+
+    pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
+        let mut result = ParseItemSet::default();
+        for entry in &item_set.entries {
+            if let Some(productions) = self
+                .inlines
+                .inlined_productions(entry.item.production, entry.item.step_index)
+            {
+                for production in productions {
+                    self.add_item(
+                        &mut result,
+                        &ParseItemSetEntry {
+                            item: entry.item.substitute_production(production),
+                            lookaheads: entry.lookaheads.clone(),
+                            following_reserved_word_set: entry.following_reserved_word_set,
+                        },
+                    );
+                }
+            } else {
+                self.add_item(&mut result, entry);
+            }
+        }
+        result
+    }
+
+    pub fn first_set(&self, symbol: &Symbol) -> &TokenSet {
+        &self.first_sets[symbol]
+    }
+
+    pub fn reserved_first_set(&self, symbol: &Symbol) -> Option<&TokenSet> {
+        let id = *self.reserved_first_sets.get(symbol)?;
+        Some(&self.syntax_grammar.reserved_word_sets[id.0])
+    }
+
+    pub fn last_set(&self, symbol: &Symbol) -> &TokenSet {
+        &self.last_sets[symbol]
+    }
+
+    fn add_item(&self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
+        if let Some(step) = entry.item.step() {
+            if step.symbol.is_non_terminal() {
+                let next_step = entry.item.successor().step();
+
+                // Determine which tokens can follow this non-terminal.
+                let (following_tokens, following_reserved_tokens) =
+                    if let Some(next_step) = next_step {
+                        (
+                            self.first_sets.get(&next_step.symbol).unwrap(),
+                            *self.reserved_first_sets.get(&next_step.symbol).unwrap(),
+                        )
+                    } else {
+                        (&entry.lookaheads, entry.following_reserved_word_set)
+                    };
+
+                // Use the pre-computed *additions* to expand the non-terminal.
+                for addition in &self.transitive_closure_additions[step.symbol.index] {
+                    let entry = set.insert(addition.item);
+                    entry.lookaheads.insert_all(&addition.info.lookaheads);
+
+                    if let Some(word_token) = self.syntax_grammar.word_token {
+                        if addition.info.lookaheads.contains(&word_token) {
+                            entry.following_reserved_word_set = entry
+                                .following_reserved_word_set
+                                .max(addition.info.reserved_lookaheads);
+                        }
+                    }
+
+                    if addition.info.propagates_lookaheads {
+                        entry.lookaheads.insert_all(following_tokens);
+
+                        if let Some(word_token) = self.syntax_grammar.word_token {
+                            if following_tokens.contains(&word_token) {
+                                entry.following_reserved_word_set = entry
+                                    .following_reserved_word_set
+                                    .max(following_reserved_tokens);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        let e = set.insert(entry.item);
+        e.lookaheads.insert_all(&entry.lookaheads);
+        e.following_reserved_word_set = e
+            .following_reserved_word_set
+            .max(entry.following_reserved_word_set);
+    }
+}
+
+impl fmt::Debug for ParseItemSetBuilder<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "ParseItemSetBuilder {{")?;
+
+        writeln!(f, "  first_sets: {{")?;
+        for (symbol, first_set) in &self.first_sets {
+            let name = match symbol.kind {
+                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
+                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
+                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
+                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
+            };
+            writeln!(
+                f,
+                "    first({name:?}): {}",
+                TokenSetDisplay(first_set, self.syntax_grammar, self.lexical_grammar)
+            )?;
+        }
+        writeln!(f, "  }}")?;
+
+        writeln!(f, "  last_sets: {{")?;
+        for (symbol, last_set) in &self.last_sets {
+            let name = match symbol.kind {
+                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
+                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
+                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
+                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
+            };
+            writeln!(
+                f,
+                "    last({name:?}): {}",
+                TokenSetDisplay(last_set, self.syntax_grammar, self.lexical_grammar)
+            )?;
+        }
+        writeln!(f, "  }}")?;
+
+        writeln!(f, "  additions: {{")?;
+        for (i, variable) in self.syntax_grammar.variables.iter().enumerate() {
+            writeln!(f, "    {}: {{", variable.name)?;
+            for addition in &self.transitive_closure_additions[i] {
+                writeln!(
+                    f,
+                    "      {}",
+                    ParseItemDisplay(&addition.item, self.syntax_grammar, self.lexical_grammar)
+                )?;
+            }
+            writeln!(f, "    }},")?;
+        }
+        write!(f, "  }},")?;
+
+        write!(f, "}}")?;
+        Ok(())
+    }
+}

cli/generate/src/build_tables/minimize_parse_table.rs

@@ -0,0 +1,492 @@
+use std::{
+    collections::{HashMap, HashSet},
+    mem,
+};
+
+use log::info;
+
+use super::token_conflicts::TokenConflictMap;
+use crate::{
+    dedup::split_state_id_groups,
+    grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
+    rules::{AliasMap, Symbol, TokenSet},
+    tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
+};
+
+pub fn minimize_parse_table(
+    parse_table: &mut ParseTable,
+    syntax_grammar: &SyntaxGrammar,
+    lexical_grammar: &LexicalGrammar,
+    simple_aliases: &AliasMap,
+    token_conflict_map: &TokenConflictMap,
+    keywords: &TokenSet,
+) {
+    let mut minimizer = Minimizer {
+        parse_table,
+        syntax_grammar,
+        lexical_grammar,
+        token_conflict_map,
+        keywords,
+        simple_aliases,
+    };
+    minimizer.merge_compatible_states();
+    minimizer.remove_unit_reductions();
+    minimizer.remove_unused_states();
+    minimizer.reorder_states_by_descending_size();
+}
+
+struct Minimizer<'a> {
+    parse_table: &'a mut ParseTable,
+    syntax_grammar: &'a SyntaxGrammar,
+    lexical_grammar: &'a LexicalGrammar,
+    token_conflict_map: &'a TokenConflictMap<'a>,
+    keywords: &'a TokenSet,
+    simple_aliases: &'a AliasMap,
+}
+
+impl Minimizer<'_> {
+    fn remove_unit_reductions(&mut self) {
+        let mut aliased_symbols = HashSet::new();
+        for variable in &self.syntax_grammar.variables {
+            for production in &variable.productions {
+                for step in &production.steps {
+                    if step.alias.is_some() {
+                        aliased_symbols.insert(step.symbol);
+                    }
+                }
+            }
+        }
+
+        let mut unit_reduction_symbols_by_state = HashMap::new();
+        for (i, state) in self.parse_table.states.iter().enumerate() {
+            let mut only_unit_reductions = true;
+            let mut unit_reduction_symbol = None;
+            for (_, entry) in &state.terminal_entries {
+                for action in &entry.actions {
+                    match action {
+                        ParseAction::ShiftExtra => continue,
+                        ParseAction::Reduce {
+                            child_count: 1,
+                            production_id: 0,
+                            symbol,
+                            ..
+                        } if !self.simple_aliases.contains_key(symbol)
+                            && !self.syntax_grammar.supertype_symbols.contains(symbol)
+                            && !self.syntax_grammar.extra_symbols.contains(symbol)
+                            && !aliased_symbols.contains(symbol)
+                            && self.syntax_grammar.variables[symbol.index].kind
+                                != VariableType::Named
+                            && (unit_reduction_symbol.is_none()
+                                || unit_reduction_symbol == Some(symbol)) =>
+                        {
+                            unit_reduction_symbol = Some(symbol);
+                            continue;
+                        }
+                        _ => {}
+                    }
+                    only_unit_reductions = false;
+                    break;
+                }
+
+                if !only_unit_reductions {
+                    break;
+                }
+            }
+
+            if let Some(symbol) = unit_reduction_symbol {
+                if only_unit_reductions {
+                    unit_reduction_symbols_by_state.insert(i, *symbol);
+                }
+            }
+        }
+
+        for state in &mut self.parse_table.states {
+            let mut done = false;
+            while !done {
+                done = true;
+                state.update_referenced_states(|other_state_id, state| {
+                    unit_reduction_symbols_by_state.get(&other_state_id).map_or(
+                        other_state_id,
+                        |symbol| {
+                            done = false;
+                            match state.nonterminal_entries.get(symbol) {
+                                Some(GotoAction::Goto(state_id)) => *state_id,
+                                _ => other_state_id,
+                            }
+                        },
+                    )
+                });
+            }
+        }
+    }
+
+    fn merge_compatible_states(&mut self) {
+        let core_count = 1 + self
+            .parse_table
+            .states
+            .iter()
+            .map(|state| state.core_id)
+            .max()
+            .unwrap();
+
+        // Initially group the states by their parse item set core.
+        let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
+        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
+        for (i, state) in self.parse_table.states.iter().enumerate() {
+            state_ids_by_group_id[state.core_id].push(i);
+            group_ids_by_state_id.push(state.core_id);
+        }
+
+        split_state_id_groups(
+            &self.parse_table.states,
+            &mut state_ids_by_group_id,
+            &mut group_ids_by_state_id,
+            0,
+            |left, right, groups| self.states_conflict(left, right, groups),
+        );
+
+        while split_state_id_groups(
+            &self.parse_table.states,
+            &mut state_ids_by_group_id,
+            &mut group_ids_by_state_id,
+            0,
+            |left, right, groups| self.state_successors_differ(left, right, groups),
+        ) {}
+
+        let error_group_index = state_ids_by_group_id
+            .iter()
+            .position(|g| g.contains(&0))
+            .unwrap();
+        let start_group_index = state_ids_by_group_id
+            .iter()
+            .position(|g| g.contains(&1))
+            .unwrap();
+        state_ids_by_group_id.swap(error_group_index, 0);
+        state_ids_by_group_id.swap(start_group_index, 1);
+
+        // Create a list of new parse states: one state for each group of old states.
+        let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
+        for state_ids in &state_ids_by_group_id {
+            // Initialize the new state based on the first old state in the group.
+            let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0]]);
+
+            // Extend the new state with all of the actions from the other old states
+            // in the group.
+            for state_id in &state_ids[1..] {
+                let other_parse_state = mem::take(&mut self.parse_table.states[*state_id]);
+
+                parse_state
+                    .terminal_entries
+                    .extend(other_parse_state.terminal_entries);
+                parse_state
+                    .nonterminal_entries
+                    .extend(other_parse_state.nonterminal_entries);
+                parse_state
+                    .reserved_words
+                    .insert_all(&other_parse_state.reserved_words);
+                for symbol in parse_state.terminal_entries.keys() {
+                    parse_state.reserved_words.remove(symbol);
+                }
+            }
+
+            // Update the new state's outgoing references using the new grouping.
+            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
+            new_states.push(parse_state);
+        }
+
+        self.parse_table.states = new_states;
+    }
+
+    fn states_conflict(
+        &self,
+        left_state: &ParseState,
+        right_state: &ParseState,
+        group_ids_by_state_id: &[ParseStateId],
+    ) -> bool {
+        for (token, left_entry) in &left_state.terminal_entries {
+            if let Some(right_entry) = right_state.terminal_entries.get(token) {
+                if self.entries_conflict(
+                    left_state.id,
+                    right_state.id,
+                    token,
+                    left_entry,
+                    right_entry,
+                    group_ids_by_state_id,
+                ) {
+                    return true;
+                }
+            } else if self.token_conflicts(left_state.id, right_state.id, right_state, *token) {
+                return true;
+            }
+        }
+
+        for token in right_state.terminal_entries.keys() {
+            if !left_state.terminal_entries.contains_key(token)
+                && self.token_conflicts(left_state.id, right_state.id, left_state, *token)
+            {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn state_successors_differ(
+        &self,
+        state1: &ParseState,
+        state2: &ParseState,
+        group_ids_by_state_id: &[ParseStateId],
+    ) -> bool {
+        for (token, entry1) in &state1.terminal_entries {
+            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap() {
+                if let Some(entry2) = state2.terminal_entries.get(token) {
+                    if let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap() {
+                        let group1 = group_ids_by_state_id[*s1];
+                        let group2 = group_ids_by_state_id[*s2];
+                        if group1 != group2 {
+                            info!(
+                                "split states {} {} - successors for {} are split: {s1} {s2}",
+                                state1.id,
+                                state2.id,
+                                self.symbol_name(token),
+                            );
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+
+        for (symbol, s1) in &state1.nonterminal_entries {
+            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
+                match (s1, s2) {
+                    (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => {}
+                    (GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
+                        let group1 = group_ids_by_state_id[*s1];
+                        let group2 = group_ids_by_state_id[*s2];
+                        if group1 != group2 {
+                            info!(
+                                "split states {} {} - successors for {} are split: {s1} {s2}",
+                                state1.id,
+                                state2.id,
+                                self.symbol_name(symbol),
+                            );
+                            return true;
+                        }
+                    }
+                    _ => return true,
+                }
+            }
+        }
+
+        false
+    }
+
+    fn entries_conflict(
+        &self,
+        state_id1: ParseStateId,
+        state_id2: ParseStateId,
+        token: &Symbol,
+        entry1: &ParseTableEntry,
+        entry2: &ParseTableEntry,
+        group_ids_by_state_id: &[ParseStateId],
+    ) -> bool {
+        // To be compatible, entries need to have the same actions.
+        let actions1 = &entry1.actions;
+        let actions2 = &entry2.actions;
+        if actions1.len() != actions2.len() {
+            info!(
+                "split states {state_id1} {state_id2} - differing action counts for token {}",
+                self.symbol_name(token)
+            );
+            return true;
+        }
+
+        for (i, action1) in actions1.iter().enumerate() {
+            let action2 = &actions2[i];
+
+            // Two shift actions are equivalent if their destinations are in the same group.
+            if let (
+                ParseAction::Shift {
+                    state: s1,
+                    is_repetition: is_repetition1,
+                },
+                ParseAction::Shift {
+                    state: s2,
+                    is_repetition: is_repetition2,
+                },
+            ) = (action1, action2)
+            {
+                let group1 = group_ids_by_state_id[*s1];
+                let group2 = group_ids_by_state_id[*s2];
+                if group1 == group2 && is_repetition1 == is_repetition2 {
+                    continue;
+                }
+                info!(
+                    "split states {state_id1} {state_id2} - successors for {} are split: {s1} {s2}",
+                    self.symbol_name(token),
+                );
+                return true;
+            } else if action1 != action2 {
+                info!(
+                    "split states {state_id1} {state_id2} - unequal actions for {}",
+                    self.symbol_name(token),
+                );
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn token_conflicts(
+        &self,
+        left_id: ParseStateId,
+        right_id: ParseStateId,
+        right_state: &ParseState,
+        new_token: Symbol,
+    ) -> bool {
+        if new_token == Symbol::end_of_nonterminal_extra() {
+            info!("split states {left_id} {right_id} - end of non-terminal extra",);
+            return true;
+        }
+
+        // Do not add external tokens; they could conflict lexically with any of the state's
+        // existing lookahead tokens.
+        if new_token.is_external() {
+            info!(
+                "split states {left_id} {right_id} - external token {}",
+                self.symbol_name(&new_token),
+            );
+            return true;
+        }
+
+        if right_state.reserved_words.contains(&new_token) {
+            return false;
+        }
+
+        // Do not add tokens which are both internal and external. Their validity could
+        // influence the behavior of the external scanner.
+        if self
+            .syntax_grammar
+            .external_tokens
+            .iter()
+            .any(|external| external.corresponding_internal_token == Some(new_token))
+        {
+            info!(
+                "split states {left_id} {right_id} - internal/external token {}",
+                self.symbol_name(&new_token),
+            );
+            return true;
+        }
+
+        // Do not add a token if it conflicts with an existing token.
+        for token in right_state.terminal_entries.keys().copied() {
+            if !token.is_terminal() {
+                continue;
+            }
+            if self.syntax_grammar.word_token == Some(token) && self.keywords.contains(&new_token) {
+                continue;
+            }
+            if self.syntax_grammar.word_token == Some(new_token) && self.keywords.contains(&token) {
+                continue;
+            }
+
+            if self
+                .token_conflict_map
+                .does_conflict(new_token.index, token.index)
+                || self
+                    .token_conflict_map
+                    .does_match_same_string(new_token.index, token.index)
+            {
+                info!(
+                    "split states {} {} - token {} conflicts with {}",
+                    left_id,
+                    right_id,
+                    self.symbol_name(&new_token),
+                    self.symbol_name(&token),
+                );
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn symbol_name(&self, symbol: &Symbol) -> &String {
+        if symbol.is_non_terminal() {
+            &self.syntax_grammar.variables[symbol.index].name
+        } else if symbol.is_external() {
+            &self.syntax_grammar.external_tokens[symbol.index].name
+        } else {
+            &self.lexical_grammar.variables[symbol.index].name
+        }
+    }
+
+    fn remove_unused_states(&mut self) {
+        let mut state_usage_map = vec![false; self.parse_table.states.len()];
+
+        state_usage_map[0] = true;
+        state_usage_map[1] = true;
+
+        for state in &self.parse_table.states {
+            for referenced_state in state.referenced_states() {
+                state_usage_map[referenced_state] = true;
+            }
+        }
+        let mut removed_predecessor_count = 0;
+        let mut state_replacement_map = vec![0; self.parse_table.states.len()];
+        for state_id in 0..self.parse_table.states.len() {
+            state_replacement_map[state_id] = state_id - removed_predecessor_count;
+            if !state_usage_map[state_id] {
+                removed_predecessor_count += 1;
+            }
+        }
+        let mut state_id = 0;
+        let mut original_state_id = 0;
+        while state_id < self.parse_table.states.len() {
+            if state_usage_map[original_state_id] {
+                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
+                    state_replacement_map[other_state_id]
+                });
+                state_id += 1;
+            } else {
+                self.parse_table.states.remove(state_id);
+            }
+            original_state_id += 1;
+        }
+    }
+
+    fn reorder_states_by_descending_size(&mut self) {
+        // Get a mapping of old state index -> new_state_index
+        let mut old_ids_by_new_id = (0..self.parse_table.states.len()).collect::<Vec<_>>();
+        old_ids_by_new_id.sort_unstable_by_key(|i| {
+            // Don't changes states 0 (the error state) or 1 (the start state).
+            if *i <= 1 {
+                return *i as i64 - 1_000_000;
+            }
+
+            // Reorder all the other states by descending symbol count.
+            let state = &self.parse_table.states[*i];
+            -((state.terminal_entries.len() + state.nonterminal_entries.len()) as i64)
+        });
+
+        // Get the inverse mapping
+        let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
+        for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
+            new_ids_by_old_id[*old_id] = id;
+        }
+
+        // Reorder the parse states and update their references to reflect
+        // the new ordering.
+        self.parse_table.states = old_ids_by_new_id
+            .iter()
+            .map(|old_id| {
+                let mut state = ParseState::default();
+                mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
+                state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
+                state
+            })
+            .collect();
+    }
+}

cli/generate/src/build_tables/mod.rs

@@ -6,31 +6,27 @@ mod item_set_builder;
 mod minimize_parse_table;
 mod token_conflicts;
 
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashMap};
 
 pub use build_lex_table::LARGE_CHARACTER_RANGE_COUNT;
 use build_parse_table::BuildTableResult;
 pub use build_parse_table::ParseTableBuilderError;
-use log::{debug, info};
-use rustc_hash::FxHashMap;
+use log::info;
 
 use self::{
     build_lex_table::build_lex_table,
-    build_parse_table::{ParseStateInfo, build_parse_table},
+    build_parse_table::{build_parse_table, ParseStateInfo},
     coincident_tokens::CoincidentTokenIndex,
-    item::ItemKeyMap,
     item_set_builder::ParseItemSetBuilder,
     minimize_parse_table::minimize_parse_table,
     token_conflicts::TokenConflictMap,
 };
 use crate::{
-    Diagnostic, OptLevel,
     grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
     nfa::{CharacterSet, NfaCursor},
     node_types::VariableInfo,
     rules::{AliasMap, Symbol, SymbolType, TokenSet},
-    strpool::StrPool,
-    tables::{ActionList, ActionListPool, LexTable, ParseAction, ParseTable, ParseTableEntry},
+    tables::{LexTable, ParseAction, ParseTable, ParseTableEntry},
 };
 
 pub struct Tables {
@@ -40,42 +36,31 @@ pub struct Tables {
     pub large_character_sets: Vec<(Option<Symbol>, CharacterSet)>,
 }
 
-#[expect(
-    clippy::too_many_arguments,
-    reason = "all parameters are required for table building"
-)]
 pub fn build_tables(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     simple_aliases: &AliasMap,
     variable_info: &[VariableInfo],
     inlines: &InlinedProductionMap,
-    str_pool: &StrPool,
     report_symbol_name: Option<&str>,
-    optimizations: OptLevel,
-    diagnostics: &mut Vec<Diagnostic>,
 ) -> BuildTableResult<Tables> {
-    let item_key_map = ItemKeyMap::new(syntax_grammar, str_pool);
-    let item_set_builder =
-        ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines, &item_key_map);
-    let following_tokens = get_following_tokens(syntax_grammar, lexical_grammar, &item_set_builder);
+    let item_set_builder = ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines);
+    let following_tokens =
+        get_following_tokens(syntax_grammar, lexical_grammar, inlines, &item_set_builder);
     let (mut parse_table, parse_state_info) = build_parse_table(
         syntax_grammar,
         lexical_grammar,
         item_set_builder,
         variable_info,
-        str_pool,
-        diagnostics,
     )?;
     let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
-    let coincident_token_index =
-        CoincidentTokenIndex::new(&parse_table, lexical_grammar, syntax_grammar.word_token);
+    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
     let keywords = identify_keywords(
         lexical_grammar,
+        &parse_table,
         syntax_grammar.word_token,
         &token_conflict_map,
         &coincident_token_index,
-        str_pool,
     );
     populate_error_state(
         &mut parse_table,
@@ -84,10 +69,8 @@ pub fn build_tables(
         &coincident_token_index,
         &token_conflict_map,
         &keywords,
-        str_pool,
     );
     populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
-    let mut parse_table = ActionListPool::intern_table(parse_table);
     minimize_parse_table(
         &mut parse_table,
         syntax_grammar,
@@ -95,8 +78,6 @@ pub fn build_tables(
         simple_aliases,
         &token_conflict_map,
         &keywords,
-        str_pool,
-        optimizations,
     );
     let lex_tables = build_lex_table(
         &mut parse_table,
@@ -107,10 +88,7 @@ pub fn build_tables(
         &token_conflict_map,
     );
     populate_external_lex_states(&mut parse_table, syntax_grammar);
-    mark_fragile_tokens(&mut parse_table, &token_conflict_map);
-    parse_table
-        .action_lists
-        .canonicalize(&mut parse_table.states);
+    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
 
     if let Some(report_symbol_name) = report_symbol_name {
         report_state_info(
@@ -118,15 +96,10 @@ pub fn build_tables(
             lexical_grammar,
             &parse_table,
             &parse_state_info,
-            str_pool,
             report_symbol_name,
         );
     }
 
-    if parse_table.states.len() > u16::MAX as usize {
-        Err(ParseTableBuilderError::StateCount(parse_table.states.len()))?;
-    }
-
     Ok(Tables {
         parse_table,
         main_lex_table: lex_tables.main_lex_table,
@@ -138,25 +111,28 @@ pub fn build_tables(
 fn get_following_tokens(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
+    inlines: &InlinedProductionMap,
     builder: &ParseItemSetBuilder,
 ) -> Vec<TokenSet> {
-    let n_terminals = lexical_grammar.variables.len();
-    let n_externals = syntax_grammar.external_tokens.len();
-    let mut result = vec![TokenSet::with_capacity(n_terminals, n_externals); n_terminals];
+    let mut result = vec![TokenSet::new(); lexical_grammar.variables.len()];
+    let productions = syntax_grammar
+        .variables
+        .iter()
+        .flat_map(|v| &v.productions)
+        .chain(&inlines.productions);
     let all_tokens = (0..result.len())
         .map(Symbol::terminal)
         .collect::<TokenSet>();
-    for production in &syntax_grammar.productions {
-        let steps = &syntax_grammar.steps[production.step_range()];
-        for i in 1..steps.len() {
-            let left_tokens = builder.last_set(steps[i - 1].symbol());
-            let right_tokens = builder.first_set(steps[i].symbol());
-            let right_reserved_tokens = builder.reserved_first_set(steps[i].symbol());
+    for production in productions {
+        for i in 1..production.steps.len() {
+            let left_tokens = builder.last_set(&production.steps[i - 1].symbol);
+            let right_tokens = builder.first_set(&production.steps[i].symbol);
+            let right_reserved_tokens = builder.reserved_first_set(&production.steps[i].symbol);
             for left_token in left_tokens.iter() {
                 if left_token.is_terminal() {
-                    result[left_token.index as usize].insert_all_terminals(right_tokens);
+                    result[left_token.index].insert_all_terminals(right_tokens);
                     if let Some(reserved_tokens) = right_reserved_tokens {
-                        result[left_token.index as usize].insert_all_terminals(reserved_tokens);
+                        result[left_token.index].insert_all_terminals(reserved_tokens);
                     }
                 }
             }
@@ -167,20 +143,19 @@ fn get_following_tokens(
             for entry in &mut result {
                 entry.insert(*extra);
             }
-            result[extra.index as usize] = all_tokens.clone();
+            result[extra.index] = all_tokens.clone();
         }
     }
     result
 }
 
 fn populate_error_state(
-    parse_table: &mut ParseTable<ParseTableEntry>,
+    parse_table: &mut ParseTable,
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     coincident_token_index: &CoincidentTokenIndex,
     token_conflict_map: &TokenConflictMap,
     keywords: &TokenSet,
-    str_pool: &StrPool,
 ) {
     let state = &mut parse_table.states[0];
     let n = lexical_grammar.variables.len();
@@ -197,9 +172,9 @@ fn populate_error_state(
             if conflicts_with_other_tokens {
                 None
             } else {
-                debug!(
+                info!(
                     "error recovery - token {} has no conflicts",
-                    str_pool.resolve(lexical_grammar.variables[i].name)
+                    lexical_grammar.variables[i].name
                 );
                 Some(Symbol::terminal(i))
             }
@@ -208,31 +183,31 @@ fn populate_error_state(
 
     let recover_entry = ParseTableEntry {
         reusable: false,
-        actions: ActionList::One(ParseAction::Recover),
+        actions: vec![ParseAction::Recover],
     };
 
     // Exclude from the error-recovery state any token that conflicts with one of
     // the *conflict-free tokens* identified above.
     for i in 0..n {
         let symbol = Symbol::terminal(i);
-        if !conflict_free_tokens.contains(symbol)
-            && !keywords.contains(symbol)
+        if !conflict_free_tokens.contains(&symbol)
+            && !keywords.contains(&symbol)
             && syntax_grammar.word_token != Some(symbol)
-            && let Some(t) = conflict_free_tokens.iter().find(|t| {
-                !coincident_token_index.contains(symbol, *t)
-                    && token_conflict_map.does_conflict(symbol.index as usize, t.index as usize)
-            })
         {
-            debug!(
-                "error recovery - exclude token {} because of conflict with {}",
-                str_pool.resolve(lexical_grammar.variables[i].name),
-                str_pool.resolve(lexical_grammar.variables[t.index as usize].name)
-            );
-            continue;
+            if let Some(t) = conflict_free_tokens.iter().find(|t| {
+                !coincident_token_index.contains(symbol, *t)
+                    && token_conflict_map.does_conflict(symbol.index, t.index)
+            }) {
+                info!(
+                    "error recovery - exclude token {} because of conflict with {}",
+                    lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
+                );
+                continue;
+            }
         }
-        debug!(
+        info!(
             "error recovery - include token {}",
-            str_pool.resolve(lexical_grammar.variables[i].name)
+            lexical_grammar.variables[i].name
         );
         state
             .terminal_entries
@@ -253,7 +228,7 @@ fn populate_error_state(
 }
 
 fn populate_used_symbols(
-    parse_table: &mut ParseTable<ParseTableEntry>,
+    parse_table: &mut ParseTable,
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
 ) {
@@ -263,13 +238,13 @@ fn populate_used_symbols(
     for state in &parse_table.states {
         for symbol in state.terminal_entries.keys() {
             match symbol.kind {
-                SymbolType::Terminal => terminal_usages[symbol.index as usize] = true,
-                SymbolType::External => external_usages[symbol.index as usize] = true,
+                SymbolType::Terminal => terminal_usages[symbol.index] = true,
+                SymbolType::External => external_usages[symbol.index] = true,
                 _ => {}
             }
         }
         for symbol in state.nonterminal_entries.keys() {
-            non_terminal_usages[symbol.index as usize] = true;
+            non_terminal_usages[symbol.index] = true;
         }
     }
     parse_table.symbols.push(Symbol::end());
@@ -281,10 +256,7 @@ fn populate_used_symbols(
             // ensure that a subtree's symbol can be successfully reassigned to the word token
             // without having to move the subtree to the heap.
             // See https://github.com/tree-sitter/tree-sitter/issues/258
-            if syntax_grammar
-                .word_token
-                .is_some_and(|t| t.index as usize == i)
-            {
+            if syntax_grammar.word_token.is_some_and(|t| t.index == i) {
                 parse_table.symbols.insert(1, Symbol::terminal(i));
             } else {
                 parse_table.symbols.push(Symbol::terminal(i));
@@ -304,7 +276,7 @@ fn populate_used_symbols(
 }
 
 fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &SyntaxGrammar) {
-    let mut external_tokens_by_corresponding_internal_token = FxHashMap::default();
+    let mut external_tokens_by_corresponding_internal_token = HashMap::new();
     for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
         if let Some(symbol) = external_token.corresponding_internal_token {
             external_tokens_by_corresponding_internal_token.insert(symbol.index, i);
@@ -320,11 +292,12 @@ fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &S
         for token in parse_table.states[i].terminal_entries.keys() {
             if token.is_external() {
                 external_tokens.insert(*token);
-            } else if token.is_terminal()
-                && let Some(index) =
+            } else if token.is_terminal() {
+                if let Some(index) =
                     external_tokens_by_corresponding_internal_token.get(&token.index)
-            {
-                external_tokens.insert(Symbol::external(*index));
+                {
+                    external_tokens.insert(Symbol::external(*index));
+                }
             }
         }
 
@@ -335,16 +308,16 @@ fn populate_external_lex_states(parse_table: &mut ParseTable, syntax_grammar: &S
             .unwrap_or_else(|| {
                 parse_table.external_lex_states.push(external_tokens);
                 parse_table.external_lex_states.len() - 1
-            }) as u32;
+            });
     }
 }
 
 fn identify_keywords(
     lexical_grammar: &LexicalGrammar,
+    parse_table: &ParseTable,
     word_token: Option<Symbol>,
     token_conflict_map: &TokenConflictMap,
     coincident_token_index: &CoincidentTokenIndex,
-    str_pool: &StrPool,
 ) -> TokenSet {
     if word_token.is_none() {
         return TokenSet::new();
@@ -362,12 +335,12 @@ fn identify_keywords(
         .filter_map(|(i, variable)| {
             cursor.reset(vec![variable.start_state]);
             if all_chars_are_alphabetical(&cursor)
-                && token_conflict_map.does_match_same_string(i, word_token.index as usize)
-                && !token_conflict_map.does_match_different_string(i, word_token.index as usize)
+                && token_conflict_map.does_match_same_string(i, word_token.index)
+                && !token_conflict_map.does_match_different_string(i, word_token.index)
             {
-                debug!(
+                info!(
                     "Keywords - add candidate {}",
-                    str_pool.resolve(lexical_grammar.variables[i].name)
+                    lexical_grammar.variables[i].name
                 );
                 Some(Symbol::terminal(i))
             } else {
@@ -382,14 +355,12 @@ fn identify_keywords(
         .filter(|token| {
             for other_token in keyword_candidates.iter() {
                 if other_token != *token
-                    && token_conflict_map
-                        .does_match_same_string(other_token.index as usize, token.index as usize)
+                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
                 {
-                    debug!(
+                    info!(
                         "Keywords - exclude {} because it matches the same string as {}",
-                        str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
-                        str_pool
-                            .resolve(lexical_grammar.variables[other_token.index as usize].name)
+                        lexical_grammar.variables[token.index].name,
+                        lexical_grammar.variables[other_token.index].name
                     );
                     return false;
                 }
@@ -400,12 +371,11 @@ fn identify_keywords(
 
     // Exclude keyword candidates for which substituting the keyword capture
     // token would introduce new lexical conflicts with other tokens.
-
-    keywords
+    let keywords = keywords
         .iter()
         .filter(|token| {
             for other_index in 0..lexical_grammar.variables.len() {
-                if keyword_candidates.contains(Symbol::terminal(other_index)) {
+                if keyword_candidates.contains(&Symbol::terminal(other_index)) {
                     continue;
                 }
 
@@ -413,48 +383,62 @@ fn identify_keywords(
                 // this keyword candidate, then substituting the word token won't
                 // introduce any new lexical conflicts.
                 if coincident_token_index
-                    .all_coincident_states_have_word(*token, Symbol::terminal(other_index))
+                    .states_with(*token, Symbol::terminal(other_index))
+                    .iter()
+                    .all(|state_id| {
+                        parse_table.states[*state_id]
+                            .terminal_entries
+                            .contains_key(&word_token)
+                    })
                 {
                     continue;
                 }
 
                 if !token_conflict_map.has_same_conflict_status(
-                    token.index as usize,
-                    word_token.index as usize,
+                    token.index,
+                    word_token.index,
                     other_index,
                 ) {
-                    debug!(
+                    info!(
                         "Keywords - exclude {} because of conflict with {}",
-                        str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
-                        str_pool.resolve(lexical_grammar.variables[other_index].name)
+                        lexical_grammar.variables[token.index].name,
+                        lexical_grammar.variables[other_index].name
                     );
                     return false;
                 }
             }
 
-            debug!(
+            info!(
                 "Keywords - include {}",
-                str_pool.resolve(lexical_grammar.variables[token.index as usize].name),
+                lexical_grammar.variables[token.index].name,
             );
             true
         })
-        .collect()
+        .collect();
+
+    keywords
 }
 
-fn mark_fragile_tokens(parse_table: &mut ParseTable, token_conflict_map: &TokenConflictMap) {
-    let mut valid_terminal_indices = Vec::new();
+fn mark_fragile_tokens(
+    parse_table: &mut ParseTable,
+    lexical_grammar: &LexicalGrammar,
+    token_conflict_map: &TokenConflictMap,
+) {
+    let n = lexical_grammar.variables.len();
+    let mut valid_tokens_mask = Vec::with_capacity(n);
     for state in &mut parse_table.states {
-        valid_terminal_indices.clear();
+        valid_tokens_mask.clear();
+        valid_tokens_mask.resize(n, false);
         for token in state.terminal_entries.keys() {
             if token.is_terminal() {
-                valid_terminal_indices.push(token.index);
+                valid_tokens_mask[token.index] = true;
             }
         }
-        for (token, id) in &mut state.terminal_entries {
+        for (token, entry) in &mut state.terminal_entries {
             if token.is_terminal() {
-                for &i in &valid_terminal_indices {
-                    if token_conflict_map.does_overlap(i as usize, token.index as usize) {
-                        id.set_reusable(false);
+                for (i, is_valid) in valid_tokens_mask.iter().enumerate() {
+                    if *is_valid && token_conflict_map.does_overlap(i, token.index) {
+                        entry.reusable = false;
                         break;
                     }
                 }
@@ -467,8 +451,7 @@ fn report_state_info<'a>(
     syntax_grammar: &SyntaxGrammar,
     lexical_grammar: &LexicalGrammar,
     parse_table: &ParseTable,
-    parse_state_info: &ParseStateInfo<'a>,
-    str_pool: &StrPool,
+    parse_state_info: &[ParseStateInfo<'a>],
     report_symbol_name: &'a str,
 ) {
     let mut all_state_indices = BTreeSet::new();
@@ -478,8 +461,8 @@ fn report_state_info<'a>(
 
     for (i, state) in parse_table.states.iter().enumerate() {
         all_state_indices.insert(i);
-        let item_set = parse_state_info.item_set(state.id);
-        for entry in &item_set.entries {
+        let item_set = &parse_state_info[state.id];
+        for entry in &item_set.1.entries {
             if !entry.item.is_augmented() {
                 symbols_with_state_indices[entry.item.variable_index as usize]
                     .1
@@ -493,18 +476,18 @@ fn report_state_info<'a>(
     let max_symbol_name_length = syntax_grammar
         .variables
         .iter()
-        .map(|v| str_pool.resolve(v.name).len())
+        .map(|v| v.name.len())
         .max()
         .unwrap();
     for (symbol, states) in &symbols_with_state_indices {
-        info!(
+        eprintln!(
             "{:width$}\t{}",
-            str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name),
+            syntax_grammar.variables[symbol.index].name,
             states.len(),
             width = max_symbol_name_length
         );
     }
-    info!("");
+    eprintln!();
 
     let state_indices = if report_symbol_name == "*" {
         Some(&all_state_indices)
@@ -512,9 +495,7 @@ fn report_state_info<'a>(
         symbols_with_state_indices
             .iter()
             .find_map(|(symbol, state_indices)| {
-                if str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name)
-                    == report_symbol_name
-                {
+                if syntax_grammar.variables[symbol.index].name == report_symbol_name {
                     Some(state_indices)
                 } else {
                     None
@@ -528,36 +509,23 @@ fn report_state_info<'a>(
 
         for state_index in state_indices {
             let id = parse_table.states[state_index].id;
-            let preceding_symbols = &parse_state_info.preceding_symbols_by_id[id as usize];
-            let item_set = parse_state_info.item_set(id);
-            info!("state index: {state_index}");
-            info!("state id: {id}");
-            info!(
-                "symbol sequence: {}",
-                preceding_symbols
-                    .iter()
-                    .map(|symbol| {
-                        if symbol.is_terminal() {
-                            str_pool.resolve(lexical_grammar.variables[symbol.index as usize].name)
-                        } else if symbol.is_external() {
-                            str_pool
-                                .resolve(syntax_grammar.external_tokens[symbol.index as usize].name)
-                        } else {
-                            str_pool.resolve(syntax_grammar.variables[symbol.index as usize].name)
-                        }
-                    })
-                    .collect::<Vec<_>>()
-                    .join(" ")
-            );
-            info!(
+            let (preceding_symbols, item_set) = &parse_state_info[id];
+            eprintln!("state index: {state_index}");
+            eprintln!("state id: {id}");
+            eprint!("symbol sequence:");
+            for symbol in preceding_symbols {
+                let name = if symbol.is_terminal() {
+                    &lexical_grammar.variables[symbol.index].name
+                } else if symbol.is_external() {
+                    &syntax_grammar.external_tokens[symbol.index].name
+                } else {
+                    &syntax_grammar.variables[symbol.index].name
+                };
+                eprint!(" {name}");
+            }
+            eprintln!(
                 "\nitems:\n{}",
-                item::ParseItemSetDisplay(
-                    item_set,
-                    syntax_grammar,
-                    lexical_grammar,
-                    str_pool,
-                    &parse_state_info.lookaheads
-                ),
+                item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar),
             );
         }
     }

cli/generate/src/build_tables/token_conflicts.rs

@@ -0,0 +1,529 @@
+use std::{cmp::Ordering, collections::HashSet, fmt};
+
+use crate::{
+    build_tables::item::TokenSetDisplay,
+    grammars::{LexicalGrammar, SyntaxGrammar},
+    nfa::{CharacterSet, NfaCursor, NfaTransition},
+    rules::TokenSet,
+};
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+struct TokenConflictStatus {
+    matches_prefix: bool,
+    does_match_continuation: bool,
+    does_match_valid_continuation: bool,
+    does_match_separators: bool,
+    matches_same_string: bool,
+    matches_different_string: bool,
+}
+
+pub struct TokenConflictMap<'a> {
+    n: usize,
+    status_matrix: Vec<TokenConflictStatus>,
+    following_tokens: Vec<TokenSet>,
+    starting_chars_by_index: Vec<CharacterSet>,
+    following_chars_by_index: Vec<CharacterSet>,
+    grammar: &'a LexicalGrammar,
+}
+
+impl<'a> TokenConflictMap<'a> {
+    /// Create a token conflict map based on a lexical grammar, which describes the structure
+    /// each token, and a `following_token` map, which indicates which tokens may be appear
+    /// immediately after each other token.
+    ///
+    /// This analyzes the possible kinds of overlap between each pair of tokens and stores
+    /// them in a matrix.
+    pub fn new(grammar: &'a LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
+        let mut cursor = NfaCursor::new(&grammar.nfa, Vec::new());
+        let starting_chars = get_starting_chars(&mut cursor, grammar);
+        let following_chars = get_following_chars(&starting_chars, &following_tokens);
+
+        let n = grammar.variables.len();
+        let mut status_matrix = vec![TokenConflictStatus::default(); n * n];
+        for i in 0..grammar.variables.len() {
+            for j in 0..i {
+                let status = compute_conflict_status(&mut cursor, grammar, &following_chars, i, j);
+                status_matrix[matrix_index(n, i, j)] = status.0;
+                status_matrix[matrix_index(n, j, i)] = status.1;
+            }
+        }
+
+        TokenConflictMap {
+            n,
+            status_matrix,
+            following_tokens,
+            starting_chars_by_index: starting_chars,
+            following_chars_by_index: following_chars,
+            grammar,
+        }
+    }
+
+    /// Does token `i` match any strings that token `j` also matches, such that token `i`
+    /// is preferred over token `j`?
+    pub fn has_same_conflict_status(&self, a: usize, b: usize, other: usize) -> bool {
+        let left = &self.status_matrix[matrix_index(self.n, a, other)];
+        let right = &self.status_matrix[matrix_index(self.n, b, other)];
+        left == right
+    }
+
+    /// Does token `i` match any strings that token `j` does *not* match?
+    pub fn does_match_different_string(&self, i: usize, j: usize) -> bool {
+        self.status_matrix[matrix_index(self.n, i, j)].matches_different_string
+    }
+
+    /// Does token `i` match any strings that token `j` also matches, where
+    /// token `i` is preferred over token `j`?
+    pub fn does_match_same_string(&self, i: usize, j: usize) -> bool {
+        self.status_matrix[matrix_index(self.n, i, j)].matches_same_string
+    }
+
+    pub fn does_conflict(&self, i: usize, j: usize) -> bool {
+        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
+        entry.does_match_valid_continuation
+            || entry.does_match_separators
+            || entry.matches_same_string
+    }
+
+    /// Does token `i` match any strings that are *prefixes* of strings matched by `j`?
+    pub fn does_match_prefix(&self, i: usize, j: usize) -> bool {
+        self.status_matrix[matrix_index(self.n, i, j)].matches_prefix
+    }
+
+    pub fn does_match_shorter_or_longer(&self, i: usize, j: usize) -> bool {
+        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
+        let reverse_entry = &self.status_matrix[matrix_index(self.n, j, i)];
+        (entry.does_match_valid_continuation || entry.does_match_separators)
+            && !reverse_entry.does_match_separators
+    }
+
+    pub fn does_overlap(&self, i: usize, j: usize) -> bool {
+        let status = &self.status_matrix[matrix_index(self.n, i, j)];
+        status.does_match_separators
+            || status.matches_prefix
+            || status.matches_same_string
+            || status.does_match_continuation
+    }
+
+    pub fn prefer_token(grammar: &LexicalGrammar, left: (i32, usize), right: (i32, usize)) -> bool {
+        match left.0.cmp(&right.0) {
+            Ordering::Less => false,
+            Ordering::Greater => true,
+            Ordering::Equal => match grammar.variables[left.1]
+                .implicit_precedence
+                .cmp(&grammar.variables[right.1].implicit_precedence)
+            {
+                Ordering::Less => false,
+                Ordering::Greater => true,
+                Ordering::Equal => left.1 < right.1,
+            },
+        }
+    }
+
+    pub fn prefer_transition(
+        grammar: &LexicalGrammar,
+        t: &NfaTransition,
+        completed_id: usize,
+        completed_precedence: i32,
+        has_separator_transitions: bool,
+    ) -> bool {
+        if t.precedence < completed_precedence {
+            return false;
+        }
+        if t.precedence == completed_precedence {
+            if t.is_separator {
+                return false;
+            }
+            if has_separator_transitions
+                && !grammar
+                    .variable_indices_for_nfa_states(&t.states)
+                    .any(|i| i == completed_id)
+            {
+                return false;
+            }
+        }
+        true
+    }
+}
+
+impl fmt::Debug for TokenConflictMap<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "TokenConflictMap {{")?;
+
+        let syntax_grammar = SyntaxGrammar::default();
+
+        writeln!(f, "  following_tokens: {{")?;
+        for (i, following_tokens) in self.following_tokens.iter().enumerate() {
+            writeln!(
+                f,
+                "    follow({:?}): {},",
+                self.grammar.variables[i].name,
+                TokenSetDisplay(following_tokens, &syntax_grammar, self.grammar)
+            )?;
+        }
+        writeln!(f, "  }},")?;
+
+        writeln!(f, "  starting_characters: {{")?;
+        for i in 0..self.n {
+            writeln!(
+                f,
+                "    {:?}: {:?},",
+                self.grammar.variables[i].name, self.starting_chars_by_index[i]
+            )?;
+        }
+        writeln!(f, "  }},")?;
+
+        writeln!(f, "  following_characters: {{")?;
+        for i in 0..self.n {
+            writeln!(
+                f,
+                "    {:?}: {:?},",
+                self.grammar.variables[i].name, self.following_chars_by_index[i]
+            )?;
+        }
+        writeln!(f, "  }},")?;
+
+        writeln!(f, "  status_matrix: {{")?;
+        for i in 0..self.n {
+            writeln!(f, "    {:?}: {{", self.grammar.variables[i].name)?;
+            for j in 0..self.n {
+                writeln!(
+                    f,
+                    "      {:?}: {:?},",
+                    self.grammar.variables[j].name,
+                    self.status_matrix[matrix_index(self.n, i, j)]
+                )?;
+            }
+            writeln!(f, "    }},")?;
+        }
+        write!(f, "  }},")?;
+        write!(f, "}}")?;
+        Ok(())
+    }
+}
+
+const fn matrix_index(variable_count: usize, i: usize, j: usize) -> usize {
+    variable_count * i + j
+}
+
+fn get_starting_chars(cursor: &mut NfaCursor, grammar: &LexicalGrammar) -> Vec<CharacterSet> {
+    let mut result = Vec::with_capacity(grammar.variables.len());
+    for variable in &grammar.variables {
+        cursor.reset(vec![variable.start_state]);
+        let mut all_chars = CharacterSet::empty();
+        for (chars, _) in cursor.transition_chars() {
+            all_chars = all_chars.add(chars);
+        }
+        result.push(all_chars);
+    }
+    result
+}
+
+fn get_following_chars(
+    starting_chars: &[CharacterSet],
+    following_tokens: &[TokenSet],
+) -> Vec<CharacterSet> {
+    following_tokens
+        .iter()
+        .map(|following_tokens| {
+            let mut chars = CharacterSet::empty();
+            for token in following_tokens.iter() {
+                if token.is_terminal() {
+                    chars = chars.add(&starting_chars[token.index]);
+                }
+            }
+            chars
+        })
+        .collect()
+}
+
+fn compute_conflict_status(
+    cursor: &mut NfaCursor,
+    grammar: &LexicalGrammar,
+    following_chars: &[CharacterSet],
+    i: usize,
+    j: usize,
+) -> (TokenConflictStatus, TokenConflictStatus) {
+    let mut visited_state_sets = HashSet::new();
+    let mut state_set_queue = vec![vec![
+        grammar.variables[i].start_state,
+        grammar.variables[j].start_state,
+    ]];
+    let mut result = (
+        TokenConflictStatus::default(),
+        TokenConflictStatus::default(),
+    );
+
+    while let Some(state_set) = state_set_queue.pop() {
+        let mut live_variable_indices = grammar.variable_indices_for_nfa_states(&state_set);
+
+        // If only one of the two tokens could possibly match from this state, then
+        // there is no reason to analyze any of its successors. Just record the fact
+        // that the token matches a string that the other token does not match.
+        let first_live_variable_index = live_variable_indices.next().unwrap();
+        if live_variable_indices.count() == 0 {
+            if first_live_variable_index == i {
+                result.0.matches_different_string = true;
+            } else {
+                result.1.matches_different_string = true;
+            }
+            continue;
+        }
+
+        // Don't pursue states where there's no potential for conflict.
+        cursor.reset(state_set);
+        let within_separator = cursor.transition_chars().any(|(_, sep)| sep);
+
+        // Examine each possible completed token in this state.
+        let mut completion = None;
+        for (id, precedence) in cursor.completions() {
+            if within_separator {
+                if id == i {
+                    result.0.does_match_separators = true;
+                } else {
+                    result.1.does_match_separators = true;
+                }
+            }
+
+            // If the other token has already completed, then this is
+            // a same-string conflict.
+            if let Some((prev_id, prev_precedence)) = completion {
+                if id == prev_id {
+                    continue;
+                }
+
+                // Determine which of the two tokens is preferred.
+                let preferred_id;
+                if TokenConflictMap::prefer_token(
+                    grammar,
+                    (prev_precedence, prev_id),
+                    (precedence, id),
+                ) {
+                    preferred_id = prev_id;
+                } else {
+                    preferred_id = id;
+                    completion = Some((id, precedence));
+                }
+
+                if preferred_id == i {
+                    result.0.matches_same_string = true;
+                } else {
+                    result.1.matches_same_string = true;
+                }
+            } else {
+                completion = Some((id, precedence));
+            }
+        }
+
+        // Examine each possible transition from this state to detect substring conflicts.
+        for transition in cursor.transitions() {
+            let mut can_advance = true;
+
+            // If there is already a completed token in this state, then determine
+            // if the next state can also match the completed token. If so, then
+            // this is *not* a conflict.
+            if let Some((completed_id, completed_precedence)) = completion {
+                let mut advanced_id = None;
+                let mut successor_contains_completed_id = false;
+                for variable_id in grammar.variable_indices_for_nfa_states(&transition.states) {
+                    if variable_id == completed_id {
+                        successor_contains_completed_id = true;
+                        break;
+                    }
+                    advanced_id = Some(variable_id);
+                }
+
+                // Determine which action is preferred: matching the already complete
+                // token, or continuing on to try and match the other longer token.
+                if let (Some(advanced_id), false) = (advanced_id, successor_contains_completed_id) {
+                    if TokenConflictMap::prefer_transition(
+                        grammar,
+                        &transition,
+                        completed_id,
+                        completed_precedence,
+                        within_separator,
+                    ) {
+                        can_advance = true;
+                        if advanced_id == i {
+                            result.0.does_match_continuation = true;
+                            if transition.characters.does_intersect(&following_chars[j]) {
+                                result.0.does_match_valid_continuation = true;
+                            }
+                        } else {
+                            result.1.does_match_continuation = true;
+                            if transition.characters.does_intersect(&following_chars[i]) {
+                                result.1.does_match_valid_continuation = true;
+                            }
+                        }
+                    } else if completed_id == i {
+                        result.0.matches_prefix = true;
+                    } else {
+                        result.1.matches_prefix = true;
+                    }
+                }
+            }
+
+            if can_advance && visited_state_sets.insert(transition.states.clone()) {
+                state_set_queue.push(transition.states);
+            }
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::{Variable, VariableType},
+        prepare_grammar::{expand_tokens, ExtractedLexicalGrammar},
+        rules::{Precedence, Rule, Symbol},
+    };
+
+    #[test]
+    fn test_starting_characters() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: Vec::new(),
+            variables: vec![
+                Variable {
+                    name: "token_0".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::pattern("[a-f]1|0x\\d", ""),
+                },
+                Variable {
+                    name: "token_1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::pattern("d*ef", ""),
+                },
+            ],
+        })
+        .unwrap();
+
+        let token_map = TokenConflictMap::new(&grammar, Vec::new());
+
+        assert_eq!(
+            token_map.starting_chars_by_index[0],
+            CharacterSet::empty().add_range('a', 'f').add_char('0')
+        );
+        assert_eq!(
+            token_map.starting_chars_by_index[1],
+            CharacterSet::empty().add_range('d', 'e')
+        );
+    }
+
+    #[test]
+    fn test_token_conflicts() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: Vec::new(),
+            variables: vec![
+                Variable {
+                    name: "in".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("in"),
+                },
+                Variable {
+                    name: "identifier".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::pattern("\\w+", ""),
+                },
+                Variable {
+                    name: "instanceof".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("instanceof"),
+                },
+            ],
+        })
+        .unwrap();
+
+        let var = |name| index_of_var(&grammar, name);
+
+        let token_map = TokenConflictMap::new(
+            &grammar,
+            vec![
+                std::iter::once(&Symbol::terminal(var("identifier")))
+                    .copied()
+                    .collect(),
+                std::iter::once(&Symbol::terminal(var("in")))
+                    .copied()
+                    .collect(),
+                std::iter::once(&Symbol::terminal(var("identifier")))
+                    .copied()
+                    .collect(),
+            ],
+        );
+
+        // Given the string "in", the `in` token is preferred over the `identifier` token
+        assert!(token_map.does_match_same_string(var("in"), var("identifier")));
+        assert!(!token_map.does_match_same_string(var("identifier"), var("in")));
+
+        // Depending on what character follows, the string "in" may be treated as part of an
+        // `identifier` token.
+        assert!(token_map.does_conflict(var("identifier"), var("in")));
+
+        // Depending on what character follows, the string "instanceof" may be treated as part of
+        // an `identifier` token.
+        assert!(token_map.does_conflict(var("identifier"), var("instanceof")));
+        assert!(token_map.does_conflict(var("instanceof"), var("in")));
+    }
+
+    #[test]
+    fn test_token_conflicts_with_separators() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: vec![Rule::pattern("\\s", "")],
+            variables: vec![
+                Variable {
+                    name: "x".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                Variable {
+                    name: "newline".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("\n"),
+                },
+            ],
+        })
+        .unwrap();
+
+        let var = |name| index_of_var(&grammar, name);
+
+        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
+
+        assert!(token_map.does_conflict(var("newline"), var("x")));
+        assert!(!token_map.does_conflict(var("x"), var("newline")));
+    }
+
+    #[test]
+    fn test_token_conflicts_with_open_ended_tokens() {
+        let grammar = expand_tokens(ExtractedLexicalGrammar {
+            separators: vec![Rule::pattern("\\s", "")],
+            variables: vec![
+                Variable {
+                    name: "x".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::string("x"),
+                },
+                Variable {
+                    name: "anything".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*", "")),
+                },
+            ],
+        })
+        .unwrap();
+
+        let var = |name| index_of_var(&grammar, name);
+
+        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
+
+        assert!(token_map.does_match_shorter_or_longer(var("anything"), var("x")));
+        assert!(!token_map.does_match_shorter_or_longer(var("x"), var("anything")));
+    }
+
+    fn index_of_var(grammar: &LexicalGrammar, name: &str) -> usize {
+        grammar
+            .variables
+            .iter()
+            .position(|v| v.name == name)
+            .unwrap()
+    }
+}

cli/generate/src/dedup.rs

@@ -1,25 +1,21 @@
-#[inline]
 pub fn split_state_id_groups<S>(
     states: &[S],
-    state_ids_by_group_id: &mut Vec<Vec<u32>>,
-    group_ids_by_state_id: &mut [u32],
-    start_group_id: u32,
-    mut should_split: impl FnMut(&S, &S, &[u32]) -> bool,
+    state_ids_by_group_id: &mut Vec<Vec<usize>>,
+    group_ids_by_state_id: &mut [usize],
+    start_group_id: usize,
+    mut should_split: impl FnMut(&S, &S, &[usize]) -> bool,
 ) -> bool {
     let mut result = false;
 
-    // Use a flat bitset instead of Vec::contains for O(1) membership tests.
-    let mut is_split = vec![false; states.len()];
-
     let mut group_id = start_group_id;
-    while (group_id as usize) < state_ids_by_group_id.len() {
-        let state_ids = &state_ids_by_group_id[group_id as usize];
+    while group_id < state_ids_by_group_id.len() {
+        let state_ids = &state_ids_by_group_id[group_id];
         let mut split_state_ids = Vec::new();
 
         let mut i = 0;
         while i < state_ids.len() {
-            let left_state_id = state_ids[i] as usize;
-            if is_split[left_state_id] {
+            let left_state_id = state_ids[i];
+            if split_state_ids.contains(&left_state_id) {
                 i += 1;
                 continue;
             }
@@ -30,16 +26,15 @@ pub fn split_state_id_groups<S>(
             // this state.
             let mut j = i + 1;
             while j < state_ids.len() {
-                let right_state_id = state_ids[j] as usize;
-                if is_split[right_state_id] {
+                let right_state_id = state_ids[j];
+                if split_state_ids.contains(&right_state_id) {
                     j += 1;
                     continue;
                 }
                 let right_state = &states[right_state_id];
 
                 if should_split(left_state, right_state, group_ids_by_state_id) {
-                    split_state_ids.push(right_state_id as u32);
-                    is_split[right_state_id] = true;
+                    split_state_ids.push(right_state_id);
                 }
 
                 j += 1;
@@ -51,12 +46,11 @@ pub fn split_state_id_groups<S>(
         // If any states were removed from the group, add them all as a new group.
         if !split_state_ids.is_empty() {
             result = true;
-            state_ids_by_group_id[group_id as usize].retain(|i| !is_split[*i as usize]);
+            state_ids_by_group_id[group_id].retain(|i| !split_state_ids.contains(i));
 
-            let new_group_id = state_ids_by_group_id.len() as u32;
+            let new_group_id = state_ids_by_group_id.len();
             for id in &split_state_ids {
-                group_ids_by_state_id[*id as usize] = new_group_id;
-                is_split[*id as usize] = false;
+                group_ids_by_state_id[*id] = new_group_id;
             }
 
             state_ids_by_group_id.push(split_state_ids);

cli/generate/src/dsl.js

@@ -33,16 +33,7 @@ function blank() {
   };
 }
 
-function eof() {
-  return {
-    type: "EOF"
-  };
-}
-
 function field(name, rule) {
-  if (typeof name !== "string" || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
-    throw new Error(`Invalid field name '${name}': field names must start with a letter or underscore, followed by letters, digits, or underscores`);
-  }
   return {
     type: "FIELD",
     name,
@@ -79,7 +70,7 @@ function prec(number, rule) {
   };
 }
 
-prec.left = function (number, rule) {
+prec.left = function(number, rule) {
   if (rule == null) {
     rule = number;
     number = 0;
@@ -101,7 +92,7 @@ prec.left = function (number, rule) {
   };
 }
 
-prec.right = function (number, rule) {
+prec.right = function(number, rule) {
   if (rule == null) {
     rule = number;
     number = 0;
@@ -123,7 +114,7 @@ prec.right = function (number, rule) {
   };
 }
 
-prec.dynamic = function (number, rule) {
+prec.dynamic = function(number, rule) {
   checkPrecedence(number);
   checkArguments(
     arguments,
@@ -193,7 +184,7 @@ function token(value) {
   };
 }
 
-token.immediate = function (value) {
+token.immediate = function(value) {
   checkArguments(arguments, arguments.length, token.immediate, 'token.immediate', '', 'literal');
   return {
     type: "IMMEDIATE_TOKEN",
@@ -340,7 +331,7 @@ function grammar(baseGrammar, options) {
     }
   }
 
-  let reserved = { ...baseGrammar.reserved };
+  let reserved = baseGrammar.reserved;
   if (options.reserved) {
     if (typeof options.reserved !== "object") {
       throw new Error("Grammar's 'reserved' property must be an object.");
@@ -380,16 +371,14 @@ function grammar(baseGrammar, options) {
 
   let word = baseGrammar.word;
   if (options.word) {
-    const wordRule = options.word.call(ruleBuilder, ruleBuilder);
-    if (wordRule?.name === 'ReferenceError') {
-      throw new Error("Grammar's 'word' property must be a valid named rule.");
-    }
-
-    if (wordRule?.type !== 'SYMBOL' || typeof wordRule.name !== 'string') {
+    word = options.word.call(ruleBuilder, ruleBuilder).name;
+    if (typeof word != 'string') {
       throw new Error("Grammar's 'word' property must be a named rule.");
     }
 
-    word = wordRule.name;
+    if (word === 'ReferenceError') {
+      throw new Error("Grammar's 'word' property must be a valid rule name.");
+    }
   }
 
   let conflicts = baseGrammar.conflicts;
@@ -410,13 +399,7 @@ function grammar(baseGrammar, options) {
         throw new Error("Grammar's conflicts must be an array of arrays of rules.");
       }
 
-      return conflictSet.map(symbol => {
-        const rule = normalize(symbol);
-        if (rule.type !== 'SYMBOL') {
-          throw new Error("Grammar's conflicts must contain only named rules.");
-        }
-        return rule.name;
-      });
+      return conflictSet.map(symbol => normalize(symbol).name);
     });
   }
 
@@ -434,15 +417,12 @@ function grammar(baseGrammar, options) {
     }
 
     inline = inlineRules.filter((symbol, index, self) => {
-      if (symbol?.name === 'ReferenceError') {
-        console.log(`Warning: inline rule '${symbol.symbol.name}' is not defined.`);
+      if (self.findIndex(s => s.name === symbol.name) !== index) {
+        console.log(`Warning: duplicate inline rule '${symbol.name}'`);
         return false;
       }
-      if (symbol?.type !== 'SYMBOL' || typeof symbol.name !== 'string') {
-        throw new Error("Grammar's inline property must contain only named rules.");
-      }
-      if (self.findIndex(s => s?.name === symbol.name) !== index) {
-        console.log(`Warning: duplicate inline rule '${symbol.name}'`);
+      if (symbol.name === 'ReferenceError') {
+        console.log(`Warning: inline rule '${symbol.symbol.name}' is not defined.`);
         return false;
       }
       return true;
@@ -463,12 +443,9 @@ function grammar(baseGrammar, options) {
     }
 
     supertypes = supertypeRules.map(symbol => {
-      if (symbol?.name === 'ReferenceError') {
+      if (symbol.name === 'ReferenceError') {
         throw new Error(`Supertype rule \`${symbol.symbol.name}\` is not defined.`);
       }
-      if (symbol?.type !== 'SYMBOL' || typeof symbol.name !== 'string') {
-        throw new Error("Grammar's supertypes property must contain only named rules.");
-      }
       return symbol.name;
     });
   }
@@ -486,15 +463,7 @@ function grammar(baseGrammar, options) {
       if (!Array.isArray(list)) {
         throw new Error("Grammar's precedences must be an array of arrays of rules.");
       }
-      return list.map(entry => {
-        const rule = normalize(entry);
-        if (rule.type !== 'STRING' && rule.type !== 'SYMBOL') {
-          throw new Error(
-            "Grammar's precedences must contain only precedence names or named rules."
-          );
-        }
-        return rule;
-      });
+      return list.map(normalize);
     });
   }
 
@@ -548,7 +517,6 @@ function checkPrecedence(value) {
 }
 
 function getEnv(name) {
-  if (globalThis.native) return globalThis.__ts_grammar_path;
   if (globalThis.process) return process.env[name]; // Node/Bun
   if (globalThis.Deno) return Deno.env.get(name); // Deno
   throw Error("Unsupported JS runtime");
@@ -556,13 +524,12 @@ function getEnv(name) {
 
 globalThis.alias = alias;
 globalThis.blank = blank;
-globalThis.eof = eof;
 globalThis.choice = choice;
 globalThis.optional = optional;
 globalThis.prec = prec;
 globalThis.repeat = repeat;
 globalThis.repeat1 = repeat1;
-globalThis.reserved = reserved;
+global.reserved = reserved;
 globalThis.seq = seq;
 globalThis.sym = sym;
 globalThis.token = token;
@@ -570,23 +537,14 @@ globalThis.grammar = grammar;
 globalThis.field = field;
 globalThis.RustRegex = RustRegex;
 
-const grammarPath = getEnv("TREE_SITTER_GRAMMAR_PATH");
-let result = await import(grammarPath);
-let grammarObj = result.default?.grammar ?? result.grammar;
-
-if (globalThis.native && !grammarObj) {
-  grammarObj = module.exports.grammar;
-}
-
+const result = await import(getEnv("TREE_SITTER_GRAMMAR_PATH"));
 const object = {
   "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
-  ...grammarObj,
+  ...(result.default?.grammar ?? result.grammar)
 };
 const output = JSON.stringify(object);
 
-if (globalThis.native) {
-  globalThis.output = output;
-} else if (globalThis.process) { // Node/Bun
+if (globalThis.process) { // Node/Bun
   process.stdout.write(output);
 } else if (globalThis.Deno) { // Deno
   Deno.stdout.writeSync(new TextEncoder().encode(output));

cli/generate/src/grammar_files.rs

@@ -0,0 +1 @@
+

cli/generate/src/grammars.rs

@@ -0,0 +1,273 @@
+use std::{collections::HashMap, fmt};
+
+use super::{
+    nfa::Nfa,
+    rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet},
+};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum VariableType {
+    Hidden,
+    Auxiliary,
+    Anonymous,
+    Named,
+}
+
+// Input grammar
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Variable {
+    pub name: String,
+    pub kind: VariableType,
+    pub rule: Rule,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum PrecedenceEntry {
+    Name(String),
+    Symbol(String),
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct InputGrammar {
+    pub name: String,
+    pub variables: Vec<Variable>,
+    pub extra_symbols: Vec<Rule>,
+    pub expected_conflicts: Vec<Vec<String>>,
+    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
+    pub external_tokens: Vec<Rule>,
+    pub variables_to_inline: Vec<String>,
+    pub supertype_symbols: Vec<String>,
+    pub word_token: Option<String>,
+    pub reserved_words: Vec<ReservedWordContext<Rule>>,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct ReservedWordContext<T> {
+    pub name: String,
+    pub reserved_words: Vec<T>,
+}
+
+// Extracted lexical grammar
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct LexicalVariable {
+    pub name: String,
+    pub kind: VariableType,
+    pub implicit_precedence: i32,
+    pub start_state: u32,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct LexicalGrammar {
+    pub nfa: Nfa,
+    pub variables: Vec<LexicalVariable>,
+}
+
+// Extracted syntax grammar
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct ProductionStep {
+    pub symbol: Symbol,
+    pub precedence: Precedence,
+    pub associativity: Option<Associativity>,
+    pub alias: Option<Alias>,
+    pub field_name: Option<String>,
+    pub reserved_word_set_id: ReservedWordSetId,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct ReservedWordSetId(pub usize);
+
+impl fmt::Display for ReservedWordSetId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+pub const NO_RESERVED_WORDS: ReservedWordSetId = ReservedWordSetId(usize::MAX);
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Production {
+    pub steps: Vec<ProductionStep>,
+    pub dynamic_precedence: i32,
+}
+
+#[derive(Default)]
+pub struct InlinedProductionMap {
+    pub productions: Vec<Production>,
+    pub production_map: HashMap<(*const Production, u32), Vec<usize>>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SyntaxVariable {
+    pub name: String,
+    pub kind: VariableType,
+    pub productions: Vec<Production>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ExternalToken {
+    pub name: String,
+    pub kind: VariableType,
+    pub corresponding_internal_token: Option<Symbol>,
+}
+
+#[derive(Debug, Default)]
+pub struct SyntaxGrammar {
+    pub variables: Vec<SyntaxVariable>,
+    pub extra_symbols: Vec<Symbol>,
+    pub expected_conflicts: Vec<Vec<Symbol>>,
+    pub external_tokens: Vec<ExternalToken>,
+    pub supertype_symbols: Vec<Symbol>,
+    pub variables_to_inline: Vec<Symbol>,
+    pub word_token: Option<Symbol>,
+    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
+    pub reserved_word_sets: Vec<TokenSet>,
+}
+
+#[cfg(test)]
+impl ProductionStep {
+    #[must_use]
+    pub fn new(symbol: Symbol) -> Self {
+        Self {
+            symbol,
+            precedence: Precedence::None,
+            associativity: None,
+            alias: None,
+            field_name: None,
+            reserved_word_set_id: ReservedWordSetId::default(),
+        }
+    }
+
+    pub fn with_prec(
+        mut self,
+        precedence: Precedence,
+        associativity: Option<Associativity>,
+    ) -> Self {
+        self.precedence = precedence;
+        self.associativity = associativity;
+        self
+    }
+
+    pub fn with_alias(mut self, value: &str, is_named: bool) -> Self {
+        self.alias = Some(Alias {
+            value: value.to_string(),
+            is_named,
+        });
+        self
+    }
+
+    pub fn with_field_name(mut self, name: &str) -> Self {
+        self.field_name = Some(name.to_string());
+        self
+    }
+}
+
+impl Production {
+    pub fn first_symbol(&self) -> Option<Symbol> {
+        self.steps.first().map(|s| s.symbol)
+    }
+}
+
+#[cfg(test)]
+impl Variable {
+    pub fn named(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Named,
+            rule,
+        }
+    }
+
+    pub fn auxiliary(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Auxiliary,
+            rule,
+        }
+    }
+
+    pub fn hidden(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Hidden,
+            rule,
+        }
+    }
+
+    pub fn anonymous(name: &str, rule: Rule) -> Self {
+        Self {
+            name: name.to_string(),
+            kind: VariableType::Anonymous,
+            rule,
+        }
+    }
+}
+
+impl VariableType {
+    pub fn is_visible(self) -> bool {
+        self == Self::Named || self == Self::Anonymous
+    }
+}
+
+impl LexicalGrammar {
+    pub fn variable_indices_for_nfa_states<'a>(
+        &'a self,
+        state_ids: &'a [u32],
+    ) -> impl Iterator<Item = usize> + 'a {
+        let mut prev = None;
+        state_ids.iter().filter_map(move |state_id| {
+            let variable_id = self.variable_index_for_nfa_state(*state_id);
+            if prev == Some(variable_id) {
+                None
+            } else {
+                prev = Some(variable_id);
+                prev
+            }
+        })
+    }
+
+    pub fn variable_index_for_nfa_state(&self, state_id: u32) -> usize {
+        self.variables
+            .iter()
+            .position(|v| v.start_state >= state_id)
+            .unwrap()
+    }
+}
+
+impl SyntaxVariable {
+    pub fn is_auxiliary(&self) -> bool {
+        self.kind == VariableType::Auxiliary
+    }
+
+    pub fn is_hidden(&self) -> bool {
+        self.kind == VariableType::Hidden || self.kind == VariableType::Auxiliary
+    }
+}
+
+impl InlinedProductionMap {
+    pub fn inlined_productions<'a>(
+        &'a self,
+        production: &Production,
+        step_index: u32,
+    ) -> Option<impl Iterator<Item = &'a Production> + 'a> {
+        self.production_map
+            .get(&(std::ptr::from_ref::<Production>(production), step_index))
+            .map(|production_indices| {
+                production_indices
+                    .iter()
+                    .copied()
+                    .map(move |index| &self.productions[index])
+            })
+    }
+}
+
+impl fmt::Display for PrecedenceEntry {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Name(n) => write!(f, "'{n}'"),
+            Self::Symbol(s) => write!(f, "$.{s}"),
+        }
+    }
+}

cli/generate/src/lib.rs

@@ -0,0 +1,439 @@
+use std::{
+    env, fs,
+    io::Write,
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+    sync::LazyLock,
+};
+
+use anyhow::Result;
+use regex::{Regex, RegexBuilder};
+use semver::Version;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+mod build_tables;
+mod dedup;
+mod grammar_files;
+mod grammars;
+mod nfa;
+mod node_types;
+pub mod parse_grammar;
+mod prepare_grammar;
+mod render;
+mod rules;
+mod tables;
+
+use build_tables::build_tables;
+pub use build_tables::ParseTableBuilderError;
+use grammars::InputGrammar;
+pub use node_types::VariableInfoError;
+use parse_grammar::parse_grammar;
+pub use parse_grammar::ParseGrammarError;
+use prepare_grammar::prepare_grammar;
+pub use prepare_grammar::PrepareGrammarError;
+use render::render_c_code;
+pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
+
+static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
+    RegexBuilder::new("^\\s*//.*")
+        .multi_line(true)
+        .build()
+        .unwrap()
+});
+
+struct GeneratedParser {
+    c_code: String,
+    node_types_json: String,
+}
+
+pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h");
+pub const ARRAY_HEADER: &str = include_str!("templates/array.h");
+
+pub type GenerateResult<T> = Result<T, GenerateError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum GenerateError {
+    #[error("Error with specified path -- {0}")]
+    GrammarPath(String),
+    #[error("{0}")]
+    IO(String),
+    #[error(transparent)]
+    LoadGrammarFile(#[from] LoadGrammarError),
+    #[error(transparent)]
+    ParseGrammar(#[from] ParseGrammarError),
+    #[error(transparent)]
+    Prepare(#[from] PrepareGrammarError),
+    #[error(transparent)]
+    VariableInfo(#[from] VariableInfoError),
+    #[error(transparent)]
+    BuildTables(#[from] ParseTableBuilderError),
+    #[error(transparent)]
+    ParseVersion(#[from] ParseVersionError),
+}
+
+impl From<std::io::Error> for GenerateError {
+    fn from(value: std::io::Error) -> Self {
+        Self::IO(value.to_string())
+    }
+}
+
+pub type LoadGrammarFileResult<T> = Result<T, LoadGrammarError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum LoadGrammarError {
+    #[error("Path to a grammar file with `.js` or `.json` extension is required")]
+    InvalidPath,
+    #[error("Failed to load grammar.js -- {0}")]
+    LoadJSGrammarFile(#[from] JSError),
+    #[error("Failed to load grammar.json -- {0}")]
+    IO(String),
+    #[error("Unknown grammar file extension: {0:?}")]
+    FileExtension(PathBuf),
+}
+
+impl From<std::io::Error> for LoadGrammarError {
+    fn from(value: std::io::Error) -> Self {
+        Self::IO(value.to_string())
+    }
+}
+
+#[derive(Debug, Error, Serialize)]
+pub enum ParseVersionError {
+    #[error("{0}")]
+    Version(String),
+    #[error("{0}")]
+    JSON(String),
+    #[error("{0}")]
+    IO(String),
+}
+
+pub type JSResult<T> = Result<T, JSError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum JSError {
+    #[error("Failed to run `{runtime}` -- {error}")]
+    JSRuntimeSpawn { runtime: String, error: String },
+    #[error("Got invalid UTF8 from `{runtime}` -- {error}")]
+    JSRuntimeUtf8 { runtime: String, error: String },
+    #[error("`{runtime}` process exited with status {code}")]
+    JSRuntimeExit { runtime: String, code: i32 },
+    #[error("{0}")]
+    IO(String),
+    #[error("Could not parse this package's version as semver -- {0}")]
+    Semver(String),
+    #[error("Failed to serialze grammar JSON -- {0}")]
+    Serialzation(String),
+}
+
+impl From<std::io::Error> for JSError {
+    fn from(value: std::io::Error) -> Self {
+        Self::IO(value.to_string())
+    }
+}
+
+impl From<serde_json::Error> for JSError {
+    fn from(value: serde_json::Error) -> Self {
+        Self::Serialzation(value.to_string())
+    }
+}
+
+impl From<semver::Error> for JSError {
+    fn from(value: semver::Error) -> Self {
+        Self::Semver(value.to_string())
+    }
+}
+
+pub fn generate_parser_in_directory(
+    repo_path: &Path,
+    out_path: Option<&str>,
+    grammar_path: Option<&str>,
+    mut abi_version: usize,
+    report_symbol_name: Option<&str>,
+    js_runtime: Option<&str>,
+) -> GenerateResult<()> {
+    let mut repo_path = repo_path.to_owned();
+    let mut grammar_path = grammar_path;
+
+    // Populate a new empty grammar directory.
+    if let Some(path) = grammar_path {
+        let path = PathBuf::from(path);
+        if !path
+            .try_exists()
+            .map_err(|e| GenerateError::GrammarPath(e.to_string()))?
+        {
+            fs::create_dir_all(&path)?;
+            grammar_path = None;
+            repo_path = path;
+        }
+    }
+
+    let grammar_path = grammar_path.map_or_else(|| repo_path.join("grammar.js"), PathBuf::from);
+
+    // Read the grammar file.
+    let grammar_json = load_grammar_file(&grammar_path, js_runtime)?;
+
+    let src_path = out_path.map_or_else(|| repo_path.join("src"), PathBuf::from);
+    let header_path = src_path.join("tree_sitter");
+
+    // Ensure that the output directories exist.
+    fs::create_dir_all(&src_path)?;
+    fs::create_dir_all(&header_path)?;
+
+    if grammar_path.file_name().unwrap() != "grammar.json" {
+        fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| {
+            GenerateError::IO(format!(
+                "Failed to write grammar.json to {src_path:?} -- {e}"
+            ))
+        })?;
+    }
+
+    // Parse and preprocess the grammar.
+    let input_grammar = parse_grammar(&grammar_json)?;
+
+    let semantic_version = read_grammar_version(&repo_path)?;
+
+    if semantic_version.is_none() && abi_version > ABI_VERSION_MIN {
+        println!("Warning: No `tree-sitter.json` file found in your grammar, this file is required to generate with ABI {abi_version}. Using ABI version {ABI_VERSION_MIN} instead.");
+        println!("This file can be set up with `tree-sitter init`. For more information, see https://tree-sitter.github.io/tree-sitter/cli/init.");
+        abi_version = ABI_VERSION_MIN;
+    }
+
+    // Generate the parser and related files.
+    let GeneratedParser {
+        c_code,
+        node_types_json,
+    } = generate_parser_for_grammar_with_opts(
+        &input_grammar,
+        abi_version,
+        semantic_version.map(|v| (v.major as u8, v.minor as u8, v.patch as u8)),
+        report_symbol_name,
+    )?;
+
+    write_file(&src_path.join("parser.c"), c_code)?;
+    write_file(&src_path.join("node-types.json"), node_types_json)?;
+    write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
+    write_file(&header_path.join("array.h"), ARRAY_HEADER)?;
+    write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
+
+    Ok(())
+}
+
+pub fn generate_parser_for_grammar(
+    grammar_json: &str,
+    semantic_version: Option<(u8, u8, u8)>,
+) -> GenerateResult<(String, String)> {
+    let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n");
+    let input_grammar = parse_grammar(&grammar_json)?;
+    let parser = generate_parser_for_grammar_with_opts(
+        &input_grammar,
+        tree_sitter::LANGUAGE_VERSION,
+        semantic_version,
+        None,
+    )?;
+    Ok((input_grammar.name, parser.c_code))
+}
+
+fn generate_parser_for_grammar_with_opts(
+    input_grammar: &InputGrammar,
+    abi_version: usize,
+    semantic_version: Option<(u8, u8, u8)>,
+    report_symbol_name: Option<&str>,
+) -> GenerateResult<GeneratedParser> {
+    let (syntax_grammar, lexical_grammar, inlines, simple_aliases) =
+        prepare_grammar(input_grammar)?;
+    let variable_info =
+        node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?;
+    let node_types_json = node_types::generate_node_types_json(
+        &syntax_grammar,
+        &lexical_grammar,
+        &simple_aliases,
+        &variable_info,
+    );
+    let supertype_symbol_map =
+        node_types::get_supertype_symbol_map(&syntax_grammar, &simple_aliases, &variable_info);
+    let tables = build_tables(
+        &syntax_grammar,
+        &lexical_grammar,
+        &simple_aliases,
+        &variable_info,
+        &inlines,
+        report_symbol_name,
+    )?;
+    let c_code = render_c_code(
+        &input_grammar.name,
+        tables,
+        syntax_grammar,
+        lexical_grammar,
+        simple_aliases,
+        abi_version,
+        semantic_version,
+        supertype_symbol_map,
+    );
+    Ok(GeneratedParser {
+        c_code,
+        node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(),
+    })
+}
+
+/// This will read the `tree-sitter.json` config file and attempt to extract the version.
+///
+/// If the file is not found in the current directory or any of its parent directories, this will
+/// return `None` to maintain backwards compatibility. If the file is found but the version cannot
+/// be parsed as semver, this will return an error.
+fn read_grammar_version(repo_path: &Path) -> Result<Option<Version>, ParseVersionError> {
+    #[derive(Deserialize)]
+    struct TreeSitterJson {
+        metadata: Metadata,
+    }
+
+    #[derive(Deserialize)]
+    struct Metadata {
+        version: String,
+    }
+
+    let filename = "tree-sitter.json";
+    let mut path = repo_path.join(filename);
+
+    loop {
+        let json = path
+            .exists()
+            .then(|| {
+                let contents = fs::read_to_string(path.as_path()).map_err(|e| {
+                    ParseVersionError::IO(format!("Failed to read `{}` -- {e}", path.display()))
+                })?;
+                serde_json::from_str::<TreeSitterJson>(&contents).map_err(|e| {
+                    ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display()))
+                })
+            })
+            .transpose()?;
+        if let Some(json) = json {
+            return Version::parse(&json.metadata.version)
+                .map_err(|e| {
+                    ParseVersionError::Version(format!(
+                        "Failed to parse `{}` version as semver -- {e}",
+                        path.display()
+                    ))
+                })
+                .map(Some);
+        }
+        path.pop(); // filename
+        if !path.pop() {
+            return Ok(None);
+        }
+        path.push(filename);
+    }
+}
+
+pub fn load_grammar_file(
+    grammar_path: &Path,
+    js_runtime: Option<&str>,
+) -> LoadGrammarFileResult<String> {
+    if grammar_path.is_dir() {
+        Err(LoadGrammarError::InvalidPath)?;
+    }
+    match grammar_path.extension().and_then(|e| e.to_str()) {
+        Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?),
+        Some("json") => Ok(fs::read_to_string(grammar_path)?),
+        _ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?,
+    }
+}
+
+fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult<String> {
+    let grammar_path = fs::canonicalize(grammar_path)?;
+
+    #[cfg(windows)]
+    let grammar_path = url::Url::from_file_path(grammar_path)
+        .expect("Failed to convert path to URL")
+        .to_string();
+
+    let js_runtime = js_runtime.unwrap_or("node");
+
+    let mut js_command = Command::new(js_runtime);
+    match js_runtime {
+        "node" => {
+            js_command.args(["--input-type=module", "-"]);
+        }
+        "bun" => {
+            js_command.arg("-");
+        }
+        "deno" => {
+            js_command.args(["run", "--allow-all", "-"]);
+        }
+        _ => {}
+    }
+
+    let mut js_process = js_command
+        .env("TREE_SITTER_GRAMMAR_PATH", grammar_path)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .spawn()
+        .map_err(|e| JSError::JSRuntimeSpawn {
+            runtime: js_runtime.to_string(),
+            error: e.to_string(),
+        })?;
+
+    let mut js_stdin = js_process
+        .stdin
+        .take()
+        .ok_or_else(|| JSError::IO(format!("Failed to open stdin for `{js_runtime}`")))?;
+
+    let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?;
+    write!(
+        js_stdin,
+        "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {};
+         globalThis.TREE_SITTER_CLI_VERSION_MINOR = {};
+         globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
+        cli_version.major, cli_version.minor, cli_version.patch,
+    )
+    .map_err(|e| {
+        JSError::IO(format!(
+            "Failed to write tree-sitter version to `{js_runtime}`'s stdin -- {e}"
+        ))
+    })?;
+    js_stdin.write(include_bytes!("./dsl.js")).map_err(|e| {
+        JSError::IO(format!(
+            "Failed to write grammar dsl to `{js_runtime}`'s stdin -- {e}"
+        ))
+    })?;
+    drop(js_stdin);
+
+    let output = js_process
+        .wait_with_output()
+        .map_err(|e| JSError::IO(format!("Failed to read output from `{js_runtime}` -- {e}")))?;
+    match output.status.code() {
+        None => panic!("`{js_runtime}` process was killed"),
+        Some(0) => {
+            let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 {
+                runtime: js_runtime.to_string(),
+                error: e.to_string(),
+            })?;
+
+            let mut grammar_json = &stdout[..];
+
+            if let Some(pos) = stdout.rfind('\n') {
+                // If there's a newline, split the last line from the rest of the output
+                let node_output = &stdout[..pos];
+                grammar_json = &stdout[pos + 1..];
+
+                let mut stdout = std::io::stdout().lock();
+                stdout.write_all(node_output.as_bytes())?;
+                stdout.write_all(b"\n")?;
+                stdout.flush()?;
+            }
+
+            Ok(serde_json::to_string_pretty(&serde_json::from_str::<
+                serde_json::Value,
+            >(grammar_json)?)?)
+        }
+        Some(code) => Err(JSError::JSRuntimeExit {
+            runtime: js_runtime.to_string(),
+            code,
+        }),
+    }
+}
+
+pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> {
+    fs::write(path, body)
+        .map_err(|e| GenerateError::IO(format!("Failed to write {:?} -- {e}", path.file_name())))
+}

cli/generate/src/nfa.rs

@@ -1,5 +1,6 @@
 use std::{
-    cmp::{Ordering, max},
+    char,
+    cmp::{max, Ordering},
     fmt,
     iter::ExactSizeIterator,
     mem::{self, swap},
@@ -51,16 +52,12 @@ const END: u32 = char::MAX as u32 + 1;
 
 impl CharacterSet {
     /// Create a character set with a single character.
-    #[must_use]
     pub const fn empty() -> Self {
         Self { ranges: Vec::new() }
     }
 
     /// Create a character set with a given *inclusive* range of characters.
-    #[expect(
-        clippy::single_range_in_vec_init,
-        reason = "Vec is the backing store for CharacterSet"
-    )]
+    #[allow(clippy::single_range_in_vec_init)]
     #[cfg(test)]
     fn from_range(mut first: char, mut last: char) -> Self {
         if first > last {
@@ -72,11 +69,7 @@ impl CharacterSet {
     }
 
     /// Create a character set with a single character.
-    #[must_use]
-    #[expect(
-        clippy::single_range_in_vec_init,
-        reason = "Vec is the backing store for CharacterSet"
-    )]
+    #[allow(clippy::single_range_in_vec_init)]
     pub fn from_char(c: char) -> Self {
         Self {
             ranges: vec![(c as u32)..(c as u32 + 1)],
@@ -85,7 +78,6 @@ impl CharacterSet {
 
     /// Create a character set containing all characters *not* present
     /// in this character set.
-    #[must_use]
     pub fn negate(mut self) -> Self {
         let mut i = 0;
         let mut previous_end = 0;
@@ -106,20 +98,16 @@ impl CharacterSet {
         self
     }
 
-    #[must_use]
     pub fn add_char(mut self, c: char) -> Self {
         self.add_int_range(0, c as u32, c as u32 + 1);
         self
     }
 
-    #[must_use]
     pub fn add_range(mut self, start: char, end: char) -> Self {
         self.add_int_range(0, start as u32, end as u32 + 1);
         self
     }
 
-    #[must_use]
-    #[allow(clippy::should_implement_trait)]
     pub fn add(mut self, other: &Self) -> Self {
         let mut index = 0;
         for range in &other.ranges {
@@ -158,7 +146,6 @@ impl CharacterSet {
         i
     }
 
-    #[must_use]
     pub fn does_intersect(&self, other: &Self) -> bool {
         let mut left_ranges = self.ranges.iter();
         let mut right_ranges = other.ranges.iter();
@@ -179,7 +166,6 @@ impl CharacterSet {
     /// Get the set of characters that are present in both this set
     /// and the other set. Remove those common characters from both
     /// of the operands.
-    #[allow(clippy::return_self_not_must_use)]
     pub fn remove_intersection(&mut self, other: &mut Self) -> Self {
         let mut intersection = Vec::new();
         let mut left_i = 0;
@@ -294,11 +280,6 @@ impl CharacterSet {
 
     /// Produces a `CharacterSet` containing every character in `self` that is not present in
     /// `other`.
-    #[allow(
-        clippy::must_use_candidate,
-        clippy::return_self_not_must_use,
-        dead_code
-    )]
     pub fn difference(mut self, mut other: Self) -> Self {
         self.remove_intersection(&mut other);
         self
@@ -320,8 +301,7 @@ impl CharacterSet {
         self.char_codes().filter_map(char::from_u32)
     }
 
-    #[must_use]
-    pub const fn range_count(&self) -> usize {
+    pub fn range_count(&self) -> usize {
         self.ranges.len()
     }
 
@@ -333,14 +313,12 @@ impl CharacterSet {
         })
     }
 
-    #[must_use]
-    pub const fn is_empty(&self) -> bool {
+    pub fn is_empty(&self) -> bool {
         self.ranges.is_empty()
     }
 
     /// Get a reduced list of character ranges, assuming that a given
     /// set of characters can be safely ignored.
-    #[must_use]
     pub fn simplify_ignoring(&self, ruled_out_characters: &Self) -> Self {
         let mut prev_range: Option<Range<u32>> = None;
         Self {
@@ -355,12 +333,13 @@ impl CharacterSet {
                             return None;
                         }
 
-                        if let Some(prev_range) = &mut prev_range
-                            && ruled_out_characters
+                        if let Some(prev_range) = &mut prev_range {
+                            if ruled_out_characters
                                 .contains_codepoint_range(prev_range.end..range.start)
-                        {
-                            prev_range.end = range.end;
-                            return None;
+                            {
+                                prev_range.end = range.end;
+                                return None;
+                            }
                         }
                     }
 
@@ -372,7 +351,6 @@ impl CharacterSet {
         }
     }
 
-    #[must_use]
     pub fn contains_codepoint_range(&self, seek_range: Range<u32>) -> bool {
         let ix = match self.ranges.binary_search_by(|probe| {
             if probe.end <= seek_range.start {
@@ -390,7 +368,6 @@ impl CharacterSet {
             .is_some_and(|range| range.start <= seek_range.start && range.end >= seek_range.end)
     }
 
-    #[must_use]
     pub fn contains(&self, c: char) -> bool {
         self.contains_codepoint_range(c as u32..c as u32 + 1)
     }
@@ -456,9 +433,7 @@ impl Nfa {
         Self { states: Vec::new() }
     }
 
-    #[must_use]
     pub fn last_state_id(&self) -> u32 {
-        assert_ne!(self.states, [] as [NfaState; 0]);
         self.states.len() as u32 - 1
     }
 }
@@ -475,7 +450,6 @@ impl fmt::Debug for Nfa {
 }
 
 impl<'a> NfaCursor<'a> {
-    #[must_use]
     pub fn new(nfa: &'a Nfa, mut states: Vec<u32>) -> Self {
         let mut result = Self {
             nfa,
@@ -498,25 +472,10 @@ impl<'a> NfaCursor<'a> {
         self.raw_transitions().map(|t| (t.0, t.1))
     }
 
-    #[must_use]
     pub fn transitions(&self) -> Vec<NfaTransition> {
         Self::group_transitions(self.raw_transitions())
     }
 
-    /// Like [`transitions()`](Self::transitions) but also returns whether any raw NFA transition
-    /// is a separator. This is computed in the same pass, avoiding a second
-    /// iteration over `state_ids` for callers that need both.
-    #[must_use]
-    pub fn transitions_and_any_sep(&self) -> (Vec<NfaTransition>, bool) {
-        let mut any_sep = false;
-        let result =
-            Self::group_transitions(self.raw_transitions().map(|(chars, is_sep, prec, state)| {
-                any_sep |= is_sep;
-                (chars, is_sep, prec, state)
-            }));
-        (result, any_sep)
-    }
-
     fn raw_transitions(&self) -> impl Iterator<Item = (&CharacterSet, bool, i32, u32)> {
         self.state_ids.iter().filter_map(move |id| {
             if let NfaState::Advance {
@@ -537,22 +496,13 @@ impl<'a> NfaCursor<'a> {
         iter: impl Iterator<Item = (&'b CharacterSet, bool, i32, u32)>,
     ) -> Vec<NfaTransition> {
         let mut result = Vec::<NfaTransition>::new();
-        // Reuse a single CharacterSet buffer across iterations to avoid one
-        // malloc per raw transition. `assign` refills it in-place; `mem::take`
-        // donates the allocation to a result entry when chars has a remainder.
-        let mut chars = CharacterSet::empty();
-        for (input_chars, is_sep, prec, state) in iter {
-            chars.assign(input_chars);
+        for (chars, is_sep, prec, state) in iter {
+            let mut chars = chars.clone();
             let mut i = 0;
             while i < result.len() && !chars.is_empty() {
                 let intersection = result[i].characters.remove_intersection(&mut chars);
                 if !intersection.is_empty() {
-                    let chars_is_empty = result[i].characters.is_empty();
-                    let mut intersection_states = if chars_is_empty {
-                        mem::take(&mut result[i].states)
-                    } else {
-                        result[i].states.clone()
-                    };
+                    let mut intersection_states = result[i].states.clone();
                     if let Err(j) = intersection_states.binary_search(&state) {
                         intersection_states.insert(j, state);
                     }
@@ -562,23 +512,18 @@ impl<'a> NfaCursor<'a> {
                         precedence: max(result[i].precedence, prec),
                         states: intersection_states,
                     };
-                    if chars_is_empty {
+                    if result[i].characters.is_empty() {
                         result[i] = intersection_transition;
                     } else {
-                        // Push to the tail instead of inserting at i (which
-                        // would be O(n)).  After remove_intersection, the new
-                        // `chars` (C') and `intersection` (I = A∩C) are
-                        // disjoint, so when the loop later reaches I at the
-                        // tail, remove_intersection(I, C'') will be a no-op.
-                        // The final sort makes mid-loop ordering irrelevant.
-                        result.push(intersection_transition);
+                        result.insert(i, intersection_transition);
+                        i += 1;
                     }
                 }
                 i += 1;
             }
             if !chars.is_empty() {
                 result.push(NfaTransition {
-                    characters: mem::take(&mut chars),
+                    characters: chars,
                     precedence: prec,
                     states: vec![state],
                     is_separator: is_sep,
@@ -595,7 +540,7 @@ impl<'a> NfaCursor<'a> {
                 {
                     let characters = mem::take(&mut result[j].characters);
                     result[j].characters = characters.add(&result[i].characters);
-                    result.swap_remove(i);
+                    result.remove(i);
                     i -= 1;
                     break;
                 }
@@ -1115,10 +1060,7 @@ mod tests {
     }
 
     #[test]
-    #[expect(
-        clippy::single_range_in_vec_init,
-        reason = "test data intentionally uses single-element ranges"
-    )]
+    #[allow(clippy::single_range_in_vec_init)]
     fn test_character_set_simplify_ignoring() {
         struct Row {
             chars: Vec<char>,

cli/generate/src/parse_grammar.rs

@@ -0,0 +1,433 @@
+use std::collections::HashSet;
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+use thiserror::Error;
+
+use super::{
+    grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType},
+    rules::{Precedence, Rule},
+};
+use crate::grammars::ReservedWordContext;
+
+#[derive(Deserialize)]
+#[serde(tag = "type")]
+#[allow(non_camel_case_types)]
+#[allow(clippy::upper_case_acronyms)]
+enum RuleJSON {
+    ALIAS {
+        content: Box<RuleJSON>,
+        named: bool,
+        value: String,
+    },
+    BLANK,
+    STRING {
+        value: String,
+    },
+    PATTERN {
+        value: String,
+        flags: Option<String>,
+    },
+    SYMBOL {
+        name: String,
+    },
+    CHOICE {
+        members: Vec<RuleJSON>,
+    },
+    FIELD {
+        name: String,
+        content: Box<RuleJSON>,
+    },
+    SEQ {
+        members: Vec<RuleJSON>,
+    },
+    REPEAT {
+        content: Box<RuleJSON>,
+    },
+    REPEAT1 {
+        content: Box<RuleJSON>,
+    },
+    PREC_DYNAMIC {
+        value: i32,
+        content: Box<RuleJSON>,
+    },
+    PREC_LEFT {
+        value: PrecedenceValueJSON,
+        content: Box<RuleJSON>,
+    },
+    PREC_RIGHT {
+        value: PrecedenceValueJSON,
+        content: Box<RuleJSON>,
+    },
+    PREC {
+        value: PrecedenceValueJSON,
+        content: Box<RuleJSON>,
+    },
+    TOKEN {
+        content: Box<RuleJSON>,
+    },
+    IMMEDIATE_TOKEN {
+        content: Box<RuleJSON>,
+    },
+    RESERVED {
+        context_name: String,
+        content: Box<RuleJSON>,
+    },
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum PrecedenceValueJSON {
+    Integer(i32),
+    Name(String),
+}
+
+#[derive(Deserialize)]
+pub struct GrammarJSON {
+    pub name: String,
+    rules: Map<String, Value>,
+    #[serde(default)]
+    precedences: Vec<Vec<RuleJSON>>,
+    #[serde(default)]
+    conflicts: Vec<Vec<String>>,
+    #[serde(default)]
+    externals: Vec<RuleJSON>,
+    #[serde(default)]
+    extras: Vec<RuleJSON>,
+    #[serde(default)]
+    inline: Vec<String>,
+    #[serde(default)]
+    supertypes: Vec<String>,
+    #[serde(default)]
+    word: Option<String>,
+    #[serde(default)]
+    reserved: Map<String, Value>,
+}
+
+pub type ParseGrammarResult<T> = Result<T, ParseGrammarError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum ParseGrammarError {
+    #[error("{0}")]
+    Serialization(String),
+    #[error("Rules in the `extras` array must not contain empty strings")]
+    InvalidExtra,
+    #[error("Invalid rule in precedences array. Only strings and symbols are allowed")]
+    Unexpected,
+    #[error("Reserved word sets must be arrays")]
+    InvalidReservedWordSet,
+    #[error("Grammar Error: Unexpected rule `{0}` in `token()` call")]
+    UnexpectedRule(String),
+}
+
+impl From<serde_json::Error> for ParseGrammarError {
+    fn from(value: serde_json::Error) -> Self {
+        Self::Serialization(value.to_string())
+    }
+}
+
+/// Check if a rule is referenced by another rule.
+///
+/// This function is used to determine if a variable is used in a given rule,
+/// and `is_other` indicates if the rule is an external, and if it is,
+/// to not assume that a named symbol that is equal to itself means it's being referenced.
+///
+/// For example, if we have an external rule **and** a normal rule both called `foo`,
+/// `foo` should not be thought of as directly used unless it's used within another rule.
+fn rule_is_referenced(rule: &Rule, target: &str, is_external: bool) -> bool {
+    match rule {
+        Rule::NamedSymbol(name) => name == target && !is_external,
+        Rule::Choice(rules) | Rule::Seq(rules) => {
+            rules.iter().any(|r| rule_is_referenced(r, target, false))
+        }
+        Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
+            rule_is_referenced(rule, target, is_external)
+        }
+        Rule::Repeat(inner) => rule_is_referenced(inner, target, false),
+        Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false,
+    }
+}
+
+fn variable_is_used(
+    grammar_rules: &[(String, Rule)],
+    extras: &[Rule],
+    externals: &[Rule],
+    target_name: &str,
+    in_progress: &mut HashSet<String>,
+) -> bool {
+    let root = &grammar_rules.first().unwrap().0;
+    if target_name == root {
+        return true;
+    }
+
+    if extras
+        .iter()
+        .any(|rule| rule_is_referenced(rule, target_name, false))
+    {
+        return true;
+    }
+
+    if externals
+        .iter()
+        .any(|rule| rule_is_referenced(rule, target_name, true))
+    {
+        return true;
+    }
+
+    in_progress.insert(target_name.to_string());
+    let result = grammar_rules
+        .iter()
+        .filter(|(key, _)| *key != target_name)
+        .any(|(name, rule)| {
+            if !rule_is_referenced(rule, target_name, false) || in_progress.contains(name) {
+                return false;
+            }
+            variable_is_used(grammar_rules, extras, externals, name, in_progress)
+        });
+    in_progress.remove(target_name);
+
+    result
+}
+
+pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
+    let mut grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
+
+    let mut extra_symbols =
+        grammar_json
+            .extras
+            .into_iter()
+            .try_fold(Vec::<Rule>::new(), |mut acc, item| {
+                let rule = parse_rule(item, false)?;
+                if let Rule::String(ref value) = rule {
+                    if value.is_empty() {
+                        Err(ParseGrammarError::InvalidExtra)?;
+                    }
+                }
+                acc.push(rule);
+                ParseGrammarResult::Ok(acc)
+            })?;
+
+    let mut external_tokens = grammar_json
+        .externals
+        .into_iter()
+        .map(|e| parse_rule(e, false))
+        .collect::<ParseGrammarResult<Vec<_>>>()?;
+
+    let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len());
+    for list in grammar_json.precedences {
+        let mut ordering = Vec::with_capacity(list.len());
+        for entry in list {
+            ordering.push(match entry {
+                RuleJSON::STRING { value } => PrecedenceEntry::Name(value),
+                RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name),
+                _ => Err(ParseGrammarError::Unexpected)?,
+            });
+        }
+        precedence_orderings.push(ordering);
+    }
+
+    let mut variables = Vec::with_capacity(grammar_json.rules.len());
+
+    let rules = grammar_json
+        .rules
+        .into_iter()
+        .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?, false)?)))
+        .collect::<ParseGrammarResult<Vec<_>>>()?;
+
+    let mut in_progress = HashSet::new();
+
+    for (name, rule) in &rules {
+        if !variable_is_used(
+            &rules,
+            &extra_symbols,
+            &external_tokens,
+            name,
+            &mut in_progress,
+        ) && grammar_json.word.as_ref().is_none_or(|w| w != name)
+        {
+            grammar_json.conflicts.retain(|r| !r.contains(name));
+            grammar_json.supertypes.retain(|r| r != name);
+            grammar_json.inline.retain(|r| r != name);
+            extra_symbols.retain(|r| !rule_is_referenced(r, name, true));
+            external_tokens.retain(|r| !rule_is_referenced(r, name, true));
+            precedence_orderings.retain(|r| {
+                !r.iter().any(|e| {
+                    let PrecedenceEntry::Symbol(s) = e else {
+                        return false;
+                    };
+                    s == name
+                })
+            });
+            continue;
+        }
+        variables.push(Variable {
+            name: name.clone(),
+            kind: VariableType::Named,
+            rule: rule.clone(),
+        });
+    }
+
+    let reserved_words = grammar_json
+        .reserved
+        .into_iter()
+        .map(|(name, rule_values)| {
+            let mut reserved_words = Vec::new();
+
+            let Value::Array(rule_values) = rule_values else {
+                Err(ParseGrammarError::InvalidReservedWordSet)?
+            };
+
+            for value in rule_values {
+                reserved_words.push(parse_rule(serde_json::from_value(value)?, false)?);
+            }
+            Ok(ReservedWordContext {
+                name,
+                reserved_words,
+            })
+        })
+        .collect::<ParseGrammarResult<Vec<_>>>()?;
+
+    Ok(InputGrammar {
+        name: grammar_json.name,
+        word_token: grammar_json.word,
+        expected_conflicts: grammar_json.conflicts,
+        supertype_symbols: grammar_json.supertypes,
+        variables_to_inline: grammar_json.inline,
+        precedence_orderings,
+        variables,
+        extra_symbols,
+        external_tokens,
+        reserved_words,
+    })
+}
+
+fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult<Rule> {
+    match json {
+        RuleJSON::ALIAS {
+            content,
+            value,
+            named,
+        } => parse_rule(*content, is_token).map(|r| Rule::alias(r, value, named)),
+        RuleJSON::BLANK => Ok(Rule::Blank),
+        RuleJSON::STRING { value } => Ok(Rule::String(value)),
+        RuleJSON::PATTERN { value, flags } => Ok(Rule::Pattern(
+            value,
+            flags.map_or(String::new(), |f| {
+                f.matches(|c| {
+                    if c == 'i' {
+                        true
+                    } else {
+                        // silently ignore unicode flags
+                        if c != 'u' && c != 'v' {
+                            eprintln!("Warning: unsupported flag {c}");
+                        }
+                        false
+                    }
+                })
+                .collect()
+            }),
+        )),
+        RuleJSON::SYMBOL { name } => {
+            if is_token {
+                Err(ParseGrammarError::UnexpectedRule(name))?
+            } else {
+                Ok(Rule::NamedSymbol(name))
+            }
+        }
+        RuleJSON::CHOICE { members } => members
+            .into_iter()
+            .map(|m| parse_rule(m, is_token))
+            .collect::<ParseGrammarResult<Vec<_>>>()
+            .map(Rule::choice),
+        RuleJSON::FIELD { content, name } => {
+            parse_rule(*content, is_token).map(|r| Rule::field(name, r))
+        }
+        RuleJSON::SEQ { members } => members
+            .into_iter()
+            .map(|m| parse_rule(m, is_token))
+            .collect::<ParseGrammarResult<Vec<_>>>()
+            .map(Rule::seq),
+        RuleJSON::REPEAT1 { content } => parse_rule(*content, is_token).map(Rule::repeat),
+        RuleJSON::REPEAT { content } => {
+            parse_rule(*content, is_token).map(|m| Rule::choice(vec![Rule::repeat(m), Rule::Blank]))
+        }
+        RuleJSON::PREC { value, content } => {
+            parse_rule(*content, is_token).map(|r| Rule::prec(value.into(), r))
+        }
+        RuleJSON::PREC_LEFT { value, content } => {
+            parse_rule(*content, is_token).map(|r| Rule::prec_left(value.into(), r))
+        }
+        RuleJSON::PREC_RIGHT { value, content } => {
+            parse_rule(*content, is_token).map(|r| Rule::prec_right(value.into(), r))
+        }
+        RuleJSON::PREC_DYNAMIC { value, content } => {
+            parse_rule(*content, is_token).map(|r| Rule::prec_dynamic(value, r))
+        }
+        RuleJSON::RESERVED {
+            content,
+            context_name,
+        } => parse_rule(*content, is_token).map(|r| Rule::Reserved {
+            rule: Box::new(r),
+            context_name,
+        }),
+        RuleJSON::TOKEN { content } => parse_rule(*content, true).map(Rule::token),
+        RuleJSON::IMMEDIATE_TOKEN { content } => {
+            parse_rule(*content, is_token).map(Rule::immediate_token)
+        }
+    }
+}
+
+impl From<PrecedenceValueJSON> for Precedence {
+    fn from(val: PrecedenceValueJSON) -> Self {
+        match val {
+            PrecedenceValueJSON::Integer(i) => Self::Integer(i),
+            PrecedenceValueJSON::Name(i) => Self::Name(i),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_grammar() {
+        let grammar = parse_grammar(
+            r#"{
+            "name": "my_lang",
+            "rules": {
+                "file": {
+                    "type": "REPEAT1",
+                    "content": {
+                        "type": "SYMBOL",
+                        "name": "statement"
+                    }
+                },
+                "statement": {
+                    "type": "STRING",
+                    "value": "foo"
+                }
+            }
+        }"#,
+        )
+        .unwrap();
+
+        assert_eq!(grammar.name, "my_lang");
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable {
+                    name: "file".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::repeat(Rule::NamedSymbol("statement".to_string()))
+                },
+                Variable {
+                    name: "statement".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::String("foo".to_string())
+                },
+            ]
+        );
+    }
+}

cli/generate/src/prepare_grammar/expand_repeats.rs

@@ -0,0 +1,289 @@
+use std::{collections::HashMap, mem};
+
+use super::ExtractedSyntaxGrammar;
+use crate::{
+    grammars::{Variable, VariableType},
+    rules::{Rule, Symbol},
+};
+
+struct Expander {
+    variable_name: String,
+    repeat_count_in_variable: usize,
+    preceding_symbol_count: usize,
+    auxiliary_variables: Vec<Variable>,
+    existing_repeats: HashMap<Rule, Symbol>,
+}
+
+impl Expander {
+    fn expand_variable(&mut self, index: usize, variable: &mut Variable) -> bool {
+        self.variable_name.clear();
+        self.variable_name.push_str(&variable.name);
+        self.repeat_count_in_variable = 0;
+        let mut rule = Rule::Blank;
+        mem::swap(&mut rule, &mut variable.rule);
+
+        // In the special case of a hidden variable with a repetition at its top level,
+        // convert that rule itself into a binary tree structure instead of introducing
+        // another auxiliary rule.
+        if let (VariableType::Hidden, Rule::Repeat(repeated_content)) = (variable.kind, &rule) {
+            let inner_rule = self.expand_rule(repeated_content);
+            variable.rule = self.wrap_rule_in_binary_tree(Symbol::non_terminal(index), inner_rule);
+            variable.kind = VariableType::Auxiliary;
+            return true;
+        }
+
+        variable.rule = self.expand_rule(&rule);
+        false
+    }
+
+    fn expand_rule(&mut self, rule: &Rule) -> Rule {
+        match rule {
+            // For choices, sequences, and metadata, descend into the child rules,
+            // replacing any nested repetitions.
+            Rule::Choice(elements) => Rule::Choice(
+                elements
+                    .iter()
+                    .map(|element| self.expand_rule(element))
+                    .collect(),
+            ),
+
+            Rule::Seq(elements) => Rule::Seq(
+                elements
+                    .iter()
+                    .map(|element| self.expand_rule(element))
+                    .collect(),
+            ),
+
+            Rule::Metadata { rule, params } => Rule::Metadata {
+                rule: Box::new(self.expand_rule(rule)),
+                params: params.clone(),
+            },
+
+            // For repetitions, introduce an auxiliary rule that contains the
+            // repeated content, but can also contain a recursive binary tree structure.
+            Rule::Repeat(content) => {
+                let inner_rule = self.expand_rule(content);
+
+                if let Some(existing_symbol) = self.existing_repeats.get(&inner_rule) {
+                    return Rule::Symbol(*existing_symbol);
+                }
+
+                self.repeat_count_in_variable += 1;
+                let rule_name = format!(
+                    "{}_repeat{}",
+                    self.variable_name, self.repeat_count_in_variable
+                );
+                let repeat_symbol = Symbol::non_terminal(
+                    self.preceding_symbol_count + self.auxiliary_variables.len(),
+                );
+                self.existing_repeats
+                    .insert(inner_rule.clone(), repeat_symbol);
+                self.auxiliary_variables.push(Variable {
+                    name: rule_name,
+                    kind: VariableType::Auxiliary,
+                    rule: self.wrap_rule_in_binary_tree(repeat_symbol, inner_rule),
+                });
+
+                Rule::Symbol(repeat_symbol)
+            }
+
+            // For primitive rules, don't change anything.
+            _ => rule.clone(),
+        }
+    }
+
+    fn wrap_rule_in_binary_tree(&self, symbol: Symbol, rule: Rule) -> Rule {
+        Rule::choice(vec![
+            Rule::Seq(vec![Rule::Symbol(symbol), Rule::Symbol(symbol)]),
+            rule,
+        ])
+    }
+}
+
+pub(super) fn expand_repeats(mut grammar: ExtractedSyntaxGrammar) -> ExtractedSyntaxGrammar {
+    let mut expander = Expander {
+        variable_name: String::new(),
+        repeat_count_in_variable: 0,
+        preceding_symbol_count: grammar.variables.len(),
+        auxiliary_variables: Vec::new(),
+        existing_repeats: HashMap::new(),
+    };
+
+    for (i, variable) in grammar.variables.iter_mut().enumerate() {
+        let expanded_top_level_repetition = expander.expand_variable(i, variable);
+
+        // If a hidden variable had a top-level repetition and it was converted to
+        // a recursive rule, then it can't be inlined.
+        if expanded_top_level_repetition {
+            grammar
+                .variables_to_inline
+                .retain(|symbol| *symbol != Symbol::non_terminal(i));
+        }
+    }
+
+    grammar.variables.extend(expander.auxiliary_variables);
+    grammar
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_repeat_expansion() {
+        // Repeats nested inside of sequences and choices are expanded.
+        let grammar = expand_repeats(build_grammar(vec![Variable::named(
+            "rule0",
+            Rule::seq(vec![
+                Rule::terminal(10),
+                Rule::choice(vec![
+                    Rule::repeat(Rule::terminal(11)),
+                    Rule::repeat(Rule::terminal(12)),
+                ]),
+                Rule::terminal(13),
+            ]),
+        )]));
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "rule0",
+                    Rule::seq(vec![
+                        Rule::terminal(10),
+                        Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),]),
+                        Rule::terminal(13),
+                    ])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
+                        Rule::terminal(11),
+                    ])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat2",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
+                        Rule::terminal(12),
+                    ])
+                ),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_repeat_deduplication() {
+        // Terminal 4 appears inside of a repeat in three different places.
+        let grammar = expand_repeats(build_grammar(vec![
+            Variable::named(
+                "rule0",
+                Rule::choice(vec![
+                    Rule::seq(vec![Rule::terminal(1), Rule::repeat(Rule::terminal(4))]),
+                    Rule::seq(vec![Rule::terminal(2), Rule::repeat(Rule::terminal(4))]),
+                ]),
+            ),
+            Variable::named(
+                "rule1",
+                Rule::seq(vec![Rule::terminal(3), Rule::repeat(Rule::terminal(4))]),
+            ),
+        ]));
+
+        // Only one auxiliary rule is created for repeating terminal 4.
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "rule0",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::terminal(1), Rule::non_terminal(2)]),
+                        Rule::seq(vec![Rule::terminal(2), Rule::non_terminal(2)]),
+                    ])
+                ),
+                Variable::named(
+                    "rule1",
+                    Rule::seq(vec![Rule::terminal(3), Rule::non_terminal(2),])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
+                        Rule::terminal(4),
+                    ])
+                )
+            ]
+        );
+    }
+
+    #[test]
+    fn test_expansion_of_nested_repeats() {
+        let grammar = expand_repeats(build_grammar(vec![Variable::named(
+            "rule0",
+            Rule::seq(vec![
+                Rule::terminal(10),
+                Rule::repeat(Rule::seq(vec![
+                    Rule::terminal(11),
+                    Rule::repeat(Rule::terminal(12)),
+                ])),
+            ]),
+        )]));
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "rule0",
+                    Rule::seq(vec![Rule::terminal(10), Rule::non_terminal(2),])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
+                        Rule::terminal(12),
+                    ])
+                ),
+                Variable::auxiliary(
+                    "rule0_repeat2",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
+                        Rule::seq(vec![Rule::terminal(11), Rule::non_terminal(1),]),
+                    ])
+                ),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_expansion_of_repeats_at_top_of_hidden_rules() {
+        let grammar = expand_repeats(build_grammar(vec![
+            Variable::named("rule0", Rule::non_terminal(1)),
+            Variable::hidden(
+                "_rule1",
+                Rule::repeat(Rule::choice(vec![Rule::terminal(11), Rule::terminal(12)])),
+            ),
+        ]));
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named("rule0", Rule::non_terminal(1),),
+                Variable::auxiliary(
+                    "_rule1",
+                    Rule::choice(vec![
+                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1)]),
+                        Rule::terminal(11),
+                        Rule::terminal(12),
+                    ]),
+                ),
+            ]
+        );
+    }
+
+    fn build_grammar(variables: Vec<Variable>) -> ExtractedSyntaxGrammar {
+        ExtractedSyntaxGrammar {
+            variables,
+            ..Default::default()
+        }
+    }
+}

cli/generate/src/prepare_grammar/expand_tokens.rs

@@ -0,0 +1,826 @@
+use anyhow::Result;
+use regex_syntax::{
+    hir::{Class, Hir, HirKind},
+    ParserBuilder,
+};
+use serde::Serialize;
+use thiserror::Error;
+
+use super::ExtractedLexicalGrammar;
+use crate::{
+    grammars::{LexicalGrammar, LexicalVariable},
+    nfa::{CharacterSet, Nfa, NfaState},
+    rules::{Precedence, Rule},
+};
+
+struct NfaBuilder {
+    nfa: Nfa,
+    is_sep: bool,
+    precedence_stack: Vec<i32>,
+}
+
+pub type ExpandTokensResult<T> = Result<T, ExpandTokensError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum ExpandTokensError {
+    #[error(
+        "The rule `{0}` matches the empty string.
+Tree-sitter does not support syntactic rules that match the empty string
+unless they are used only as the grammar's start rule.
+        "
+    )]
+    EmptyString(String),
+    #[error(transparent)]
+    Processing(ExpandTokensProcessingError),
+    #[error(transparent)]
+    ExpandRule(ExpandRuleError),
+}
+
+#[derive(Debug, Error, Serialize)]
+pub struct ExpandTokensProcessingError {
+    rule: String,
+    error: ExpandRuleError,
+}
+
+impl std::fmt::Display for ExpandTokensProcessingError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(
+            f,
+            "Error processing rule {}: Grammar error: Unexpected rule {:?}",
+            self.rule, self.error
+        )?;
+        Ok(())
+    }
+}
+
+fn get_implicit_precedence(rule: &Rule) -> i32 {
+    match rule {
+        Rule::String(_) => 2,
+        Rule::Metadata { rule, params } => {
+            if params.is_main_token {
+                get_implicit_precedence(rule) + 1
+            } else {
+                get_implicit_precedence(rule)
+            }
+        }
+        _ => 0,
+    }
+}
+
+const fn get_completion_precedence(rule: &Rule) -> i32 {
+    if let Rule::Metadata { params, .. } = rule {
+        if let Precedence::Integer(p) = params.precedence {
+            return p;
+        }
+    }
+    0
+}
+
+pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> ExpandTokensResult<LexicalGrammar> {
+    let mut builder = NfaBuilder {
+        nfa: Nfa::new(),
+        is_sep: true,
+        precedence_stack: vec![0],
+    };
+
+    let separator_rule = if grammar.separators.is_empty() {
+        Rule::Blank
+    } else {
+        grammar.separators.push(Rule::Blank);
+        Rule::repeat(Rule::choice(grammar.separators))
+    };
+
+    let mut variables = Vec::new();
+    for (i, variable) in grammar.variables.into_iter().enumerate() {
+        if variable.rule.is_empty() {
+            Err(ExpandTokensError::EmptyString(variable.name.clone()))?;
+        }
+
+        let is_immediate_token = match &variable.rule {
+            Rule::Metadata { params, .. } => params.is_main_token,
+            _ => false,
+        };
+
+        builder.is_sep = false;
+        builder.nfa.states.push(NfaState::Accept {
+            variable_index: i,
+            precedence: get_completion_precedence(&variable.rule),
+        });
+        let last_state_id = builder.nfa.last_state_id();
+        builder
+            .expand_rule(&variable.rule, last_state_id)
+            .map_err(|e| {
+                ExpandTokensError::Processing(ExpandTokensProcessingError {
+                    rule: variable.name.clone(),
+                    error: e,
+                })
+            })?;
+
+        if !is_immediate_token {
+            builder.is_sep = true;
+            let last_state_id = builder.nfa.last_state_id();
+            builder
+                .expand_rule(&separator_rule, last_state_id)
+                .map_err(ExpandTokensError::ExpandRule)?;
+        }
+
+        variables.push(LexicalVariable {
+            name: variable.name,
+            kind: variable.kind,
+            implicit_precedence: get_implicit_precedence(&variable.rule),
+            start_state: builder.nfa.last_state_id(),
+        });
+    }
+
+    Ok(LexicalGrammar {
+        nfa: builder.nfa,
+        variables,
+    })
+}
+
+pub type ExpandRuleResult<T> = Result<T, ExpandRuleError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum ExpandRuleError {
+    #[error("Grammar error: Unexpected rule {0:?}")]
+    UnexpectedRule(Rule),
+    #[error("{0}")]
+    Parse(String),
+    #[error(transparent)]
+    ExpandRegex(ExpandRegexError),
+}
+
+pub type ExpandRegexResult<T> = Result<T, ExpandRegexError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum ExpandRegexError {
+    #[error("{0}")]
+    Utf8(String),
+    #[error("Regex error: Assertions are not supported")]
+    Assertion,
+}
+
+impl NfaBuilder {
+    fn expand_rule(&mut self, rule: &Rule, mut next_state_id: u32) -> ExpandRuleResult<bool> {
+        match rule {
+            Rule::Pattern(s, f) => {
+                // With unicode enabled, `\w`, `\s` and `\d` expand to character sets that are much
+                // larger than intended, so we replace them with the actual
+                // character sets they should represent. If the full unicode range
+                // of `\w`, `\s` or `\d` are needed then `\p{L}`, `\p{Z}` and `\p{N}` should be
+                // used.
+                let s = s
+                    .replace(r"\w", r"[0-9A-Za-z_]")
+                    .replace(r"\s", r"[\t-\r ]")
+                    .replace(r"\d", r"[0-9]")
+                    .replace(r"\W", r"[^0-9A-Za-z_]")
+                    .replace(r"\S", r"[^\t-\r ]")
+                    .replace(r"\D", r"[^0-9]");
+                let mut parser = ParserBuilder::new()
+                    .case_insensitive(f.contains('i'))
+                    .unicode(true)
+                    .utf8(false)
+                    .build();
+                let hir = parser
+                    .parse(&s)
+                    .map_err(|e| ExpandRuleError::Parse(e.to_string()))?;
+                self.expand_regex(&hir, next_state_id)
+                    .map_err(ExpandRuleError::ExpandRegex)
+            }
+            Rule::String(s) => {
+                for c in s.chars().rev() {
+                    self.push_advance(CharacterSet::empty().add_char(c), next_state_id);
+                    next_state_id = self.nfa.last_state_id();
+                }
+                Ok(!s.is_empty())
+            }
+            Rule::Choice(elements) => {
+                let mut alternative_state_ids = Vec::new();
+                for element in elements {
+                    if self.expand_rule(element, next_state_id)? {
+                        alternative_state_ids.push(self.nfa.last_state_id());
+                    } else {
+                        alternative_state_ids.push(next_state_id);
+                    }
+                }
+                alternative_state_ids.sort_unstable();
+                alternative_state_ids.dedup();
+                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
+                for alternative_state_id in alternative_state_ids {
+                    self.push_split(alternative_state_id);
+                }
+                Ok(true)
+            }
+            Rule::Seq(elements) => {
+                let mut result = false;
+                for element in elements.iter().rev() {
+                    if self.expand_rule(element, next_state_id)? {
+                        result = true;
+                    }
+                    next_state_id = self.nfa.last_state_id();
+                }
+                Ok(result)
+            }
+            Rule::Repeat(rule) => {
+                self.nfa.states.push(NfaState::Accept {
+                    variable_index: 0,
+                    precedence: 0,
+                }); // Placeholder for split
+                let split_state_id = self.nfa.last_state_id();
+                if self.expand_rule(rule, split_state_id)? {
+                    self.nfa.states[split_state_id as usize] =
+                        NfaState::Split(self.nfa.last_state_id(), next_state_id);
+                    Ok(true)
+                } else {
+                    Ok(false)
+                }
+            }
+            Rule::Metadata { rule, params } => {
+                let has_precedence = if let Precedence::Integer(precedence) = &params.precedence {
+                    self.precedence_stack.push(*precedence);
+                    true
+                } else {
+                    false
+                };
+                let result = self.expand_rule(rule, next_state_id);
+                if has_precedence {
+                    self.precedence_stack.pop();
+                }
+                result
+            }
+            Rule::Blank => Ok(false),
+            _ => Err(ExpandRuleError::UnexpectedRule(rule.clone()))?,
+        }
+    }
+
+    fn expand_regex(&mut self, hir: &Hir, mut next_state_id: u32) -> ExpandRegexResult<bool> {
+        match hir.kind() {
+            HirKind::Empty => Ok(false),
+            HirKind::Literal(literal) => {
+                for character in std::str::from_utf8(&literal.0)
+                    .map_err(|e| ExpandRegexError::Utf8(e.to_string()))?
+                    .chars()
+                    .rev()
+                {
+                    let char_set = CharacterSet::from_char(character);
+                    self.push_advance(char_set, next_state_id);
+                    next_state_id = self.nfa.last_state_id();
+                }
+
+                Ok(true)
+            }
+            HirKind::Class(class) => match class {
+                Class::Unicode(class) => {
+                    let mut chars = CharacterSet::default();
+                    for c in class.ranges() {
+                        chars = chars.add_range(c.start(), c.end());
+                    }
+
+                    // For some reason, the long s `ſ` is included if the letter `s` is in a
+                    // pattern, so we remove it.
+                    if chars.range_count() == 3
+                        && chars
+                            .ranges()
+                            // exact check to ensure that `ſ` wasn't intentionally added.
+                            .all(|r| ['s'..='s', 'S'..='S', 'ſ'..='ſ'].contains(&r))
+                    {
+                        chars = chars.difference(CharacterSet::from_char('ſ'));
+                    }
+                    self.push_advance(chars, next_state_id);
+                    Ok(true)
+                }
+                Class::Bytes(bytes_class) => {
+                    let mut chars = CharacterSet::default();
+                    for c in bytes_class.ranges() {
+                        chars = chars.add_range(c.start().into(), c.end().into());
+                    }
+                    self.push_advance(chars, next_state_id);
+                    Ok(true)
+                }
+            },
+            HirKind::Look(_) => Err(ExpandRegexError::Assertion)?,
+            HirKind::Repetition(repetition) => match (repetition.min, repetition.max) {
+                (0, Some(1)) => self.expand_zero_or_one(&repetition.sub, next_state_id),
+                (1, None) => self.expand_one_or_more(&repetition.sub, next_state_id),
+                (0, None) => self.expand_zero_or_more(&repetition.sub, next_state_id),
+                (min, Some(max)) if min == max => {
+                    self.expand_count(&repetition.sub, min, next_state_id)
+                }
+                (min, None) => {
+                    if self.expand_zero_or_more(&repetition.sub, next_state_id)? {
+                        self.expand_count(&repetition.sub, min, next_state_id)
+                    } else {
+                        Ok(false)
+                    }
+                }
+                (min, Some(max)) => {
+                    let mut result = self.expand_count(&repetition.sub, min, next_state_id)?;
+                    for _ in min..max {
+                        if result {
+                            next_state_id = self.nfa.last_state_id();
+                        }
+                        if self.expand_zero_or_one(&repetition.sub, next_state_id)? {
+                            result = true;
+                        }
+                    }
+                    Ok(result)
+                }
+            },
+            HirKind::Capture(capture) => self.expand_regex(&capture.sub, next_state_id),
+            HirKind::Concat(concat) => {
+                let mut result = false;
+                for hir in concat.iter().rev() {
+                    if self.expand_regex(hir, next_state_id)? {
+                        result = true;
+                        next_state_id = self.nfa.last_state_id();
+                    }
+                }
+                Ok(result)
+            }
+            HirKind::Alternation(alternations) => {
+                let mut alternative_state_ids = Vec::new();
+                for hir in alternations {
+                    if self.expand_regex(hir, next_state_id)? {
+                        alternative_state_ids.push(self.nfa.last_state_id());
+                    } else {
+                        alternative_state_ids.push(next_state_id);
+                    }
+                }
+                alternative_state_ids.sort_unstable();
+                alternative_state_ids.dedup();
+                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
+                for alternative_state_id in alternative_state_ids {
+                    self.push_split(alternative_state_id);
+                }
+                Ok(true)
+            }
+        }
+    }
+
+    fn expand_one_or_more(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult<bool> {
+        self.nfa.states.push(NfaState::Accept {
+            variable_index: 0,
+            precedence: 0,
+        }); // Placeholder for split
+        let split_state_id = self.nfa.last_state_id();
+        if self.expand_regex(hir, split_state_id)? {
+            self.nfa.states[split_state_id as usize] =
+                NfaState::Split(self.nfa.last_state_id(), next_state_id);
+            Ok(true)
+        } else {
+            self.nfa.states.pop();
+            Ok(false)
+        }
+    }
+
+    fn expand_zero_or_one(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult<bool> {
+        if self.expand_regex(hir, next_state_id)? {
+            self.push_split(next_state_id);
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    fn expand_zero_or_more(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult<bool> {
+        if self.expand_one_or_more(hir, next_state_id)? {
+            self.push_split(next_state_id);
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    fn expand_count(
+        &mut self,
+        hir: &Hir,
+        count: u32,
+        mut next_state_id: u32,
+    ) -> ExpandRegexResult<bool> {
+        let mut result = false;
+        for _ in 0..count {
+            if self.expand_regex(hir, next_state_id)? {
+                result = true;
+                next_state_id = self.nfa.last_state_id();
+            }
+        }
+        Ok(result)
+    }
+
+    fn push_advance(&mut self, chars: CharacterSet, state_id: u32) {
+        let precedence = *self.precedence_stack.last().unwrap();
+        self.nfa.states.push(NfaState::Advance {
+            chars,
+            state_id,
+            precedence,
+            is_sep: self.is_sep,
+        });
+    }
+
+    fn push_split(&mut self, state_id: u32) {
+        let last_state_id = self.nfa.last_state_id();
+        self.nfa
+            .states
+            .push(NfaState::Split(state_id, last_state_id));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::Variable,
+        nfa::{NfaCursor, NfaTransition},
+    };
+
+    fn simulate_nfa<'a>(grammar: &'a LexicalGrammar, s: &'a str) -> Option<(usize, &'a str)> {
+        let start_states = grammar.variables.iter().map(|v| v.start_state).collect();
+        let mut cursor = NfaCursor::new(&grammar.nfa, start_states);
+
+        let mut result = None;
+        let mut result_precedence = i32::MIN;
+        let mut start_char = 0;
+        let mut end_char = 0;
+        for c in s.chars() {
+            for (id, precedence) in cursor.completions() {
+                if result.is_none() || result_precedence <= precedence {
+                    result = Some((id, &s[start_char..end_char]));
+                    result_precedence = precedence;
+                }
+            }
+            if let Some(NfaTransition {
+                states,
+                is_separator,
+                ..
+            }) = cursor
+                .transitions()
+                .into_iter()
+                .find(|t| t.characters.contains(c) && t.precedence >= result_precedence)
+            {
+                cursor.reset(states);
+                end_char += c.len_utf8();
+                if is_separator {
+                    start_char = end_char;
+                }
+            } else {
+                break;
+            }
+        }
+
+        for (id, precedence) in cursor.completions() {
+            if result.is_none() || result_precedence <= precedence {
+                result = Some((id, &s[start_char..end_char]));
+                result_precedence = precedence;
+            }
+        }
+
+        result
+    }
+
+    #[test]
+    fn test_rule_expansion() {
+        struct Row {
+            rules: Vec<Rule>,
+            separators: Vec<Rule>,
+            examples: Vec<(&'static str, Option<(usize, &'static str)>)>,
+        }
+
+        let table = [
+            // regex with sequences and alternatives
+            Row {
+                rules: vec![Rule::pattern("(a|b|c)d(e|f|g)h?", "")],
+                separators: vec![],
+                examples: vec![
+                    ("ade1", Some((0, "ade"))),
+                    ("bdf1", Some((0, "bdf"))),
+                    ("bdfh1", Some((0, "bdfh"))),
+                    ("ad1", None),
+                ],
+            },
+            // regex with repeats
+            Row {
+                rules: vec![Rule::pattern("a*", "")],
+                separators: vec![],
+                examples: vec![("aaa1", Some((0, "aaa"))), ("b", Some((0, "")))],
+            },
+            // regex with repeats in sequences
+            Row {
+                rules: vec![Rule::pattern("a((bc)+|(de)*)f", "")],
+                separators: vec![],
+                examples: vec![
+                    ("af1", Some((0, "af"))),
+                    ("adedef1", Some((0, "adedef"))),
+                    ("abcbcbcf1", Some((0, "abcbcbcf"))),
+                    ("a", None),
+                ],
+            },
+            // regex with character ranges
+            Row {
+                rules: vec![Rule::pattern("[a-fA-F0-9]+", "")],
+                separators: vec![],
+                examples: vec![("A1ff0.", Some((0, "A1ff0")))],
+            },
+            // regex with perl character classes
+            Row {
+                rules: vec![Rule::pattern("\\w\\d\\s", "")],
+                separators: vec![],
+                examples: vec![("_0  ", Some((0, "_0 ")))],
+            },
+            // string
+            Row {
+                rules: vec![Rule::string("abc")],
+                separators: vec![],
+                examples: vec![("abcd", Some((0, "abc"))), ("ab", None)],
+            },
+            // complex rule containing strings and regexes
+            Row {
+                rules: vec![Rule::repeat(Rule::seq(vec![
+                    Rule::string("{"),
+                    Rule::pattern("[a-f]+", ""),
+                    Rule::string("}"),
+                ]))],
+                separators: vec![],
+                examples: vec![
+                    ("{a}{", Some((0, "{a}"))),
+                    ("{a}{d", Some((0, "{a}"))),
+                    ("ab", None),
+                ],
+            },
+            // longest match rule
+            Row {
+                rules: vec![
+                    Rule::pattern("a|bc", ""),
+                    Rule::pattern("aa", ""),
+                    Rule::pattern("bcd", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("a.", Some((0, "a"))),
+                    ("bc.", Some((0, "bc"))),
+                    ("aa.", Some((1, "aa"))),
+                    ("bcd?", Some((2, "bcd"))),
+                    ("b.", None),
+                    ("c.", None),
+                ],
+            },
+            // regex with an alternative including the empty string
+            Row {
+                rules: vec![Rule::pattern("a(b|)+c", "")],
+                separators: vec![],
+                examples: vec![
+                    ("ac.", Some((0, "ac"))),
+                    ("abc.", Some((0, "abc"))),
+                    ("abbc.", Some((0, "abbc"))),
+                ],
+            },
+            // separators
+            Row {
+                rules: vec![Rule::pattern("[a-f]+", "")],
+                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
+                examples: vec![
+                    ("  a", Some((0, "a"))),
+                    ("  \nb", Some((0, "b"))),
+                    ("  \\a", None),
+                    ("  \\\na", Some((0, "a"))),
+                ],
+            },
+            // shorter tokens with higher precedence
+            Row {
+                rules: vec![
+                    Rule::prec(Precedence::Integer(2), Rule::pattern("abc", "")),
+                    Rule::prec(Precedence::Integer(1), Rule::pattern("ab[cd]e", "")),
+                    Rule::pattern("[a-e]+", ""),
+                ],
+                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
+                examples: vec![
+                    ("abceef", Some((0, "abc"))),
+                    ("abdeef", Some((1, "abde"))),
+                    ("aeeeef", Some((2, "aeeee"))),
+                ],
+            },
+            // immediate tokens with higher precedence
+            Row {
+                rules: vec![
+                    Rule::prec(Precedence::Integer(1), Rule::pattern("[^a]+", "")),
+                    Rule::immediate_token(Rule::prec(
+                        Precedence::Integer(2),
+                        Rule::pattern("[^ab]+", ""),
+                    )),
+                ],
+                separators: vec![Rule::pattern("\\s", "")],
+                examples: vec![("cccb", Some((1, "ccc")))],
+            },
+            Row {
+                rules: vec![Rule::seq(vec![
+                    Rule::string("a"),
+                    Rule::choice(vec![Rule::string("b"), Rule::string("c")]),
+                    Rule::string("d"),
+                ])],
+                separators: vec![],
+                examples: vec![
+                    ("abd", Some((0, "abd"))),
+                    ("acd", Some((0, "acd"))),
+                    ("abc", None),
+                    ("ad", None),
+                    ("d", None),
+                    ("a", None),
+                ],
+            },
+            // nested choices within sequences
+            Row {
+                rules: vec![Rule::seq(vec![
+                    Rule::pattern("[0-9]+", ""),
+                    Rule::choice(vec![
+                        Rule::Blank,
+                        Rule::choice(vec![Rule::seq(vec![
+                            Rule::choice(vec![Rule::string("e"), Rule::string("E")]),
+                            Rule::choice(vec![
+                                Rule::Blank,
+                                Rule::choice(vec![Rule::string("+"), Rule::string("-")]),
+                            ]),
+                            Rule::pattern("[0-9]+", ""),
+                        ])]),
+                    ]),
+                ])],
+                separators: vec![],
+                examples: vec![
+                    ("12", Some((0, "12"))),
+                    ("12e", Some((0, "12"))),
+                    ("12g", Some((0, "12"))),
+                    ("12e3", Some((0, "12e3"))),
+                    ("12e+", Some((0, "12"))),
+                    ("12E+34 +", Some((0, "12E+34"))),
+                    ("12e34", Some((0, "12e34"))),
+                ],
+            },
+            // nested groups
+            Row {
+                rules: vec![Rule::seq(vec![Rule::pattern(r"([^x\\]|\\(.|\n))+", "")])],
+                separators: vec![],
+                examples: vec![("abcx", Some((0, "abc"))), ("abc\\0x", Some((0, "abc\\0")))],
+            },
+            // allowing unrecognized escape sequences
+            Row {
+                rules: vec![
+                    // Escaped forward slash (used in JS because '/' is the regex delimiter)
+                    Rule::pattern(r"\/", ""),
+                    // Escaped quotes
+                    Rule::pattern(r#"\"\'"#, ""),
+                    // Quote preceded by a literal backslash
+                    Rule::pattern(r"[\\']+", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("/", Some((0, "/"))),
+                    ("\"\'", Some((1, "\"\'"))),
+                    (r"'\'a", Some((2, r"'\'"))),
+                ],
+            },
+            // unicode property escapes
+            Row {
+                rules: vec![
+                    Rule::pattern(r"\p{L}+\P{L}+", ""),
+                    Rule::pattern(r"\p{White_Space}+\P{White_Space}+[\p{White_Space}]*", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("  123   abc", Some((1, "  123   "))),
+                    ("ბΨƁ___ƀƔ", Some((0, "ბΨƁ___"))),
+                ],
+            },
+            // unicode property escapes in bracketed sets
+            Row {
+                rules: vec![Rule::pattern(r"[\p{L}\p{Nd}]+", "")],
+                separators: vec![],
+                examples: vec![("abΨ12٣٣, ok", Some((0, "abΨ12٣٣")))],
+            },
+            // unicode character escapes
+            Row {
+                rules: vec![
+                    Rule::pattern(r"\u{00dc}", ""),
+                    Rule::pattern(r"\U{000000dd}", ""),
+                    Rule::pattern(r"\u00de", ""),
+                    Rule::pattern(r"\U000000df", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("\u{00dc}", Some((0, "\u{00dc}"))),
+                    ("\u{00dd}", Some((1, "\u{00dd}"))),
+                    ("\u{00de}", Some((2, "\u{00de}"))),
+                    ("\u{00df}", Some((3, "\u{00df}"))),
+                ],
+            },
+            Row {
+                rules: vec![
+                    Rule::pattern(r"u\{[0-9a-fA-F]+\}", ""),
+                    // Already-escaped curly braces
+                    Rule::pattern(r"\{[ab]{3}\}", ""),
+                    // Unicode codepoints
+                    Rule::pattern(r"\u{1000A}", ""),
+                    // Unicode codepoints (lowercase)
+                    Rule::pattern(r"\u{1000b}", ""),
+                ],
+                separators: vec![],
+                examples: vec![
+                    ("u{1234} ok", Some((0, "u{1234}"))),
+                    ("{aba}}", Some((1, "{aba}"))),
+                    ("\u{1000A}", Some((2, "\u{1000A}"))),
+                    ("\u{1000b}", Some((3, "\u{1000b}"))),
+                ],
+            },
+            // Emojis
+            Row {
+                rules: vec![Rule::pattern(r"\p{Emoji}+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("🐎", Some((0, "🐎"))),
+                    ("🐴🐴", Some((0, "🐴🐴"))),
+                    ("#0", Some((0, "#0"))), // These chars are technically emojis!
+                    ("⻢", None),
+                    ("♞", None),
+                    ("horse", None),
+                ],
+            },
+            // Intersection
+            Row {
+                rules: vec![Rule::pattern(r"[[0-7]&&[4-9]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("456", Some((0, "456"))),
+                    ("64", Some((0, "64"))),
+                    ("452", Some((0, "45"))),
+                    ("91", None),
+                    ("8", None),
+                    ("3", None),
+                ],
+            },
+            // Difference
+            Row {
+                rules: vec![Rule::pattern(r"[[0-9]--[4-7]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("123", Some((0, "123"))),
+                    ("83", Some((0, "83"))),
+                    ("9", Some((0, "9"))),
+                    ("124", Some((0, "12"))),
+                    ("67", None),
+                    ("4", None),
+                ],
+            },
+            // Symmetric difference
+            Row {
+                rules: vec![Rule::pattern(r"[[0-7]~~[4-9]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("123", Some((0, "123"))),
+                    ("83", Some((0, "83"))),
+                    ("9", Some((0, "9"))),
+                    ("124", Some((0, "12"))),
+                    ("67", None),
+                    ("4", None),
+                ],
+            },
+            // Nested set operations
+            Row {
+                //               0 1 2 3 4 5 6 7 8 9
+                // [0-5]:        y y y y y y
+                // [2-4]:            y y y
+                // [0-5]--[2-4]: y y       y
+                // [3-9]:              y y y y y y y
+                // [6-7]:                    y y
+                // [3-9]--[5-7]:       y y y     y y
+                // final regex:  y y   y y       y y
+                rules: vec![Rule::pattern(r"[[[0-5]--[2-4]]~~[[3-9]--[6-7]]]+", "")],
+                separators: vec![],
+                examples: vec![
+                    ("01", Some((0, "01"))),
+                    ("432", Some((0, "43"))),
+                    ("8", Some((0, "8"))),
+                    ("9", Some((0, "9"))),
+                    ("2", None),
+                    ("567", None),
+                ],
+            },
+        ];
+
+        for Row {
+            rules,
+            separators,
+            examples,
+        } in &table
+        {
+            let grammar = expand_tokens(ExtractedLexicalGrammar {
+                separators: separators.clone(),
+                variables: rules
+                    .iter()
+                    .map(|rule| Variable::named("", rule.clone()))
+                    .collect(),
+            })
+            .unwrap();
+
+            for (haystack, needle) in examples {
+                assert_eq!(simulate_nfa(&grammar, haystack), *needle);
+            }
+        }
+    }
+}

cli/generate/src/prepare_grammar/extract_default_aliases.rs

@@ -0,0 +1,304 @@
+use crate::{
+    grammars::{LexicalGrammar, SyntaxGrammar},
+    rules::{Alias, AliasMap, Symbol, SymbolType},
+};
+
+#[derive(Clone, Default)]
+struct SymbolStatus {
+    aliases: Vec<(Alias, usize)>,
+    appears_unaliased: bool,
+}
+
+// Update the grammar by finding symbols that always are aliased, and for each such symbol,
+// promoting one of its aliases to a "default alias", which is applied globally instead
+// of in a context-specific way.
+//
+// This has two benefits:
+// * It reduces the overhead of storing production-specific alias info in the parse table.
+// * Within an `ERROR` node, no context-specific aliases will be applied. This transformation
+//   ensures that the children of an `ERROR` node have symbols that are consistent with the way that
+//   they would appear in a valid syntax tree.
+pub(super) fn extract_default_aliases(
+    syntax_grammar: &mut SyntaxGrammar,
+    lexical_grammar: &LexicalGrammar,
+) -> AliasMap {
+    let mut terminal_status_list = vec![SymbolStatus::default(); lexical_grammar.variables.len()];
+    let mut non_terminal_status_list =
+        vec![SymbolStatus::default(); syntax_grammar.variables.len()];
+    let mut external_status_list =
+        vec![SymbolStatus::default(); syntax_grammar.external_tokens.len()];
+
+    // For each grammar symbol, find all of the aliases under which the symbol appears,
+    // and determine whether or not the symbol ever appears *unaliased*.
+    for variable in &syntax_grammar.variables {
+        for production in &variable.productions {
+            for step in &production.steps {
+                let status = match step.symbol.kind {
+                    SymbolType::External => &mut external_status_list[step.symbol.index],
+                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
+                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
+                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                        panic!("Unexpected end token")
+                    }
+                };
+
+                // Default aliases don't work for inlined variables.
+                if syntax_grammar.variables_to_inline.contains(&step.symbol) {
+                    continue;
+                }
+
+                if let Some(alias) = &step.alias {
+                    if let Some(count_for_alias) = status
+                        .aliases
+                        .iter_mut()
+                        .find_map(|(a, count)| if a == alias { Some(count) } else { None })
+                    {
+                        *count_for_alias += 1;
+                    } else {
+                        status.aliases.push((alias.clone(), 1));
+                    }
+                } else {
+                    status.appears_unaliased = true;
+                }
+            }
+        }
+    }
+
+    for symbol in &syntax_grammar.extra_symbols {
+        let status = match symbol.kind {
+            SymbolType::External => &mut external_status_list[symbol.index],
+            SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
+            SymbolType::Terminal => &mut terminal_status_list[symbol.index],
+            SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                panic!("Unexpected end token")
+            }
+        };
+        status.appears_unaliased = true;
+    }
+
+    let symbols_with_statuses = (terminal_status_list
+        .iter_mut()
+        .enumerate()
+        .map(|(i, status)| (Symbol::terminal(i), status)))
+    .chain(
+        non_terminal_status_list
+            .iter_mut()
+            .enumerate()
+            .map(|(i, status)| (Symbol::non_terminal(i), status)),
+    )
+    .chain(
+        external_status_list
+            .iter_mut()
+            .enumerate()
+            .map(|(i, status)| (Symbol::external(i), status)),
+    );
+
+    // For each symbol that always appears aliased, find the alias the occurs most often,
+    // and designate that alias as the symbol's "default alias". Store all of these
+    // default aliases in a map that will be returned.
+    let mut result = AliasMap::new();
+    for (symbol, status) in symbols_with_statuses {
+        if status.appears_unaliased {
+            status.aliases.clear();
+        } else if let Some(default_entry) = status
+            .aliases
+            .iter()
+            .enumerate()
+            .max_by_key(|(i, (_, count))| (count, -(*i as i64)))
+            .map(|(_, entry)| entry.clone())
+        {
+            status.aliases.clear();
+            status.aliases.push(default_entry.clone());
+            result.insert(symbol, default_entry.0);
+        }
+    }
+
+    // Wherever a symbol is aliased as its default alias, remove the usage of the alias,
+    // because it will now be redundant.
+    let mut alias_positions_to_clear = Vec::new();
+    for variable in &mut syntax_grammar.variables {
+        alias_positions_to_clear.clear();
+
+        for (i, production) in variable.productions.iter().enumerate() {
+            for (j, step) in production.steps.iter().enumerate() {
+                let status = match step.symbol.kind {
+                    SymbolType::External => &mut external_status_list[step.symbol.index],
+                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
+                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
+                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
+                        panic!("Unexpected end token")
+                    }
+                };
+
+                // If this step is aliased as the symbol's default alias, then remove that alias.
+                if step.alias.is_some()
+                    && step.alias.as_ref() == status.aliases.first().map(|t| &t.0)
+                {
+                    let mut other_productions_must_use_this_alias_at_this_index = false;
+                    for (other_i, other_production) in variable.productions.iter().enumerate() {
+                        if other_i != i
+                            && other_production.steps.len() > j
+                            && other_production.steps[j].alias == step.alias
+                            && result.get(&other_production.steps[j].symbol) != step.alias.as_ref()
+                        {
+                            other_productions_must_use_this_alias_at_this_index = true;
+                            break;
+                        }
+                    }
+
+                    if !other_productions_must_use_this_alias_at_this_index {
+                        alias_positions_to_clear.push((i, j));
+                    }
+                }
+            }
+        }
+
+        for (production_index, step_index) in &alias_positions_to_clear {
+            variable.productions[*production_index].steps[*step_index].alias = None;
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType},
+        nfa::Nfa,
+    };
+
+    #[test]
+    fn test_extract_simple_aliases() {
+        let mut syntax_grammar = SyntaxGrammar {
+            variables: vec![
+                SyntaxVariable {
+                    name: "v1".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
+                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
+                            ProductionStep::new(Symbol::terminal(2)).with_alias("a3", true),
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "v2".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            // Token 0 is always aliased as "a1".
+                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
+                            // Token 1 is aliased within rule `v1` above, but not here.
+                            ProductionStep::new(Symbol::terminal(1)),
+                            // Token 2 is aliased differently here than in `v1`. The alias from
+                            // `v1` should be promoted to the default alias, because `v1` appears
+                            // first in the grammar.
+                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
+                            // Token 3 is also aliased differently here than in `v1`. In this case,
+                            // this alias should be promoted to the default alias, because it is
+                            // used a greater number of times (twice).
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
+                        ],
+                    }],
+                },
+            ],
+            ..Default::default()
+        };
+
+        let lexical_grammar = LexicalGrammar {
+            nfa: Nfa::new(),
+            variables: vec![
+                LexicalVariable {
+                    name: "t0".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+                LexicalVariable {
+                    name: "t1".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+                LexicalVariable {
+                    name: "t2".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+                LexicalVariable {
+                    name: "t3".to_string(),
+                    kind: VariableType::Anonymous,
+                    implicit_precedence: 0,
+                    start_state: 0,
+                },
+            ],
+        };
+
+        let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
+        assert_eq!(default_aliases.len(), 3);
+
+        assert_eq!(
+            default_aliases.get(&Symbol::terminal(0)),
+            Some(&Alias {
+                value: "a1".to_string(),
+                is_named: true,
+            })
+        );
+        assert_eq!(
+            default_aliases.get(&Symbol::terminal(2)),
+            Some(&Alias {
+                value: "a3".to_string(),
+                is_named: true,
+            })
+        );
+        assert_eq!(
+            default_aliases.get(&Symbol::terminal(3)),
+            Some(&Alias {
+                value: "a6".to_string(),
+                is_named: true,
+            })
+        );
+        assert_eq!(default_aliases.get(&Symbol::terminal(1)), None);
+
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![
+                SyntaxVariable {
+                    name: "v1".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(0)),
+                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
+                            ProductionStep::new(Symbol::terminal(2)),
+                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
+                        ],
+                    },],
+                },
+                SyntaxVariable {
+                    name: "v2".to_owned(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(0)),
+                            ProductionStep::new(Symbol::terminal(1)),
+                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
+                            ProductionStep::new(Symbol::terminal(3)),
+                            ProductionStep::new(Symbol::terminal(3)),
+                        ],
+                    },],
+                },
+            ]
+        );
+    }
+}

cli/generate/src/prepare_grammar/extract_tokens.rs

@@ -0,0 +1,609 @@
+use std::collections::HashMap;
+
+use anyhow::Result;
+use serde::Serialize;
+use thiserror::Error;
+
+use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar};
+use crate::{
+    grammars::{ExternalToken, ReservedWordContext, Variable, VariableType},
+    rules::{MetadataParams, Rule, Symbol, SymbolType},
+};
+
+pub type ExtractTokensResult<T> = Result<T, ExtractTokensError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum ExtractTokensError {
+    #[error(
+        "The rule `{0}` contains an empty string.
+
+Tree-sitter does not support syntactic rules that contain an empty string
+unless they are used only as the grammar's start rule.
+"
+    )]
+    EmptyString(String),
+    #[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
+    ExternalTokenNonTerminal(String),
+    #[error("Non-symbol rules cannot be used as external tokens")]
+    NonSymbolExternalToken,
+    #[error("Non-terminal symbol '{0}' cannot be used as the word token, because its rule is duplicated in '{1}'")]
+    NonTerminalWordToken(String, String),
+    #[error("Reserved words must be tokens")]
+    NonTokenReservedWord,
+}
+
+pub(super) fn extract_tokens(
+    mut grammar: InternedGrammar,
+) -> ExtractTokensResult<(ExtractedSyntaxGrammar, ExtractedLexicalGrammar)> {
+    let mut extractor = TokenExtractor {
+        current_variable_name: String::new(),
+        current_variable_token_count: 0,
+        is_first_rule: false,
+        extracted_variables: Vec::new(),
+        extracted_usage_counts: Vec::new(),
+    };
+
+    for (i, variable) in &mut grammar.variables.iter_mut().enumerate() {
+        extractor.extract_tokens_in_variable(i == 0, variable)?;
+    }
+
+    for variable in &mut grammar.external_tokens {
+        extractor.extract_tokens_in_variable(false, variable)?;
+    }
+
+    let mut lexical_variables = Vec::with_capacity(extractor.extracted_variables.len());
+    for variable in extractor.extracted_variables {
+        lexical_variables.push(variable);
+    }
+
+    // If a variable's entire rule was extracted as a token and that token didn't
+    // appear within any other rule, then remove that variable from the syntax
+    // grammar, giving its name to the token in the lexical grammar. Any symbols
+    // that pointed to that variable will need to be updated to point to the
+    // variable in the lexical grammar. Symbols that pointed to later variables
+    // will need to have their indices decremented.
+    let mut variables = Vec::new();
+    let mut symbol_replacer = SymbolReplacer {
+        replacements: HashMap::new(),
+    };
+    for (i, variable) in grammar.variables.into_iter().enumerate() {
+        if let Rule::Symbol(Symbol {
+            kind: SymbolType::Terminal,
+            index,
+        }) = variable.rule
+        {
+            if i > 0 && extractor.extracted_usage_counts[index] == 1 {
+                let lexical_variable = &mut lexical_variables[index];
+                if lexical_variable.kind == VariableType::Auxiliary
+                    || variable.kind != VariableType::Hidden
+                {
+                    lexical_variable.kind = variable.kind;
+                    lexical_variable.name = variable.name;
+                    symbol_replacer.replacements.insert(i, index);
+                    continue;
+                }
+            }
+        }
+        variables.push(variable);
+    }
+
+    for variable in &mut variables {
+        variable.rule = symbol_replacer.replace_symbols_in_rule(&variable.rule);
+    }
+
+    let expected_conflicts = grammar
+        .expected_conflicts
+        .into_iter()
+        .map(|conflict| {
+            let mut result = conflict
+                .iter()
+                .map(|symbol| symbol_replacer.replace_symbol(*symbol))
+                .collect::<Vec<_>>();
+            result.sort_unstable();
+            result.dedup();
+            result
+        })
+        .collect();
+
+    let supertype_symbols = grammar
+        .supertype_symbols
+        .into_iter()
+        .map(|symbol| symbol_replacer.replace_symbol(symbol))
+        .collect();
+
+    let variables_to_inline = grammar
+        .variables_to_inline
+        .into_iter()
+        .map(|symbol| symbol_replacer.replace_symbol(symbol))
+        .collect();
+
+    let mut separators = Vec::new();
+    let mut extra_symbols = Vec::new();
+    for rule in grammar.extra_symbols {
+        if let Rule::Symbol(symbol) = rule {
+            extra_symbols.push(symbol_replacer.replace_symbol(symbol));
+        } else if let Some(index) = lexical_variables.iter().position(|v| v.rule == rule) {
+            extra_symbols.push(Symbol::terminal(index));
+        } else {
+            separators.push(rule);
+        }
+    }
+
+    let mut external_tokens = Vec::new();
+    for external_token in grammar.external_tokens {
+        let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule);
+        if let Rule::Symbol(symbol) = rule {
+            if symbol.is_non_terminal() {
+                Err(ExtractTokensError::ExternalTokenNonTerminal(
+                    variables[symbol.index].name.clone(),
+                ))?;
+            }
+
+            if symbol.is_external() {
+                external_tokens.push(ExternalToken {
+                    name: external_token.name,
+                    kind: external_token.kind,
+                    corresponding_internal_token: None,
+                });
+            } else {
+                external_tokens.push(ExternalToken {
+                    name: lexical_variables[symbol.index].name.clone(),
+                    kind: external_token.kind,
+                    corresponding_internal_token: Some(symbol),
+                });
+            }
+        } else {
+            Err(ExtractTokensError::NonSymbolExternalToken)?;
+        }
+    }
+
+    let mut word_token = None;
+    if let Some(token) = grammar.word_token {
+        let token = symbol_replacer.replace_symbol(token);
+        if token.is_non_terminal() {
+            let word_token_variable = &variables[token.index];
+            let conflicting_variable = variables
+                .iter()
+                .enumerate()
+                .find(|(i, v)| *i != token.index && v.rule == word_token_variable.rule)
+                .expect("Failed to find a variable with the same rule as the word token");
+
+            Err(ExtractTokensError::NonTerminalWordToken(
+                word_token_variable.name.clone(),
+                conflicting_variable.1.name.clone(),
+            ))?;
+        }
+        word_token = Some(token);
+    }
+
+    let mut reserved_word_contexts = Vec::new();
+    for reserved_word_context in grammar.reserved_word_sets {
+        let mut reserved_words = Vec::new();
+        for reserved_rule in reserved_word_context.reserved_words {
+            if let Rule::Symbol(symbol) = reserved_rule {
+                reserved_words.push(symbol_replacer.replace_symbol(symbol));
+            } else if let Some(index) = lexical_variables
+                .iter()
+                .position(|v| v.rule == reserved_rule)
+            {
+                reserved_words.push(Symbol::terminal(index));
+            } else {
+                Err(ExtractTokensError::NonTokenReservedWord)?;
+            }
+        }
+        reserved_word_contexts.push(ReservedWordContext {
+            name: reserved_word_context.name,
+            reserved_words,
+        });
+    }
+
+    Ok((
+        ExtractedSyntaxGrammar {
+            variables,
+            expected_conflicts,
+            extra_symbols,
+            variables_to_inline,
+            supertype_symbols,
+            external_tokens,
+            word_token,
+            precedence_orderings: grammar.precedence_orderings,
+            reserved_word_sets: reserved_word_contexts,
+        },
+        ExtractedLexicalGrammar {
+            variables: lexical_variables,
+            separators,
+        },
+    ))
+}
+
+struct TokenExtractor {
+    current_variable_name: String,
+    current_variable_token_count: usize,
+    is_first_rule: bool,
+    extracted_variables: Vec<Variable>,
+    extracted_usage_counts: Vec<usize>,
+}
+
+struct SymbolReplacer {
+    replacements: HashMap<usize, usize>,
+}
+
+impl TokenExtractor {
+    fn extract_tokens_in_variable(
+        &mut self,
+        is_first: bool,
+        variable: &mut Variable,
+    ) -> ExtractTokensResult<()> {
+        self.current_variable_name.clear();
+        self.current_variable_name.push_str(&variable.name);
+        self.current_variable_token_count = 0;
+        self.is_first_rule = is_first;
+        variable.rule = self.extract_tokens_in_rule(&variable.rule)?;
+        Ok(())
+    }
+
+    fn extract_tokens_in_rule(&mut self, input: &Rule) -> ExtractTokensResult<Rule> {
+        match input {
+            Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()),
+            Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()),
+            Rule::Metadata { params, rule } => {
+                if params.is_token {
+                    let mut params = params.clone();
+                    params.is_token = false;
+
+                    let mut string_value = None;
+                    if let Rule::String(value) = rule.as_ref() {
+                        string_value = Some(value);
+                    }
+
+                    let rule_to_extract = if params == MetadataParams::default() {
+                        rule.as_ref()
+                    } else {
+                        input
+                    };
+
+                    Ok(self.extract_token(rule_to_extract, string_value)?.into())
+                } else {
+                    Ok(Rule::Metadata {
+                        params: params.clone(),
+                        rule: Box::new(self.extract_tokens_in_rule(rule)?),
+                    })
+                }
+            }
+            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(
+                self.extract_tokens_in_rule(content)?,
+            ))),
+            Rule::Seq(elements) => Ok(Rule::Seq(
+                elements
+                    .iter()
+                    .map(|e| self.extract_tokens_in_rule(e))
+                    .collect::<ExtractTokensResult<Vec<_>>>()?,
+            )),
+            Rule::Choice(elements) => Ok(Rule::Choice(
+                elements
+                    .iter()
+                    .map(|e| self.extract_tokens_in_rule(e))
+                    .collect::<ExtractTokensResult<Vec<_>>>()?,
+            )),
+            Rule::Reserved { rule, context_name } => Ok(Rule::Reserved {
+                rule: Box::new(self.extract_tokens_in_rule(rule)?),
+                context_name: context_name.clone(),
+            }),
+            _ => Ok(input.clone()),
+        }
+    }
+
+    fn extract_token(
+        &mut self,
+        rule: &Rule,
+        string_value: Option<&String>,
+    ) -> ExtractTokensResult<Symbol> {
+        for (i, variable) in self.extracted_variables.iter_mut().enumerate() {
+            if variable.rule == *rule {
+                self.extracted_usage_counts[i] += 1;
+                return Ok(Symbol::terminal(i));
+            }
+        }
+
+        let index = self.extracted_variables.len();
+        let variable = if let Some(string_value) = string_value {
+            if string_value.is_empty() && !self.is_first_rule {
+                Err(ExtractTokensError::EmptyString(
+                    self.current_variable_name.clone(),
+                ))?;
+            }
+            Variable {
+                name: string_value.clone(),
+                kind: VariableType::Anonymous,
+                rule: rule.clone(),
+            }
+        } else {
+            self.current_variable_token_count += 1;
+            Variable {
+                name: format!(
+                    "{}_token{}",
+                    self.current_variable_name, self.current_variable_token_count
+                ),
+                kind: VariableType::Auxiliary,
+                rule: rule.clone(),
+            }
+        };
+
+        self.extracted_variables.push(variable);
+        self.extracted_usage_counts.push(1);
+        Ok(Symbol::terminal(index))
+    }
+}
+
+impl SymbolReplacer {
+    fn replace_symbols_in_rule(&mut self, rule: &Rule) -> Rule {
+        match rule {
+            Rule::Symbol(symbol) => self.replace_symbol(*symbol).into(),
+            Rule::Choice(elements) => Rule::Choice(
+                elements
+                    .iter()
+                    .map(|e| self.replace_symbols_in_rule(e))
+                    .collect(),
+            ),
+            Rule::Seq(elements) => Rule::Seq(
+                elements
+                    .iter()
+                    .map(|e| self.replace_symbols_in_rule(e))
+                    .collect(),
+            ),
+            Rule::Repeat(content) => Rule::Repeat(Box::new(self.replace_symbols_in_rule(content))),
+            Rule::Metadata { rule, params } => Rule::Metadata {
+                params: params.clone(),
+                rule: Box::new(self.replace_symbols_in_rule(rule)),
+            },
+            Rule::Reserved { rule, context_name } => Rule::Reserved {
+                rule: Box::new(self.replace_symbols_in_rule(rule)),
+                context_name: context_name.clone(),
+            },
+            _ => rule.clone(),
+        }
+    }
+
+    fn replace_symbol(&self, symbol: Symbol) -> Symbol {
+        if !symbol.is_non_terminal() {
+            return symbol;
+        }
+
+        if let Some(replacement) = self.replacements.get(&symbol.index) {
+            return Symbol::terminal(*replacement);
+        }
+
+        let mut adjusted_index = symbol.index;
+        for replaced_index in self.replacements.keys() {
+            if *replaced_index < symbol.index {
+                adjusted_index -= 1;
+            }
+        }
+
+        Symbol::non_terminal(adjusted_index)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_extraction() {
+        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
+            Variable::named(
+                "rule_0",
+                Rule::repeat(Rule::seq(vec![
+                    Rule::string("a"),
+                    Rule::pattern("b", ""),
+                    Rule::choice(vec![
+                        Rule::non_terminal(1),
+                        Rule::non_terminal(2),
+                        Rule::token(Rule::repeat(Rule::choice(vec![
+                            Rule::string("c"),
+                            Rule::string("d"),
+                        ]))),
+                    ]),
+                ])),
+            ),
+            Variable::named("rule_1", Rule::pattern("e", "")),
+            Variable::named("rule_2", Rule::pattern("b", "")),
+            Variable::named(
+                "rule_3",
+                Rule::seq(vec![Rule::non_terminal(2), Rule::Blank]),
+            ),
+        ]))
+        .unwrap();
+
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![
+                Variable::named(
+                    "rule_0",
+                    Rule::repeat(Rule::seq(vec![
+                        // The string "a" was replaced by a symbol referencing the lexical grammar
+                        Rule::terminal(0),
+                        // The pattern "b" was replaced by a symbol referencing the lexical grammar
+                        Rule::terminal(1),
+                        Rule::choice(vec![
+                            // The symbol referencing `rule_1` was replaced by a symbol referencing
+                            // the lexical grammar.
+                            Rule::terminal(3),
+                            // The symbol referencing `rule_2` had its index decremented because
+                            // `rule_1` was moved to the lexical grammar.
+                            Rule::non_terminal(1),
+                            // The rule wrapped in `token` was replaced by a symbol referencing
+                            // the lexical grammar.
+                            Rule::terminal(2),
+                        ])
+                    ]))
+                ),
+                // The pattern "e" was only used in once place: as the definition of `rule_1`,
+                // so that rule was moved to the lexical grammar. The pattern "b" appeared in
+                // two places, so it was not moved into the lexical grammar.
+                Variable::named("rule_2", Rule::terminal(1)),
+                Variable::named(
+                    "rule_3",
+                    Rule::seq(vec![Rule::non_terminal(1), Rule::Blank,])
+                ),
+            ]
+        );
+
+        assert_eq!(
+            lexical_grammar.variables,
+            vec![
+                Variable::anonymous("a", Rule::string("a")),
+                Variable::auxiliary("rule_0_token1", Rule::pattern("b", "")),
+                Variable::auxiliary(
+                    "rule_0_token2",
+                    Rule::repeat(Rule::choice(vec![Rule::string("c"), Rule::string("d"),]))
+                ),
+                Variable::named("rule_1", Rule::pattern("e", "")),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_start_rule_is_token() {
+        let (syntax_grammar, lexical_grammar) =
+            extract_tokens(build_grammar(vec![Variable::named(
+                "rule_0",
+                Rule::string("hello"),
+            )]))
+            .unwrap();
+
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![Variable::named("rule_0", Rule::terminal(0)),]
+        );
+        assert_eq!(
+            lexical_grammar.variables,
+            vec![Variable::anonymous("hello", Rule::string("hello")),]
+        );
+    }
+
+    #[test]
+    fn test_extracting_extra_symbols() {
+        let mut grammar = build_grammar(vec![
+            Variable::named("rule_0", Rule::string("x")),
+            Variable::named("comment", Rule::pattern("//.*", "")),
+        ]);
+        grammar.extra_symbols = vec![Rule::string(" "), Rule::non_terminal(1)];
+
+        let (syntax_grammar, lexical_grammar) = extract_tokens(grammar).unwrap();
+        assert_eq!(syntax_grammar.extra_symbols, vec![Symbol::terminal(1),]);
+        assert_eq!(lexical_grammar.separators, vec![Rule::string(" "),]);
+    }
+
+    #[test]
+    fn test_extract_externals() {
+        let mut grammar = build_grammar(vec![
+            Variable::named(
+                "rule_0",
+                Rule::seq(vec![
+                    Rule::external(0),
+                    Rule::string("a"),
+                    Rule::non_terminal(1),
+                    Rule::non_terminal(2),
+                ]),
+            ),
+            Variable::named("rule_1", Rule::string("b")),
+            Variable::named("rule_2", Rule::string("c")),
+        ]);
+        grammar.external_tokens = vec![
+            Variable::named("external_0", Rule::external(0)),
+            Variable::anonymous("a", Rule::string("a")),
+            Variable::named("rule_2", Rule::non_terminal(2)),
+        ];
+
+        let (syntax_grammar, _) = extract_tokens(grammar).unwrap();
+
+        assert_eq!(
+            syntax_grammar.external_tokens,
+            vec![
+                ExternalToken {
+                    name: "external_0".to_string(),
+                    kind: VariableType::Named,
+                    corresponding_internal_token: None,
+                },
+                ExternalToken {
+                    name: "a".to_string(),
+                    kind: VariableType::Anonymous,
+                    corresponding_internal_token: Some(Symbol::terminal(0)),
+                },
+                ExternalToken {
+                    name: "rule_2".to_string(),
+                    kind: VariableType::Named,
+                    corresponding_internal_token: Some(Symbol::terminal(2)),
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_error_on_external_with_same_name_as_non_terminal() {
+        let mut grammar = build_grammar(vec![
+            Variable::named(
+                "rule_0",
+                Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
+            ),
+            Variable::named(
+                "rule_1",
+                Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2)]),
+            ),
+            Variable::named("rule_2", Rule::string("a")),
+        ]);
+        grammar.external_tokens = vec![Variable::named("rule_1", Rule::non_terminal(1))];
+
+        match extract_tokens(grammar) {
+            Err(e) => {
+                assert_eq!(e.to_string(), "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule");
+            }
+            _ => {
+                panic!("Expected an error but got no error");
+            }
+        }
+    }
+
+    #[test]
+    fn test_extraction_on_hidden_terminal() {
+        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
+            Variable::named("rule_0", Rule::non_terminal(1)),
+            Variable::hidden("_rule_1", Rule::string("a")),
+        ]))
+        .unwrap();
+
+        // The rule `_rule_1` should not "absorb" the
+        // terminal "a", since it is hidden,
+        // so we expect two variables still
+        assert_eq!(
+            syntax_grammar.variables,
+            vec![
+                Variable::named("rule_0", Rule::non_terminal(1)),
+                Variable::hidden("_rule_1", Rule::terminal(0)),
+            ]
+        );
+
+        // We should not have a hidden rule in our lexical grammar, only the terminal "a"
+        assert_eq!(
+            lexical_grammar.variables,
+            vec![Variable::anonymous("a", Rule::string("a"))]
+        );
+    }
+
+    #[test]
+    fn test_extraction_with_empty_string() {
+        assert!(extract_tokens(build_grammar(vec![
+            Variable::named("rule_0", Rule::non_terminal(1)),
+            Variable::hidden("_rule_1", Rule::string("")),
+        ]))
+        .is_err());
+    }
+
+    fn build_grammar(variables: Vec<Variable>) -> InternedGrammar {
+        InternedGrammar {
+            variables,
+            ..Default::default()
+        }
+    }
+}

cli/generate/src/prepare_grammar/flatten_grammar.rs

@@ -0,0 +1,539 @@
+use std::collections::HashMap;
+
+use anyhow::Result;
+use serde::Serialize;
+use thiserror::Error;
+
+use super::ExtractedSyntaxGrammar;
+use crate::{
+    grammars::{
+        Production, ProductionStep, ReservedWordSetId, SyntaxGrammar, SyntaxVariable, Variable,
+    },
+    rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet},
+};
+
+pub type FlattenGrammarResult<T> = Result<T, FlattenGrammarError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum FlattenGrammarError {
+    #[error("No such reserved word set: {0}")]
+    NoReservedWordSet(String),
+    #[error(
+        "The rule `{0}` matches the empty string.
+
+Tree-sitter does not support syntactic rules that match the empty string
+unless they are used only as the grammar's start rule.
+"
+    )]
+    EmptyString(String),
+    #[error("Rule `{0}` cannot be inlined because it contains a reference to itself")]
+    RecursiveInline(String),
+}
+
+struct RuleFlattener {
+    production: Production,
+    reserved_word_set_ids: HashMap<String, ReservedWordSetId>,
+    precedence_stack: Vec<Precedence>,
+    associativity_stack: Vec<Associativity>,
+    reserved_word_stack: Vec<ReservedWordSetId>,
+    alias_stack: Vec<Alias>,
+    field_name_stack: Vec<String>,
+}
+
+impl RuleFlattener {
+    const fn new(reserved_word_set_ids: HashMap<String, ReservedWordSetId>) -> Self {
+        Self {
+            production: Production {
+                steps: Vec::new(),
+                dynamic_precedence: 0,
+            },
+            reserved_word_set_ids,
+            precedence_stack: Vec::new(),
+            associativity_stack: Vec::new(),
+            reserved_word_stack: Vec::new(),
+            alias_stack: Vec::new(),
+            field_name_stack: Vec::new(),
+        }
+    }
+
+    fn flatten_variable(&mut self, variable: Variable) -> FlattenGrammarResult<SyntaxVariable> {
+        let mut productions = Vec::new();
+        for rule in extract_choices(variable.rule) {
+            let production = self.flatten_rule(rule)?;
+            if !productions.contains(&production) {
+                productions.push(production);
+            }
+        }
+        Ok(SyntaxVariable {
+            name: variable.name,
+            kind: variable.kind,
+            productions,
+        })
+    }
+
+    fn flatten_rule(&mut self, rule: Rule) -> FlattenGrammarResult<Production> {
+        self.production = Production::default();
+        self.alias_stack.clear();
+        self.reserved_word_stack.clear();
+        self.precedence_stack.clear();
+        self.associativity_stack.clear();
+        self.field_name_stack.clear();
+        self.apply(rule, true)?;
+        Ok(self.production.clone())
+    }
+
+    fn apply(&mut self, rule: Rule, at_end: bool) -> FlattenGrammarResult<bool> {
+        match rule {
+            Rule::Seq(members) => {
+                let mut result = false;
+                let last_index = members.len() - 1;
+                for (i, member) in members.into_iter().enumerate() {
+                    result |= self.apply(member, i == last_index && at_end)?;
+                }
+                Ok(result)
+            }
+            Rule::Metadata { rule, params } => {
+                let mut has_precedence = false;
+                if !params.precedence.is_none() {
+                    has_precedence = true;
+                    self.precedence_stack.push(params.precedence);
+                }
+
+                let mut has_associativity = false;
+                if let Some(associativity) = params.associativity {
+                    has_associativity = true;
+                    self.associativity_stack.push(associativity);
+                }
+
+                let mut has_alias = false;
+                if let Some(alias) = params.alias {
+                    has_alias = true;
+                    self.alias_stack.push(alias);
+                }
+
+                let mut has_field_name = false;
+                if let Some(field_name) = params.field_name {
+                    has_field_name = true;
+                    self.field_name_stack.push(field_name);
+                }
+
+                if params.dynamic_precedence.abs() > self.production.dynamic_precedence.abs() {
+                    self.production.dynamic_precedence = params.dynamic_precedence;
+                }
+
+                let did_push = self.apply(*rule, at_end)?;
+
+                if has_precedence {
+                    self.precedence_stack.pop();
+                    if did_push && !at_end {
+                        self.production.steps.last_mut().unwrap().precedence = self
+                            .precedence_stack
+                            .last()
+                            .cloned()
+                            .unwrap_or(Precedence::None);
+                    }
+                }
+
+                if has_associativity {
+                    self.associativity_stack.pop();
+                    if did_push && !at_end {
+                        self.production.steps.last_mut().unwrap().associativity =
+                            self.associativity_stack.last().copied();
+                    }
+                }
+
+                if has_alias {
+                    self.alias_stack.pop();
+                }
+
+                if has_field_name {
+                    self.field_name_stack.pop();
+                }
+
+                Ok(did_push)
+            }
+            Rule::Reserved { rule, context_name } => {
+                self.reserved_word_stack.push(
+                    self.reserved_word_set_ids
+                        .get(&context_name)
+                        .copied()
+                        .ok_or_else(|| {
+                            FlattenGrammarError::NoReservedWordSet(context_name.clone())
+                        })?,
+                );
+                let did_push = self.apply(*rule, at_end)?;
+                self.reserved_word_stack.pop();
+                Ok(did_push)
+            }
+            Rule::Symbol(symbol) => {
+                self.production.steps.push(ProductionStep {
+                    symbol,
+                    precedence: self
+                        .precedence_stack
+                        .last()
+                        .cloned()
+                        .unwrap_or(Precedence::None),
+                    associativity: self.associativity_stack.last().copied(),
+                    reserved_word_set_id: self
+                        .reserved_word_stack
+                        .last()
+                        .copied()
+                        .unwrap_or(ReservedWordSetId::default()),
+                    alias: self.alias_stack.last().cloned(),
+                    field_name: self.field_name_stack.last().cloned(),
+                });
+                Ok(true)
+            }
+            _ => Ok(false),
+        }
+    }
+}
+
+fn extract_choices(rule: Rule) -> Vec<Rule> {
+    match rule {
+        Rule::Seq(elements) => {
+            let mut result = vec![Rule::Blank];
+            for element in elements {
+                let extraction = extract_choices(element);
+                let mut next_result = Vec::new();
+                for entry in result {
+                    for extraction_entry in &extraction {
+                        next_result.push(Rule::Seq(vec![entry.clone(), extraction_entry.clone()]));
+                    }
+                }
+                result = next_result;
+            }
+            result
+        }
+        Rule::Choice(elements) => {
+            let mut result = Vec::new();
+            for element in elements {
+                for rule in extract_choices(element) {
+                    result.push(rule);
+                }
+            }
+            result
+        }
+        Rule::Metadata { rule, params } => extract_choices(*rule)
+            .into_iter()
+            .map(|rule| Rule::Metadata {
+                rule: Box::new(rule),
+                params: params.clone(),
+            })
+            .collect(),
+        Rule::Reserved { rule, context_name } => extract_choices(*rule)
+            .into_iter()
+            .map(|rule| Rule::Reserved {
+                rule: Box::new(rule),
+                context_name: context_name.clone(),
+            })
+            .collect(),
+        _ => vec![rule],
+    }
+}
+
+fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool {
+    for variable in variables {
+        for production in &variable.productions {
+            for step in &production.steps {
+                if step.symbol == symbol {
+                    return true;
+                }
+            }
+        }
+    }
+    false
+}
+
+pub(super) fn flatten_grammar(
+    grammar: ExtractedSyntaxGrammar,
+) -> FlattenGrammarResult<SyntaxGrammar> {
+    let mut reserved_word_set_ids_by_name = HashMap::new();
+    for (ix, set) in grammar.reserved_word_sets.iter().enumerate() {
+        reserved_word_set_ids_by_name.insert(set.name.clone(), ReservedWordSetId(ix));
+    }
+
+    let mut flattener = RuleFlattener::new(reserved_word_set_ids_by_name);
+    let variables = grammar
+        .variables
+        .into_iter()
+        .map(|variable| flattener.flatten_variable(variable))
+        .collect::<FlattenGrammarResult<Vec<_>>>()?;
+
+    for (i, variable) in variables.iter().enumerate() {
+        let symbol = Symbol::non_terminal(i);
+
+        for production in &variable.productions {
+            if production.steps.is_empty() && symbol_is_used(&variables, symbol) {
+                Err(FlattenGrammarError::EmptyString(variable.name.clone()))?;
+            }
+
+            if grammar.variables_to_inline.contains(&symbol)
+                && production.steps.iter().any(|step| step.symbol == symbol)
+            {
+                Err(FlattenGrammarError::RecursiveInline(variable.name.clone()))?;
+            }
+        }
+    }
+    let mut reserved_word_sets = grammar
+        .reserved_word_sets
+        .into_iter()
+        .map(|set| set.reserved_words.into_iter().collect())
+        .collect::<Vec<_>>();
+
+    // If no default reserved word set is specified, there are no reserved words.
+    if reserved_word_sets.is_empty() {
+        reserved_word_sets.push(TokenSet::default());
+    }
+
+    Ok(SyntaxGrammar {
+        extra_symbols: grammar.extra_symbols,
+        expected_conflicts: grammar.expected_conflicts,
+        variables_to_inline: grammar.variables_to_inline,
+        precedence_orderings: grammar.precedence_orderings,
+        external_tokens: grammar.external_tokens,
+        supertype_symbols: grammar.supertype_symbols,
+        word_token: grammar.word_token,
+        reserved_word_sets,
+        variables,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::grammars::VariableType;
+
+    #[test]
+    fn test_flatten_grammar() {
+        let mut flattener = RuleFlattener::new(HashMap::default());
+        let result = flattener
+            .flatten_variable(Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::seq(vec![
+                    Rule::non_terminal(1),
+                    Rule::prec_left(
+                        Precedence::Integer(101),
+                        Rule::seq(vec![
+                            Rule::non_terminal(2),
+                            Rule::choice(vec![
+                                Rule::prec_right(
+                                    Precedence::Integer(102),
+                                    Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
+                                ),
+                                Rule::non_terminal(5),
+                            ]),
+                            Rule::non_terminal(6),
+                        ]),
+                    ),
+                    Rule::non_terminal(7),
+                ]),
+            })
+            .unwrap();
+
+        assert_eq!(
+            result.productions,
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(3))
+                            .with_prec(Precedence::Integer(102), Some(Associativity::Right)),
+                        ProductionStep::new(Symbol::non_terminal(4))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ]
+                },
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(5))
+                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ]
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_maximum_dynamic_precedence() {
+        let mut flattener = RuleFlattener::new(HashMap::default());
+        let result = flattener
+            .flatten_variable(Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::seq(vec![
+                    Rule::non_terminal(1),
+                    Rule::prec_dynamic(
+                        101,
+                        Rule::seq(vec![
+                            Rule::non_terminal(2),
+                            Rule::choice(vec![
+                                Rule::prec_dynamic(
+                                    102,
+                                    Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
+                                ),
+                                Rule::non_terminal(5),
+                            ]),
+                            Rule::non_terminal(6),
+                        ]),
+                    ),
+                    Rule::non_terminal(7),
+                ]),
+            })
+            .unwrap();
+
+        assert_eq!(
+            result.productions,
+            vec![
+                Production {
+                    dynamic_precedence: 102,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::non_terminal(3)),
+                        ProductionStep::new(Symbol::non_terminal(4)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ],
+                },
+                Production {
+                    dynamic_precedence: 101,
+                    steps: vec![
+                        ProductionStep::new(Symbol::non_terminal(1)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::non_terminal(5)),
+                        ProductionStep::new(Symbol::non_terminal(6)),
+                        ProductionStep::new(Symbol::non_terminal(7)),
+                    ],
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_final_precedence() {
+        let mut flattener = RuleFlattener::new(HashMap::default());
+        let result = flattener
+            .flatten_variable(Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::prec_left(
+                    Precedence::Integer(101),
+                    Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
+                ),
+            })
+            .unwrap();
+
+        assert_eq!(
+            result.productions,
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    ProductionStep::new(Symbol::non_terminal(1))
+                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                    ProductionStep::new(Symbol::non_terminal(2))
+                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
+                ]
+            }]
+        );
+
+        let result = flattener
+            .flatten_variable(Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::prec_left(
+                    Precedence::Integer(101),
+                    Rule::seq(vec![Rule::non_terminal(1)]),
+                ),
+            })
+            .unwrap();
+
+        assert_eq!(
+            result.productions,
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![ProductionStep::new(Symbol::non_terminal(1))
+                    .with_prec(Precedence::Integer(101), Some(Associativity::Left)),]
+            }]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_field_names() {
+        let mut flattener = RuleFlattener::new(HashMap::default());
+        let result = flattener
+            .flatten_variable(Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::seq(vec![
+                    Rule::field("first-thing".to_string(), Rule::terminal(1)),
+                    Rule::terminal(2),
+                    Rule::choice(vec![
+                        Rule::Blank,
+                        Rule::field("second-thing".to_string(), Rule::terminal(3)),
+                    ]),
+                ]),
+            })
+            .unwrap();
+
+        assert_eq!(
+            result.productions,
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
+                        ProductionStep::new(Symbol::terminal(2))
+                    ]
+                },
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
+                        ProductionStep::new(Symbol::terminal(2)),
+                        ProductionStep::new(Symbol::terminal(3)).with_field_name("second-thing"),
+                    ]
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_flatten_grammar_with_recursive_inline_variable() {
+        let result = flatten_grammar(ExtractedSyntaxGrammar {
+            extra_symbols: Vec::new(),
+            expected_conflicts: Vec::new(),
+            variables_to_inline: vec![Symbol::non_terminal(0)],
+            precedence_orderings: Vec::new(),
+            external_tokens: Vec::new(),
+            supertype_symbols: Vec::new(),
+            word_token: None,
+            reserved_word_sets: Vec::new(),
+            variables: vec![Variable {
+                name: "test".to_string(),
+                kind: VariableType::Named,
+                rule: Rule::seq(vec![
+                    Rule::non_terminal(0),
+                    Rule::non_terminal(1),
+                    Rule::non_terminal(2),
+                ]),
+            }],
+        });
+
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Rule `test` cannot be inlined because it contains a reference to itself.",
+        );
+    }
+}

cli/generate/src/prepare_grammar/intern_symbols.rs

@@ -0,0 +1,293 @@
+use anyhow::Result;
+use serde::Serialize;
+use thiserror::Error;
+
+use super::InternedGrammar;
+use crate::{
+    grammars::{InputGrammar, ReservedWordContext, Variable, VariableType},
+    rules::{Rule, Symbol},
+};
+
+pub type InternSymbolsResult<T> = Result<T, InternSymbolsError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum InternSymbolsError {
+    #[error("A grammar's start rule must be visible.")]
+    HiddenStartRule,
+    #[error("Undefined symbol `{0}`")]
+    Undefined(String),
+    #[error("Undefined symbol `{0}` in grammar's supertypes array")]
+    UndefinedSupertype(String),
+    #[error("Undefined symbol `{0}` in grammar's conflicts array")]
+    UndefinedConflict(String),
+    #[error("Undefined symbol `{0}` as grammar's word token")]
+    UndefinedWordToken(String),
+}
+
+pub(super) fn intern_symbols(grammar: &InputGrammar) -> InternSymbolsResult<InternedGrammar> {
+    let interner = Interner { grammar };
+
+    if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden {
+        Err(InternSymbolsError::HiddenStartRule)?;
+    }
+
+    let mut variables = Vec::with_capacity(grammar.variables.len());
+    for variable in &grammar.variables {
+        variables.push(Variable {
+            name: variable.name.clone(),
+            kind: variable_type_for_name(&variable.name),
+            rule: interner.intern_rule(&variable.rule, Some(&variable.name))?,
+        });
+    }
+
+    let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len());
+    for external_token in &grammar.external_tokens {
+        let rule = interner.intern_rule(external_token, None)?;
+        let (name, kind) = if let Rule::NamedSymbol(name) = external_token {
+            (name.clone(), variable_type_for_name(name))
+        } else {
+            (String::new(), VariableType::Anonymous)
+        };
+        external_tokens.push(Variable { name, kind, rule });
+    }
+
+    let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len());
+    for extra_token in &grammar.extra_symbols {
+        extra_symbols.push(interner.intern_rule(extra_token, None)?);
+    }
+
+    let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len());
+    for supertype_symbol_name in &grammar.supertype_symbols {
+        supertype_symbols.push(interner.intern_name(supertype_symbol_name).ok_or_else(|| {
+            InternSymbolsError::UndefinedSupertype(supertype_symbol_name.clone())
+        })?);
+    }
+
+    let mut reserved_words = Vec::with_capacity(grammar.reserved_words.len());
+    for reserved_word_set in &grammar.reserved_words {
+        let mut interned_set = Vec::new();
+        for rule in &reserved_word_set.reserved_words {
+            interned_set.push(interner.intern_rule(rule, None)?);
+        }
+        reserved_words.push(ReservedWordContext {
+            name: reserved_word_set.name.clone(),
+            reserved_words: interned_set,
+        });
+    }
+
+    let mut expected_conflicts = Vec::new();
+    for conflict in &grammar.expected_conflicts {
+        let mut interned_conflict = Vec::with_capacity(conflict.len());
+        for name in conflict {
+            interned_conflict.push(
+                interner
+                    .intern_name(name)
+                    .ok_or_else(|| InternSymbolsError::UndefinedConflict(name.clone()))?,
+            );
+        }
+        expected_conflicts.push(interned_conflict);
+    }
+
+    let mut variables_to_inline = Vec::new();
+    for name in &grammar.variables_to_inline {
+        if let Some(symbol) = interner.intern_name(name) {
+            variables_to_inline.push(symbol);
+        }
+    }
+
+    let mut word_token = None;
+    if let Some(name) = grammar.word_token.as_ref() {
+        word_token = Some(
+            interner
+                .intern_name(name)
+                .ok_or_else(|| InternSymbolsError::UndefinedWordToken(name.clone()))?,
+        );
+    }
+
+    for (i, variable) in variables.iter_mut().enumerate() {
+        if supertype_symbols.contains(&Symbol::non_terminal(i)) {
+            variable.kind = VariableType::Hidden;
+        }
+    }
+
+    Ok(InternedGrammar {
+        variables,
+        external_tokens,
+        extra_symbols,
+        expected_conflicts,
+        variables_to_inline,
+        supertype_symbols,
+        word_token,
+        precedence_orderings: grammar.precedence_orderings.clone(),
+        reserved_word_sets: reserved_words,
+    })
+}
+
+struct Interner<'a> {
+    grammar: &'a InputGrammar,
+}
+
+impl Interner<'_> {
+    fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> InternSymbolsResult<Rule> {
+        match rule {
+            Rule::Choice(elements) => {
+                self.check_single(elements, name);
+                let mut result = Vec::with_capacity(elements.len());
+                for element in elements {
+                    result.push(self.intern_rule(element, name)?);
+                }
+                Ok(Rule::Choice(result))
+            }
+            Rule::Seq(elements) => {
+                self.check_single(elements, name);
+                let mut result = Vec::with_capacity(elements.len());
+                for element in elements {
+                    result.push(self.intern_rule(element, name)?);
+                }
+                Ok(Rule::Seq(result))
+            }
+            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(self.intern_rule(content, name)?))),
+            Rule::Metadata { rule, params } => Ok(Rule::Metadata {
+                rule: Box::new(self.intern_rule(rule, name)?),
+                params: params.clone(),
+            }),
+            Rule::Reserved { rule, context_name } => Ok(Rule::Reserved {
+                rule: Box::new(self.intern_rule(rule, name)?),
+                context_name: context_name.clone(),
+            }),
+            Rule::NamedSymbol(name) => self.intern_name(name).map_or_else(
+                || Err(InternSymbolsError::Undefined(name.clone())),
+                |symbol| Ok(Rule::Symbol(symbol)),
+            ),
+            _ => Ok(rule.clone()),
+        }
+    }
+
+    fn intern_name(&self, symbol: &str) -> Option<Symbol> {
+        for (i, variable) in self.grammar.variables.iter().enumerate() {
+            if variable.name == symbol {
+                return Some(Symbol::non_terminal(i));
+            }
+        }
+
+        for (i, external_token) in self.grammar.external_tokens.iter().enumerate() {
+            if let Rule::NamedSymbol(name) = external_token {
+                if name == symbol {
+                    return Some(Symbol::external(i));
+                }
+            }
+        }
+
+        None
+    }
+
+    // In the case of a seq or choice rule of 1 element in a hidden rule, weird
+    // inconsistent behavior with queries can occur. So we should warn the user about it.
+    fn check_single(&self, elements: &[Rule], name: Option<&str>) {
+        if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) {
+            eprintln!(
+                "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.",
+                name.unwrap_or_default()
+            );
+        }
+    }
+}
+
+fn variable_type_for_name(name: &str) -> VariableType {
+    if name.starts_with('_') {
+        VariableType::Hidden
+    } else {
+        VariableType::Named
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_repeat_expansion() {
+        let grammar = intern_symbols(&build_grammar(vec![
+            Variable::named("x", Rule::choice(vec![Rule::named("y"), Rule::named("_z")])),
+            Variable::named("y", Rule::named("_z")),
+            Variable::named("_z", Rule::string("a")),
+        ]))
+        .unwrap();
+
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "x",
+                    Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),])
+                ),
+                Variable::named("y", Rule::non_terminal(2)),
+                Variable::hidden("_z", Rule::string("a")),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_interning_external_token_names() {
+        // Variable `y` is both an internal and an external token.
+        // Variable `z` is just an external token.
+        let mut input_grammar = build_grammar(vec![
+            Variable::named(
+                "w",
+                Rule::choice(vec![Rule::named("x"), Rule::named("y"), Rule::named("z")]),
+            ),
+            Variable::named("x", Rule::string("a")),
+            Variable::named("y", Rule::string("b")),
+        ]);
+        input_grammar
+            .external_tokens
+            .extend(vec![Rule::named("y"), Rule::named("z")]);
+
+        let grammar = intern_symbols(&input_grammar).unwrap();
+
+        // Variable `y` is referred to by its internal index.
+        // Variable `z` is referred to by its external index.
+        assert_eq!(
+            grammar.variables,
+            vec![
+                Variable::named(
+                    "w",
+                    Rule::choice(vec![
+                        Rule::non_terminal(1),
+                        Rule::non_terminal(2),
+                        Rule::external(1),
+                    ])
+                ),
+                Variable::named("x", Rule::string("a")),
+                Variable::named("y", Rule::string("b")),
+            ]
+        );
+
+        // The external token for `y` refers back to its internal index.
+        assert_eq!(
+            grammar.external_tokens,
+            vec![
+                Variable::named("y", Rule::non_terminal(2)),
+                Variable::named("z", Rule::external(1)),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_grammar_with_undefined_symbols() {
+        let result = intern_symbols(&build_grammar(vec![Variable::named("x", Rule::named("y"))]));
+
+        match result {
+            Err(e) => assert_eq!(e.to_string(), "Undefined symbol `y`"),
+            _ => panic!("Expected an error but got none"),
+        }
+    }
+
+    fn build_grammar(variables: Vec<Variable>) -> InputGrammar {
+        InputGrammar {
+            variables,
+            name: "the_language".to_string(),
+            ..Default::default()
+        }
+    }
+}

cli/generate/src/prepare_grammar/mod.rs

@@ -0,0 +1,325 @@
+mod expand_repeats;
+mod expand_tokens;
+mod extract_default_aliases;
+mod extract_tokens;
+mod flatten_grammar;
+mod intern_symbols;
+mod process_inlines;
+
+use std::{
+    cmp::Ordering,
+    collections::{hash_map, HashMap, HashSet},
+    mem,
+};
+
+use anyhow::Result;
+pub use expand_tokens::ExpandTokensError;
+pub use extract_tokens::ExtractTokensError;
+pub use flatten_grammar::FlattenGrammarError;
+pub use intern_symbols::InternSymbolsError;
+pub use process_inlines::ProcessInlinesError;
+use serde::Serialize;
+use thiserror::Error;
+
+pub use self::expand_tokens::expand_tokens;
+use self::{
+    expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
+    extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
+    intern_symbols::intern_symbols, process_inlines::process_inlines,
+};
+use super::{
+    grammars::{
+        ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
+        SyntaxGrammar, Variable,
+    },
+    rules::{AliasMap, Precedence, Rule, Symbol},
+};
+use crate::grammars::ReservedWordContext;
+
+pub struct IntermediateGrammar<T, U> {
+    variables: Vec<Variable>,
+    extra_symbols: Vec<T>,
+    expected_conflicts: Vec<Vec<Symbol>>,
+    precedence_orderings: Vec<Vec<PrecedenceEntry>>,
+    external_tokens: Vec<U>,
+    variables_to_inline: Vec<Symbol>,
+    supertype_symbols: Vec<Symbol>,
+    word_token: Option<Symbol>,
+    reserved_word_sets: Vec<ReservedWordContext<T>>,
+}
+
+pub type InternedGrammar = IntermediateGrammar<Rule, Variable>;
+
+pub type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct ExtractedLexicalGrammar {
+    pub variables: Vec<Variable>,
+    pub separators: Vec<Rule>,
+}
+
+impl<T, U> Default for IntermediateGrammar<T, U> {
+    fn default() -> Self {
+        Self {
+            variables: Vec::default(),
+            extra_symbols: Vec::default(),
+            expected_conflicts: Vec::default(),
+            precedence_orderings: Vec::default(),
+            external_tokens: Vec::default(),
+            variables_to_inline: Vec::default(),
+            supertype_symbols: Vec::default(),
+            word_token: Option::default(),
+            reserved_word_sets: Vec::default(),
+        }
+    }
+}
+
+pub type PrepareGrammarResult<T> = Result<T, PrepareGrammarError>;
+
+#[derive(Debug, Error, Serialize)]
+#[error(transparent)]
+pub enum PrepareGrammarError {
+    ValidatePrecedences(#[from] ValidatePrecedenceError),
+    InternSymbols(#[from] InternSymbolsError),
+    ExtractTokens(#[from] ExtractTokensError),
+    FlattenGrammar(#[from] FlattenGrammarError),
+    ExpandTokens(#[from] ExpandTokensError),
+    ProcessInlines(#[from] ProcessInlinesError),
+}
+
+pub type ValidatePrecedenceResult<T> = Result<T, ValidatePrecedenceError>;
+
+#[derive(Debug, Error, Serialize)]
+#[error(transparent)]
+pub enum ValidatePrecedenceError {
+    Undeclared(#[from] UndeclaredPrecedenceError),
+    Ordering(#[from] ConflictingPrecedenceOrderingError),
+}
+
+#[derive(Debug, Error, Serialize)]
+pub struct UndeclaredPrecedenceError {
+    pub precedence: String,
+    pub rule: String,
+}
+
+impl std::fmt::Display for UndeclaredPrecedenceError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Undeclared precedence '{}' in rule '{}'",
+            self.precedence, self.rule
+        )?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Error, Serialize)]
+pub struct ConflictingPrecedenceOrderingError {
+    pub precedence_1: String,
+    pub precedence_2: String,
+}
+
+impl std::fmt::Display for ConflictingPrecedenceOrderingError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Conflicting orderings for precedences {} and {}",
+            self.precedence_1, self.precedence_2
+        )?;
+        Ok(())
+    }
+}
+
+/// Transform an input grammar into separate components that are ready
+/// for parse table construction.
+pub fn prepare_grammar(
+    input_grammar: &InputGrammar,
+) -> PrepareGrammarResult<(
+    SyntaxGrammar,
+    LexicalGrammar,
+    InlinedProductionMap,
+    AliasMap,
+)> {
+    validate_precedences(input_grammar)?;
+
+    let interned_grammar = intern_symbols(input_grammar)?;
+    let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
+    let syntax_grammar = expand_repeats(syntax_grammar);
+    let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
+    let lexical_grammar = expand_tokens(lexical_grammar)?;
+    let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
+    let inlines = process_inlines(&syntax_grammar, &lexical_grammar)?;
+    Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
+}
+
+/// Check that all of the named precedences used in the grammar are declared
+/// within the `precedences` lists, and also that there are no conflicting
+/// precedence orderings declared in those lists.
+fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()> {
+    // Check that no rule contains a named precedence that is not present in
+    // any of the `precedences` lists.
+    fn validate(
+        rule_name: &str,
+        rule: &Rule,
+        names: &HashSet<&String>,
+    ) -> ValidatePrecedenceResult<()> {
+        match rule {
+            Rule::Repeat(rule) => validate(rule_name, rule, names),
+            Rule::Seq(elements) | Rule::Choice(elements) => elements
+                .iter()
+                .try_for_each(|e| validate(rule_name, e, names)),
+            Rule::Metadata { rule, params } => {
+                if let Precedence::Name(n) = &params.precedence {
+                    if !names.contains(n) {
+                        Err(UndeclaredPrecedenceError {
+                            precedence: n.to_string(),
+                            rule: rule_name.to_string(),
+                        })?;
+                    }
+                }
+                validate(rule_name, rule, names)?;
+                Ok(())
+            }
+            _ => Ok(()),
+        }
+    }
+
+    // For any two precedence names `a` and `b`, if `a` comes before `b`
+    // in some list, then it cannot come *after* `b` in any list.
+    let mut pairs = HashMap::new();
+    for list in &grammar.precedence_orderings {
+        for (i, mut entry1) in list.iter().enumerate() {
+            for mut entry2 in list.iter().skip(i + 1) {
+                if entry2 == entry1 {
+                    continue;
+                }
+                let mut ordering = Ordering::Greater;
+                if entry1 > entry2 {
+                    ordering = Ordering::Less;
+                    mem::swap(&mut entry1, &mut entry2);
+                }
+                match pairs.entry((entry1, entry2)) {
+                    hash_map::Entry::Vacant(e) => {
+                        e.insert(ordering);
+                    }
+                    hash_map::Entry::Occupied(e) => {
+                        if e.get() != &ordering {
+                            Err(ConflictingPrecedenceOrderingError {
+                                precedence_1: entry1.to_string(),
+                                precedence_2: entry2.to_string(),
+                            })?;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    let precedence_names = grammar
+        .precedence_orderings
+        .iter()
+        .flat_map(|l| l.iter())
+        .filter_map(|p| {
+            if let PrecedenceEntry::Name(n) = p {
+                Some(n)
+            } else {
+                None
+            }
+        })
+        .collect::<HashSet<&String>>();
+    for variable in &grammar.variables {
+        validate(&variable.name, &variable.rule, &precedence_names)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::grammars::VariableType;
+
+    #[test]
+    fn test_validate_precedences_with_undeclared_precedence() {
+        let grammar = InputGrammar {
+            precedence_orderings: vec![
+                vec![
+                    PrecedenceEntry::Name("a".to_string()),
+                    PrecedenceEntry::Name("b".to_string()),
+                ],
+                vec![
+                    PrecedenceEntry::Name("b".to_string()),
+                    PrecedenceEntry::Name("c".to_string()),
+                    PrecedenceEntry::Name("d".to_string()),
+                ],
+            ],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::Seq(vec![
+                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
+                    ]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::repeat(Rule::Choice(vec![
+                        Rule::prec_left(Precedence::Name("omg".to_string()), Rule::string("y")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
+                    ])),
+                },
+            ],
+            ..Default::default()
+        };
+
+        let result = validate_precedences(&grammar);
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Undeclared precedence 'omg' in rule 'v2'",
+        );
+    }
+
+    #[test]
+    fn test_validate_precedences_with_conflicting_order() {
+        let grammar = InputGrammar {
+            precedence_orderings: vec![
+                vec![
+                    PrecedenceEntry::Name("a".to_string()),
+                    PrecedenceEntry::Name("b".to_string()),
+                ],
+                vec![
+                    PrecedenceEntry::Name("b".to_string()),
+                    PrecedenceEntry::Name("c".to_string()),
+                    PrecedenceEntry::Name("a".to_string()),
+                ],
+            ],
+            variables: vec![
+                Variable {
+                    name: "v1".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::Seq(vec![
+                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
+                    ]),
+                },
+                Variable {
+                    name: "v2".to_string(),
+                    kind: VariableType::Named,
+                    rule: Rule::repeat(Rule::Choice(vec![
+                        Rule::prec_left(Precedence::Name("a".to_string()), Rule::string("y")),
+                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
+                    ])),
+                },
+            ],
+            ..Default::default()
+        };
+
+        let result = validate_precedences(&grammar);
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Conflicting orderings for precedences 'a' and 'b'",
+        );
+    }
+}

cli/generate/src/prepare_grammar/process_inlines.rs

@@ -0,0 +1,558 @@
+use std::collections::HashMap;
+
+use anyhow::Result;
+use serde::Serialize;
+use thiserror::Error;
+
+use crate::{
+    grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
+    rules::SymbolType,
+};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+struct ProductionStepId {
+    // A `None` value here means that the production itself was produced via inlining,
+    // and is stored in the builder's `productions` vector, as opposed to being
+    // stored in one of the grammar's variables.
+    variable_index: Option<usize>,
+    production_index: usize,
+    step_index: usize,
+}
+
+struct InlinedProductionMapBuilder {
+    production_indices_by_step_id: HashMap<ProductionStepId, Vec<usize>>,
+    productions: Vec<Production>,
+}
+
+impl InlinedProductionMapBuilder {
+    fn build(mut self, grammar: &SyntaxGrammar) -> InlinedProductionMap {
+        let mut step_ids_to_process = Vec::new();
+        for (variable_index, variable) in grammar.variables.iter().enumerate() {
+            for production_index in 0..variable.productions.len() {
+                step_ids_to_process.push(ProductionStepId {
+                    variable_index: Some(variable_index),
+                    production_index,
+                    step_index: 0,
+                });
+                while !step_ids_to_process.is_empty() {
+                    let mut i = 0;
+                    while i < step_ids_to_process.len() {
+                        let step_id = step_ids_to_process[i];
+                        if let Some(step) = self.production_step_for_id(step_id, grammar) {
+                            if grammar.variables_to_inline.contains(&step.symbol) {
+                                let inlined_step_ids = self
+                                    .inline_production_at_step(step_id, grammar)
+                                    .iter()
+                                    .copied()
+                                    .map(|production_index| ProductionStepId {
+                                        variable_index: None,
+                                        production_index,
+                                        step_index: step_id.step_index,
+                                    });
+                                step_ids_to_process.splice(i..=i, inlined_step_ids);
+                            } else {
+                                step_ids_to_process[i] = ProductionStepId {
+                                    variable_index: step_id.variable_index,
+                                    production_index: step_id.production_index,
+                                    step_index: step_id.step_index + 1,
+                                };
+                                i += 1;
+                            }
+                        } else {
+                            step_ids_to_process.remove(i);
+                        }
+                    }
+                }
+            }
+        }
+
+        let productions = self.productions;
+        let production_indices_by_step_id = self.production_indices_by_step_id;
+        let production_map = production_indices_by_step_id
+            .into_iter()
+            .map(|(step_id, production_indices)| {
+                let production = step_id.variable_index.map_or_else(
+                    || &productions[step_id.production_index],
+                    |variable_index| {
+                        &grammar.variables[variable_index].productions[step_id.production_index]
+                    },
+                ) as *const Production;
+                ((production, step_id.step_index as u32), production_indices)
+            })
+            .collect();
+
+        InlinedProductionMap {
+            productions,
+            production_map,
+        }
+    }
+
+    fn inline_production_at_step<'a>(
+        &'a mut self,
+        step_id: ProductionStepId,
+        grammar: &'a SyntaxGrammar,
+    ) -> &'a [usize] {
+        // Build a list of productions produced by inlining rules.
+        let mut i = 0;
+        let step_index = step_id.step_index;
+        let mut productions_to_add = vec![self.production_for_id(step_id, grammar).clone()];
+        while i < productions_to_add.len() {
+            if let Some(step) = productions_to_add[i].steps.get(step_index) {
+                let symbol = step.symbol;
+                if grammar.variables_to_inline.contains(&symbol) {
+                    // Remove the production from the vector, replacing it with a placeholder.
+                    let production = productions_to_add
+                        .splice(i..=i, std::iter::once(&Production::default()).cloned())
+                        .next()
+                        .unwrap();
+
+                    // Replace the placeholder with the inlined productions.
+                    productions_to_add.splice(
+                        i..=i,
+                        grammar.variables[symbol.index].productions.iter().map(|p| {
+                            let mut production = production.clone();
+                            let removed_step = production
+                                .steps
+                                .splice(step_index..=step_index, p.steps.iter().cloned())
+                                .next()
+                                .unwrap();
+                            let inserted_steps =
+                                &mut production.steps[step_index..(step_index + p.steps.len())];
+                            if let Some(alias) = removed_step.alias {
+                                for inserted_step in inserted_steps.iter_mut() {
+                                    inserted_step.alias = Some(alias.clone());
+                                }
+                            }
+                            if let Some(field_name) = removed_step.field_name {
+                                for inserted_step in inserted_steps.iter_mut() {
+                                    inserted_step.field_name = Some(field_name.clone());
+                                }
+                            }
+                            if let Some(last_inserted_step) = inserted_steps.last_mut() {
+                                if last_inserted_step.precedence.is_none() {
+                                    last_inserted_step.precedence = removed_step.precedence;
+                                }
+                                if last_inserted_step.associativity.is_none() {
+                                    last_inserted_step.associativity = removed_step.associativity;
+                                }
+                            }
+                            if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() {
+                                production.dynamic_precedence = p.dynamic_precedence;
+                            }
+                            production
+                        }),
+                    );
+
+                    continue;
+                }
+            }
+            i += 1;
+        }
+
+        // Store all the computed productions.
+        let result = productions_to_add
+            .into_iter()
+            .map(|production| {
+                self.productions
+                    .iter()
+                    .position(|p| *p == production)
+                    .unwrap_or_else(|| {
+                        self.productions.push(production);
+                        self.productions.len() - 1
+                    })
+            })
+            .collect();
+
+        // Cache these productions based on the original production step.
+        self.production_indices_by_step_id
+            .entry(step_id)
+            .or_insert(result)
+    }
+
+    fn production_for_id<'a>(
+        &'a self,
+        id: ProductionStepId,
+        grammar: &'a SyntaxGrammar,
+    ) -> &'a Production {
+        id.variable_index.map_or_else(
+            || &self.productions[id.production_index],
+            |variable_index| &grammar.variables[variable_index].productions[id.production_index],
+        )
+    }
+
+    fn production_step_for_id<'a>(
+        &'a self,
+        id: ProductionStepId,
+        grammar: &'a SyntaxGrammar,
+    ) -> Option<&'a ProductionStep> {
+        self.production_for_id(id, grammar).steps.get(id.step_index)
+    }
+}
+
+pub type ProcessInlinesResult<T> = Result<T, ProcessInlinesError>;
+
+#[derive(Debug, Error, Serialize)]
+pub enum ProcessInlinesError {
+    #[error("External token `{0}` cannot be inlined")]
+    ExternalToken(String),
+    #[error("Token `{0}` cannot be inlined")]
+    Token(String),
+    #[error("Rule `{0}` cannot be inlined because it is the first rule")]
+    FirstRule(String),
+}
+
+pub(super) fn process_inlines(
+    grammar: &SyntaxGrammar,
+    lexical_grammar: &LexicalGrammar,
+) -> ProcessInlinesResult<InlinedProductionMap> {
+    for symbol in &grammar.variables_to_inline {
+        match symbol.kind {
+            SymbolType::External => {
+                Err(ProcessInlinesError::ExternalToken(
+                    grammar.external_tokens[symbol.index].name.clone(),
+                ))?;
+            }
+            SymbolType::Terminal => {
+                Err(ProcessInlinesError::Token(
+                    lexical_grammar.variables[symbol.index].name.clone(),
+                ))?;
+            }
+            SymbolType::NonTerminal if symbol.index == 0 => {
+                Err(ProcessInlinesError::FirstRule(
+                    grammar.variables[symbol.index].name.clone(),
+                ))?;
+            }
+            _ => {}
+        }
+    }
+
+    Ok(InlinedProductionMapBuilder {
+        productions: Vec::new(),
+        production_indices_by_step_id: HashMap::new(),
+    }
+    .build(grammar))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        grammars::{LexicalVariable, SyntaxVariable, VariableType},
+        rules::{Associativity, Precedence, Symbol},
+    };
+
+    #[test]
+    fn test_basic_inlining() {
+        let grammar = SyntaxGrammar {
+            variables_to_inline: vec![Symbol::non_terminal(1)],
+            variables: vec![
+                SyntaxVariable {
+                    name: "non-terminal-0".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(10)),
+                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
+                            ProductionStep::new(Symbol::terminal(11)),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-1".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![
+                        Production {
+                            dynamic_precedence: 0,
+                            steps: vec![
+                                ProductionStep::new(Symbol::terminal(12)),
+                                ProductionStep::new(Symbol::terminal(13)),
+                            ],
+                        },
+                        Production {
+                            dynamic_precedence: -2,
+                            steps: vec![ProductionStep::new(Symbol::terminal(14))],
+                        },
+                    ],
+                },
+            ],
+            ..Default::default()
+        };
+
+        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
+
+        // Nothing to inline at step 0.
+        assert!(inline_map
+            .inlined_productions(&grammar.variables[0].productions[0], 0)
+            .is_none());
+
+        // Inlining variable 1 yields two productions.
+        assert_eq!(
+            inline_map
+                .inlined_productions(&grammar.variables[0].productions[0], 1)
+                .unwrap()
+                .cloned()
+                .collect::<Vec<_>>(),
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(12)),
+                        ProductionStep::new(Symbol::terminal(13)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                    ],
+                },
+                Production {
+                    dynamic_precedence: -2,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(14)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                    ],
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn test_nested_inlining() {
+        let grammar = SyntaxGrammar {
+            variables: vec![
+                SyntaxVariable {
+                    name: "non-terminal-0".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(10)),
+                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
+                            ProductionStep::new(Symbol::terminal(11)),
+                            ProductionStep::new(Symbol::non_terminal(2)), // inlined
+                            ProductionStep::new(Symbol::terminal(12)),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-1".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![
+                        Production {
+                            dynamic_precedence: 0,
+                            steps: vec![ProductionStep::new(Symbol::terminal(13))],
+                        },
+                        Production {
+                            dynamic_precedence: 0,
+                            steps: vec![
+                                ProductionStep::new(Symbol::non_terminal(3)), // inlined
+                                ProductionStep::new(Symbol::terminal(14)),
+                            ],
+                        },
+                    ],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-2".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![ProductionStep::new(Symbol::terminal(15))],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-3".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![ProductionStep::new(Symbol::terminal(16))],
+                    }],
+                },
+            ],
+            variables_to_inline: vec![
+                Symbol::non_terminal(1),
+                Symbol::non_terminal(2),
+                Symbol::non_terminal(3),
+            ],
+            ..Default::default()
+        };
+
+        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
+
+        let productions = inline_map
+            .inlined_productions(&grammar.variables[0].productions[0], 1)
+            .unwrap()
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            productions.iter().copied().cloned().collect::<Vec<_>>(),
+            vec![
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(13)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::terminal(12)),
+                    ],
+                },
+                Production {
+                    dynamic_precedence: 0,
+                    steps: vec![
+                        ProductionStep::new(Symbol::terminal(10)),
+                        ProductionStep::new(Symbol::terminal(16)),
+                        ProductionStep::new(Symbol::terminal(14)),
+                        ProductionStep::new(Symbol::terminal(11)),
+                        ProductionStep::new(Symbol::non_terminal(2)),
+                        ProductionStep::new(Symbol::terminal(12)),
+                    ],
+                },
+            ]
+        );
+
+        assert_eq!(
+            inline_map
+                .inlined_productions(productions[0], 3)
+                .unwrap()
+                .cloned()
+                .collect::<Vec<_>>(),
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    ProductionStep::new(Symbol::terminal(10)),
+                    ProductionStep::new(Symbol::terminal(13)),
+                    ProductionStep::new(Symbol::terminal(11)),
+                    ProductionStep::new(Symbol::terminal(15)),
+                    ProductionStep::new(Symbol::terminal(12)),
+                ],
+            },]
+        );
+    }
+
+    #[test]
+    fn test_inlining_with_precedence_and_alias() {
+        let grammar = SyntaxGrammar {
+            variables_to_inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
+            variables: vec![
+                SyntaxVariable {
+                    name: "non-terminal-0".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            // inlined
+                            ProductionStep::new(Symbol::non_terminal(1))
+                                .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
+                            ProductionStep::new(Symbol::terminal(10)),
+                            // inlined
+                            ProductionStep::new(Symbol::non_terminal(2))
+                                .with_alias("outer_alias", true),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-1".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![
+                            ProductionStep::new(Symbol::terminal(11))
+                                .with_prec(Precedence::Integer(2), None)
+                                .with_alias("inner_alias", true),
+                            ProductionStep::new(Symbol::terminal(12)),
+                        ],
+                    }],
+                },
+                SyntaxVariable {
+                    name: "non-terminal-2".to_string(),
+                    kind: VariableType::Named,
+                    productions: vec![Production {
+                        dynamic_precedence: 0,
+                        steps: vec![ProductionStep::new(Symbol::terminal(13))],
+                    }],
+                },
+            ],
+            ..Default::default()
+        };
+
+        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
+
+        let productions = inline_map
+            .inlined_productions(&grammar.variables[0].productions[0], 0)
+            .unwrap()
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            productions.iter().copied().cloned().collect::<Vec<_>>(),
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    // The first step in the inlined production retains its precedence
+                    // and alias.
+                    ProductionStep::new(Symbol::terminal(11))
+                        .with_prec(Precedence::Integer(2), None)
+                        .with_alias("inner_alias", true),
+                    // The final step of the inlined production inherits the precedence of
+                    // the inlined step.
+                    ProductionStep::new(Symbol::terminal(12))
+                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
+                    ProductionStep::new(Symbol::terminal(10)),
+                    ProductionStep::new(Symbol::non_terminal(2)).with_alias("outer_alias", true),
+                ]
+            }],
+        );
+
+        assert_eq!(
+            inline_map
+                .inlined_productions(productions[0], 3)
+                .unwrap()
+                .cloned()
+                .collect::<Vec<_>>(),
+            vec![Production {
+                dynamic_precedence: 0,
+                steps: vec![
+                    ProductionStep::new(Symbol::terminal(11))
+                        .with_prec(Precedence::Integer(2), None)
+                        .with_alias("inner_alias", true),
+                    ProductionStep::new(Symbol::terminal(12))
+                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
+                    ProductionStep::new(Symbol::terminal(10)),
+                    // All steps of the inlined production inherit their alias from the
+                    // inlined step.
+                    ProductionStep::new(Symbol::terminal(13)).with_alias("outer_alias", true),
+                ]
+            }],
+        );
+    }
+
+    #[test]
+    fn test_error_when_inlining_tokens() {
+        let lexical_grammar = LexicalGrammar {
+            variables: vec![LexicalVariable {
+                name: "something".to_string(),
+                kind: VariableType::Named,
+                implicit_precedence: 0,
+                start_state: 0,
+            }],
+            ..Default::default()
+        };
+
+        let grammar = SyntaxGrammar {
+            variables_to_inline: vec![Symbol::terminal(0)],
+            variables: vec![SyntaxVariable {
+                name: "non-terminal-0".to_string(),
+                kind: VariableType::Named,
+                productions: vec![Production {
+                    dynamic_precedence: 0,
+                    steps: vec![ProductionStep::new(Symbol::terminal(0))],
+                }],
+            }],
+            ..Default::default()
+        };
+
+        if let Err(error) = process_inlines(&grammar, &lexical_grammar) {
+            assert_eq!(error.to_string(), "Token `something` cannot be inlined");
+        } else {
+            panic!("expected an error, but got none");
+        }
+    }
+}

cli/generate/src/render.rs

@@ -1,49 +1,31 @@
 use std::{
     cmp,
-    collections::{BTreeMap, BTreeSet},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     fmt::Write,
     mem::swap,
 };
 
-use rustc_hash::{FxHashMap, FxHashSet};
-
-use serde::{Deserialize, Serialize};
-use thiserror::Error;
-
-use crate::tables::{ActionListId, ActionListPool};
+use indoc::indoc;
 
 use super::{
-    LANGUAGE_VERSION,
     build_tables::Tables,
-    grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
+    grammars::{ExternalToken, LexicalGrammar, SyntaxGrammar, VariableType},
     nfa::CharacterSet,
     node_types::ChildType,
-    rules::Alias,
-    rules::{AliasMap, Symbol, SymbolType, TokenSet},
-    strpool::{StrId, StrPool},
+    rules::{Alias, AliasMap, Symbol, SymbolType, TokenSet},
     tables::{
         AdvanceAction, FieldLocation, GotoAction, LexState, LexTable, ParseAction, ParseTable,
+        ParseTableEntry,
     },
 };
 
 const SMALL_STATE_THRESHOLD: usize = 64;
 pub const ABI_VERSION_MIN: usize = 14;
-pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION;
+pub const ABI_VERSION_MAX: usize = tree_sitter::LANGUAGE_VERSION;
 const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
+const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION");
+const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA");
 
-pub type RenderResult<T> = Result<T, RenderError>;
-
-#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
-pub enum RenderError {
-    #[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
-    ParseTable(usize),
-    #[error(
-        "This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {0}"
-    )]
-    ABI(usize),
-}
-
-#[clippy::format_args]
 macro_rules! add {
     ($this: tt, $($arg: tt)*) => {{
         $this.buffer.write_fmt(format_args!($($arg)*)).unwrap();
@@ -58,7 +40,6 @@ macro_rules! add_whitespace {
     }};
 }
 
-#[clippy::format_args]
 macro_rules! add_line {
     ($this: tt, $($arg: tt)*) => {
         add_whitespace!($this);
@@ -94,19 +75,18 @@ struct Generator {
     syntax_grammar: SyntaxGrammar,
     lexical_grammar: LexicalGrammar,
     default_aliases: AliasMap,
-    symbol_order: FxHashMap<Symbol, usize>,
-    symbol_ids: FxHashMap<Symbol, String>,
-    alias_ids: FxHashMap<Alias, String>,
+    symbol_order: HashMap<Symbol, usize>,
+    symbol_ids: HashMap<Symbol, String>,
+    alias_ids: HashMap<Alias, String>,
     unique_aliases: Vec<Alias>,
-    symbol_map: FxHashMap<Symbol, Symbol>,
+    symbol_map: HashMap<Symbol, Symbol>,
     reserved_word_sets: Vec<TokenSet>,
     reserved_word_set_ids_by_parse_state: Vec<usize>,
-    field_names: Vec<StrId>,
+    field_names: Vec<String>,
     supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
     supertype_map: BTreeMap<String, Vec<ChildType>>,
     abi_version: usize,
     metadata: Option<Metadata>,
-    str_pool: StrPool,
 }
 
 struct LargeCharacterSetInfo {
@@ -114,15 +94,14 @@ struct LargeCharacterSetInfo {
     is_used: bool,
 }
 
-#[derive(Clone, Copy, Default)]
 struct Metadata {
-    major: u8,
-    minor: u8,
-    patch: u8,
+    major_version: u8,
+    minor_version: u8,
+    patch_version: u8,
 }
 
 impl Generator {
-    fn generate(mut self) -> RenderResult<String> {
+    fn generate(mut self) -> String {
         self.init();
         self.add_header();
         self.add_includes();
@@ -179,7 +158,7 @@ impl Generator {
             self.add_reserved_word_sets();
         }
 
-        self.add_parse_table()?;
+        self.add_parse_table();
 
         if !self.syntax_grammar.external_tokens.is_empty() {
             self.add_external_token_enum();
@@ -189,11 +168,11 @@ impl Generator {
 
         self.add_parser_export();
 
-        Ok(self.buffer)
+        self.buffer
     }
 
     fn init(&mut self) {
-        let mut symbol_identifiers = FxHashSet::default();
+        let mut symbol_identifiers = HashSet::new();
         for i in 0..self.parse_table.symbols.len() {
             self.assign_symbol_id(self.parse_table.symbols[i], &mut symbol_identifiers);
         }
@@ -202,7 +181,7 @@ impl Generator {
             self.symbol_ids[&Symbol::end()].clone(),
         );
 
-        self.symbol_map = FxHashMap::default();
+        self.symbol_map = HashMap::new();
 
         for symbol in &self.parse_table.symbols {
             let mut mapping = symbol;
@@ -219,7 +198,7 @@ impl Generator {
                         if other_symbol < mapping && other_alias == alias {
                             mapping = other_symbol;
                         }
-                    } else if self.metadata_for_symbol(*other_symbol) == (alias.value, kind) {
+                    } else if self.metadata_for_symbol(*other_symbol) == (&alias.value, kind) {
                         mapping = other_symbol;
                         break;
                     }
@@ -234,10 +213,10 @@ impl Generator {
                 for other_symbol in &self.parse_table.symbols {
                     let other_metadata = self.metadata_for_symbol(*other_symbol);
                     if other_metadata == metadata {
-                        if let Some(mapped) = self.symbol_map.get(other_symbol)
-                            && mapped == symbol
-                        {
-                            break;
+                        if let Some(mapped) = self.symbol_map.get(other_symbol) {
+                            if mapped == symbol {
+                                break;
+                            }
                         }
                         mapping = other_symbol;
                         break;
@@ -250,43 +229,36 @@ impl Generator {
 
         for production_info in &self.parse_table.production_infos {
             // Build a list of all field names
-            for &field_name in production_info.field_map.keys() {
-                if let Err(i) = self.field_names.binary_search_by(|&sid| {
-                    self.str_pool
-                        .resolve(sid)
-                        .cmp(self.str_pool.resolve(field_name))
-                }) {
-                    self.field_names.insert(i, field_name);
+            for field_name in production_info.field_map.keys() {
+                if let Err(i) = self.field_names.binary_search(field_name) {
+                    self.field_names.insert(i, field_name.clone());
                 }
             }
 
-            // Generate a mapping from aliases to C identifiers.
-            for &alias in production_info.alias_sequence.iter().flatten() {
-                // Some aliases match an existing symbol in the grammar.
-                let alias_id = if let Some(existing_symbol) = self.symbols_for_alias(alias).first()
-                {
-                    self.symbol_ids[&self.symbol_map[existing_symbol]].clone()
+            for alias in &production_info.alias_sequence {
+                // Generate a mapping from aliases to C identifiers.
+                if let Some(alias) = &alias {
+                    // Some aliases match an existing symbol in the grammar.
+                    let alias_id =
+                        if let Some(existing_symbol) = self.symbols_for_alias(alias).first() {
+                            self.symbol_ids[&self.symbol_map[existing_symbol]].clone()
+                        }
+                        // Other aliases don't match any existing symbol, and need their own
+                        // identifiers.
+                        else {
+                            if let Err(i) = self.unique_aliases.binary_search(alias) {
+                                self.unique_aliases.insert(i, alias.clone());
+                            }
+
+                            if alias.is_named {
+                                format!("alias_sym_{}", self.sanitize_identifier(&alias.value))
+                            } else {
+                                format!("anon_alias_sym_{}", self.sanitize_identifier(&alias.value))
+                            }
+                        };
+
+                    self.alias_ids.entry(alias.clone()).or_insert(alias_id);
                 }
-                // Other aliases don't match any existing symbol, and need their own
-                // identifiers.
-                else {
-                    if let Err(i) = self.unique_aliases.binary_search_by(|candidate| {
-                        self.str_pool
-                            .resolve(candidate.value)
-                            .cmp(self.str_pool.resolve(alias.value))
-                            .then_with(|| candidate.is_named.cmp(&alias.is_named))
-                    }) {
-                        self.unique_aliases.insert(i, alias);
-                    }
-
-                    if alias.is_named {
-                        format!("alias_sym_{}", self.sanitize_identifier(alias.value))
-                    } else {
-                        format!("anon_alias_sym_{}", self.sanitize_identifier(alias.value))
-                    }
-                };
-
-                self.alias_ids.entry(alias).or_insert(alias_id);
             }
         }
 
@@ -350,7 +322,14 @@ impl Generator {
     }
 
     fn add_header(&mut self) {
-        add_line!(self, "/* Automatically @generated by tree-sitter */");
+        let version = BUILD_SHA.map_or_else(
+            || BUILD_VERSION.to_string(),
+            |build_sha| format!("{BUILD_VERSION} ({build_sha})"),
+        );
+        add_line!(
+            self,
+            "/* Automatically generated by tree-sitter v{version} */",
+        );
         add_line!(self, "");
     }
 
@@ -392,7 +371,7 @@ impl Generator {
                 if symbol.is_terminal() || symbol.is_eof() {
                     true
                 } else if symbol.is_external() {
-                    self.syntax_grammar.external_tokens[symbol.index as usize]
+                    self.syntax_grammar.external_tokens[symbol.index]
                         .corresponding_internal_token
                         .is_none()
                 } else {
@@ -474,7 +453,9 @@ impl Generator {
             let name = self.sanitize_string(
                 self.default_aliases
                     .get(symbol)
-                    .map_or_else(|| self.metadata_for_symbol(*symbol).0, |alias| alias.value),
+                    .map_or(self.metadata_for_symbol(*symbol).0, |alias| {
+                        alias.value.as_str()
+                    }),
             );
             add_line!(self, "[{}] = \"{name}\",", self.symbol_ids[symbol]);
         }
@@ -483,7 +464,7 @@ impl Generator {
                 self,
                 "[{}] = \"{}\",",
                 self.alias_ids[alias],
-                self.sanitize_string(alias.value)
+                self.sanitize_string(&alias.value)
             );
         }
         dedent!(self);
@@ -520,13 +501,8 @@ impl Generator {
     fn add_field_name_enum(&mut self) {
         add_line!(self, "enum ts_field_identifiers {{");
         indent!(self);
-        for (i, &field_name) in self.field_names.iter().enumerate() {
-            add_line!(
-                self,
-                "{} = {},",
-                Self::field_id(self.str_pool.resolve(field_name)),
-                i + 1
-            );
+        for (i, field_name) in self.field_names.iter().enumerate() {
+            add_line!(self, "{} = {},", self.field_id(field_name), i + 1);
         }
         dedent!(self);
         add_line!(self, "}};");
@@ -537,9 +513,8 @@ impl Generator {
         add_line!(self, "static const char * const ts_field_names[] = {{");
         indent!(self);
         add_line!(self, "[0] = NULL,");
-        for &field_name in &self.field_names {
-            let field_name = self.str_pool.resolve(field_name);
-            add_line!(self, "[{}] = \"{field_name}\",", Self::field_id(field_name));
+        for field_name in &self.field_names {
+            add_line!(self, "[{}] = \"{field_name}\",", self.field_id(field_name));
         }
         dedent!(self);
         add_line!(self, "}};");
@@ -629,21 +604,22 @@ impl Generator {
     }
 
     fn add_non_terminal_alias_map(&mut self) {
-        let mut alias_ids_by_symbol = FxHashMap::default();
-        for i in 0..self.syntax_grammar.variables.len() {
-            for prod_id in self.syntax_grammar.variable_prod_ids(i) {
-                for step in self.syntax_grammar.production(prod_id).steps {
-                    if let Some(alias) = step.alias()
-                        && step.symbol().is_non_terminal()
-                        && Some(alias) != self.default_aliases.get(&step.symbol()).copied()
-                        && self.symbol_ids.contains_key(&step.symbol())
-                        && let Some(alias_id) = self.alias_ids.get(&alias)
-                    {
-                        let alias_ids = alias_ids_by_symbol
-                            .entry(step.symbol())
-                            .or_insert(Vec::new());
-                        if let Err(i) = alias_ids.binary_search(&alias_id) {
-                            alias_ids.insert(i, alias_id);
+        let mut alias_ids_by_symbol = HashMap::new();
+        for variable in &self.syntax_grammar.variables {
+            for production in &variable.productions {
+                for step in &production.steps {
+                    if let Some(alias) = &step.alias {
+                        if step.symbol.is_non_terminal()
+                            && Some(alias) != self.default_aliases.get(&step.symbol)
+                            && self.symbol_ids.contains_key(&step.symbol)
+                        {
+                            if let Some(alias_id) = self.alias_ids.get(alias) {
+                                let alias_ids =
+                                    alias_ids_by_symbol.entry(step.symbol).or_insert(Vec::new());
+                                if let Err(i) = alias_ids.binary_search(&alias_id) {
+                                    alias_ids.insert(i, alias_id);
+                                }
+                            }
                         }
                     }
                 }
@@ -686,7 +662,7 @@ impl Generator {
             "static const TSStateId ts_primary_state_ids[STATE_COUNT] = {{"
         );
         indent!(self);
-        let mut first_state_for_each_core_id = FxHashMap::default();
+        let mut first_state_for_each_core_id = HashMap::new();
         for (idx, state) in self.parse_table.states.iter().enumerate() {
             let primary_state = first_state_for_each_core_id
                 .entry(state.core_id)
@@ -701,34 +677,30 @@ impl Generator {
     fn add_field_sequences(&mut self) {
         let mut flat_field_maps = vec![];
         let mut next_flat_field_map_index = 0;
-        Self::get_field_map_id(
+        self.get_field_map_id(
             Vec::new(),
             &mut flat_field_maps,
             &mut next_flat_field_map_index,
         );
 
-        let mut field_map_ids = Vec::with_capacity(self.parse_table.production_infos.len());
+        let mut field_map_ids = Vec::new();
         for production_info in &self.parse_table.production_infos {
             if production_info.field_map.is_empty() {
                 field_map_ids.push((0, 0));
             } else {
-                let mut flat_field_map = Vec::with_capacity(production_info.field_map.len());
+                let mut flat_field_map = Vec::new();
                 for (field_name, locations) in &production_info.field_map {
                     for location in locations {
-                        flat_field_map.push((*field_name, *location));
+                        flat_field_map.push((field_name.clone(), *location));
                     }
                 }
-                flat_field_map.sort_by(|(a, _), (b, _)| {
-                    self.str_pool.resolve(*a).cmp(self.str_pool.resolve(*b))
-                });
-                let field_map_len = flat_field_map.len();
                 field_map_ids.push((
-                    Self::get_field_map_id(
-                        flat_field_map,
+                    self.get_field_map_id(
+                        flat_field_map.clone(),
                         &mut flat_field_maps,
                         &mut next_flat_field_map_index,
                     ),
-                    field_map_len,
+                    flat_field_map.len(),
                 ));
             }
         }
@@ -760,12 +732,7 @@ impl Generator {
             indent!(self);
             for (field_name, location) in field_pairs {
                 add_whitespace!(self);
-                add!(
-                    self,
-                    "{{{}, {}",
-                    Self::field_id(self.str_pool.resolve(field_name)),
-                    location.index
-                );
+                add!(self, "{{{}, {}", self.field_id(&field_name), location.index);
                 if location.inherited {
                     add!(self, ", .inherited = true");
                 }
@@ -809,7 +776,7 @@ impl Generator {
                         ChildType::Aliased(alias) => {
                             self.alias_ids.get(alias).cloned().map_or_else(
                                 || {
-                                    self.symbols_for_alias(*alias)
+                                    self.symbols_for_alias(alias)
                                         .into_iter()
                                         .map(|s| self.symbol_ids.get(&s).cloned())
                                         .collect()
@@ -960,7 +927,7 @@ impl Generator {
 
             // For large character sets, find the best matching character set from
             // a pre-selected list of large character sets, which are based on the
-            // state transitions for individual tokens. This transition may not exactly
+            // state transitions for invidual tokens. This transition may not exactly
             // match one of the pre-selected character sets. In that case, determine
             // the additional checks that need to be performed to match this transition.
             let mut best_large_char_set: Option<(usize, CharacterSet, CharacterSet)> = None;
@@ -1001,7 +968,10 @@ impl Generator {
                 large_char_set_ix = Some(char_set_ix);
             }
 
-            let line_break = format!("\n{}", "  ".repeat(self.indent_level + 2));
+            let mut line_break = "\n".to_string();
+            for _ in 0..self.indent_level + 2 {
+                line_break.push_str("  ");
+            }
 
             let has_positive_condition = large_char_set_ix.is_some() || !asserted_chars.is_empty();
             let has_negative_condition = !negated_chars.is_empty();
@@ -1041,7 +1011,7 @@ impl Generator {
                     add!(self, " ||{line_break}");
                 }
 
-                // If the character set contains the max character, then it probably
+                // If the character set contains the max character, than it probably
                 // corresponds to a negated character class in a regex, so it will be more
                 // concise and readable to express it in terms of negated ranges.
                 let is_included = !asserted_chars.contains(char::MAX);
@@ -1141,11 +1111,7 @@ impl Generator {
             return;
         }
 
-        add_line!(
-            self,
-            "static const TSCharacterRange {}[] = {{",
-            info.constant_name
-        );
+        add_line!(self, "const TSCharacterRange {}[] = {{", info.constant_name);
 
         indent!(self);
         for (ix, range) in characters.ranges().enumerate() {
@@ -1248,7 +1214,11 @@ impl Generator {
         add_line!(self, "enum ts_external_scanner_symbol_identifiers {{");
         indent!(self);
         for i in 0..self.syntax_grammar.external_tokens.len() {
-            add_line!(self, "{} = {i},", self.external_token_id(i));
+            add_line!(
+                self,
+                "{} = {i},",
+                self.external_token_id(&self.syntax_grammar.external_tokens[i]),
+            );
         }
         dedent!(self);
         add_line!(self, "}};");
@@ -1269,7 +1239,7 @@ impl Generator {
             add_line!(
                 self,
                 "[{}] = {},",
-                self.external_token_id(i),
+                self.external_token_id(token),
                 self.symbol_ids[&id_token],
             );
         }
@@ -1293,7 +1263,7 @@ impl Generator {
                     add_line!(
                         self,
                         "[{}] = true,",
-                        self.external_token_id(token.index as usize)
+                        self.external_token_id(&self.syntax_grammar.external_tokens[token.index])
                     );
                 }
                 dedent!(self);
@@ -1305,15 +1275,16 @@ impl Generator {
         add_line!(self, "");
     }
 
-    fn add_parse_table(&mut self) -> RenderResult<()> {
-        let mut parse_table_entries = FxHashMap::default();
-        let mut next_parse_action_list_index = 0u32;
+    fn add_parse_table(&mut self) {
+        let mut parse_table_entries = HashMap::new();
+        let mut next_parse_action_list_index = 0;
 
         // Parse action lists zero is for the default value, when a symbol is not valid.
-        // `canonicalize` guarantees pool index 0 is the empty list.
-        Self::get_parse_action_list_id(
-            ActionListId::new(0, false),
-            &self.parse_table.action_lists,
+        self.get_parse_action_list_id(
+            &ParseTableEntry {
+                actions: Vec::new(),
+                reusable: false,
+            },
             &mut parse_table_entries,
             &mut next_parse_action_list_index,
         );
@@ -1352,16 +1323,15 @@ impl Generator {
                     "[{}] = STATE({}),",
                     self.symbol_ids[symbol],
                     match action {
-                        GotoAction::Goto(state) => *state as usize,
+                        GotoAction::Goto(state) => *state,
                         GotoAction::ShiftExtra => i,
                     }
                 );
             }
 
-            for (symbol, id) in &terminal_entries {
-                let entry_id = Self::get_parse_action_list_id(
-                    **id,
-                    &self.parse_table.action_lists,
+            for (symbol, entry) in &terminal_entries {
+                let entry_id = self.get_parse_action_list_id(
+                    entry,
                     &mut parse_table_entries,
                     &mut next_parse_action_list_index,
                 );
@@ -1381,13 +1351,8 @@ impl Generator {
             indent!(self);
 
             let mut next_table_index = 0;
-            let mut small_state_indices = Vec::with_capacity(
-                self.parse_table
-                    .states
-                    .len()
-                    .saturating_sub(self.large_state_count),
-            );
-            let mut symbols_by_value = FxHashMap::<(u32, SymbolType), Vec<Symbol>>::default();
+            let mut small_state_indices = Vec::new();
+            let mut symbols_by_value = HashMap::<(usize, SymbolType), Vec<Symbol>>::new();
             for state in self.parse_table.states.iter().skip(self.large_state_count) {
                 small_state_indices.push(next_table_index);
                 symbols_by_value.clear();
@@ -1400,9 +1365,8 @@ impl Generator {
                 // So in the "small state" representation, group symbols by their action
                 // in order to avoid repeating the action.
                 for (symbol, entry) in &terminal_entries {
-                    let entry_id = Self::get_parse_action_list_id(
-                        **entry,
-                        &self.parse_table.action_lists,
+                    let entry_id = self.get_parse_action_list_id(
+                        entry,
                         &mut parse_table_entries,
                         &mut next_parse_action_list_index,
                     );
@@ -1415,7 +1379,7 @@ impl Generator {
                     let state_id = match action {
                         GotoAction::Goto(i) => *i,
                         GotoAction::ShiftExtra => {
-                            (self.large_state_count + small_state_indices.len() - 1) as u32
+                            self.large_state_count + small_state_indices.len() - 1
                         }
                     };
                     symbols_by_value
@@ -1476,39 +1440,31 @@ impl Generator {
             add_line!(self, "}};");
             add_line!(self, "");
         }
-        if next_parse_action_list_index >= u32::from(u16::MAX) {
-            Err(RenderError::ParseTable(
-                next_parse_action_list_index as usize,
-            ))?;
-        }
 
         let mut parse_table_entries = parse_table_entries
             .into_iter()
-            .map(|(id, i)| (i, id))
+            .map(|(entry, i)| (i, entry))
             .collect::<Vec<_>>();
         parse_table_entries.sort_by_key(|(index, _)| *index);
         self.add_parse_action_list(parse_table_entries);
-
-        Ok(())
     }
 
-    fn add_parse_action_list(&mut self, parse_table_entries: Vec<(u32, ActionListId)>) {
+    fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
         add_line!(
             self,
             "static const TSParseActionEntry ts_parse_actions[] = {{"
         );
         indent!(self);
-        for (i, id) in parse_table_entries {
-            let actions = self.parse_table.action_lists.get(id);
+        for (i, entry) in parse_table_entries {
             add!(
                 self,
                 "  [{i}] = {{.entry = {{.count = {}, .reusable = {}}}}},",
-                actions.len(),
-                id.reusable(),
+                entry.actions.len(),
+                entry.reusable
             );
-            for action in actions {
+            for action in entry.actions {
                 add!(self, " ");
-                match *action {
+                match action {
                     ParseAction::Accept => add!(self, " ACCEPT_INPUT()"),
                     ParseAction::Recover => add!(self, "RECOVER()"),
                     ParseAction::ShiftExtra => add!(self, "SHIFT_EXTRA()"),
@@ -1681,13 +1637,21 @@ impl Generator {
                     .unwrap()
             );
 
-            let metadata = self.metadata.unwrap_or_default();
+            let Some(metadata) = &self.metadata else {
+                panic!(
+                    indoc! {"
+                        Metadata is required to generate ABI version {}.
+                        This means that your grammar doesn't have a tree-sitter.json config file with an appropriate version field in the metadata table.
+                    "},
+                    self.abi_version
+                );
+            };
 
             add_line!(self, ".metadata = {{");
             indent!(self);
-            add_line!(self, ".major_version = {},", metadata.major);
-            add_line!(self, ".minor_version = {},", metadata.minor);
-            add_line!(self, ".patch_version = {},", metadata.patch);
+            add_line!(self, ".major_version = {},", metadata.major_version);
+            add_line!(self, ".minor_version = {},", metadata.minor_version);
+            add_line!(self, ".patch_version = {},", metadata.patch_version);
             dedent!(self);
             add_line!(self, "}},");
         }
@@ -1703,24 +1667,25 @@ impl Generator {
     }
 
     fn get_parse_action_list_id(
-        id: ActionListId,
-        pool: &ActionListPool,
-        parse_action_list_offsets: &mut FxHashMap<ActionListId, u32>,
-        next_parse_action_list_index: &mut u32,
-    ) -> u32 {
-        if let Some(&index) = parse_action_list_offsets.get(&id) {
+        &self,
+        entry: &ParseTableEntry,
+        parse_table_entries: &mut HashMap<ParseTableEntry, usize>,
+        next_parse_action_list_index: &mut usize,
+    ) -> usize {
+        if let Some(&index) = parse_table_entries.get(entry) {
             index
         } else {
             let result = *next_parse_action_list_index;
-            parse_action_list_offsets.insert(id, result);
-            *next_parse_action_list_index += 1 + pool.get(id).len() as u32;
+            parse_table_entries.insert(entry.clone(), result);
+            *next_parse_action_list_index += 1 + entry.actions.len();
             result
         }
     }
 
     fn get_field_map_id(
-        flat_field_map: Vec<(StrId, FieldLocation)>,
-        flat_field_maps: &mut Vec<(usize, Vec<(StrId, FieldLocation)>)>,
+        &self,
+        flat_field_map: Vec<(String, FieldLocation)>,
+        flat_field_maps: &mut Vec<(usize, Vec<(String, FieldLocation)>)>,
         next_flat_field_map_index: &mut usize,
     ) -> usize {
         if let Some((index, _)) = flat_field_maps.iter().find(|(_, e)| *e == *flat_field_map) {
@@ -1733,12 +1698,14 @@ impl Generator {
         result
     }
 
-    fn external_token_id(&self, token_idx: usize) -> String {
-        let token = &self.syntax_grammar.external_tokens[token_idx];
-        format!("ts_external_token_{}", self.sanitize_identifier(token.name))
+    fn external_token_id(&self, token: &ExternalToken) -> String {
+        format!(
+            "ts_external_token_{}",
+            self.sanitize_identifier(&token.name)
+        )
     }
 
-    fn assign_symbol_id(&mut self, symbol: Symbol, used_identifiers: &mut FxHashSet<String>) {
+    fn assign_symbol_id(&mut self, symbol: Symbol, used_identifiers: &mut HashSet<String>) {
         let mut id;
         if symbol == Symbol::end() {
             id = "ts_builtin_sym_end".to_string();
@@ -1766,32 +1733,29 @@ impl Generator {
         self.symbol_ids.insert(symbol, id);
     }
 
-    fn field_id(field_name: &str) -> String {
+    fn field_id(&self, field_name: &str) -> String {
         format!("field_{field_name}")
     }
 
-    fn metadata_for_symbol(&self, symbol: Symbol) -> (StrId, VariableType) {
-        let symbol_index = symbol.index as usize;
+    fn metadata_for_symbol(&self, symbol: Symbol) -> (&str, VariableType) {
         match symbol.kind {
-            SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                (StrPool::END_NAME_ID, VariableType::Hidden)
-            }
+            SymbolType::End | SymbolType::EndOfNonTerminalExtra => ("end", VariableType::Hidden),
             SymbolType::NonTerminal => {
-                let variable = &self.syntax_grammar.variables[symbol_index];
-                (variable.name, variable.kind)
+                let variable = &self.syntax_grammar.variables[symbol.index];
+                (&variable.name, variable.kind)
             }
             SymbolType::Terminal => {
-                let variable = &self.lexical_grammar.variables[symbol_index];
-                (variable.name, variable.kind)
+                let variable = &self.lexical_grammar.variables[symbol.index];
+                (&variable.name, variable.kind)
             }
             SymbolType::External => {
-                let token = &self.syntax_grammar.external_tokens[symbol_index];
-                (token.name, token.kind)
+                let token = &self.syntax_grammar.external_tokens[symbol.index];
+                (&token.name, token.kind)
             }
         }
     }
 
-    fn symbols_for_alias(&self, alias: Alias) -> Vec<Symbol> {
+    fn symbols_for_alias(&self, alias: &Alias) -> Vec<Symbol> {
         self.parse_table
             .symbols
             .iter()
@@ -1802,14 +1766,13 @@ impl Generator {
                         let (name, kind) = self.metadata_for_symbol(*symbol);
                         name == alias.value && kind == alias.kind()
                     },
-                    |&default_alias| default_alias == alias,
+                    |default_alias| default_alias == alias,
                 )
             })
             .collect()
     }
 
-    fn sanitize_identifier(&self, name: StrId) -> String {
-        let name = self.str_pool.resolve(name);
+    fn sanitize_identifier(&self, name: &str) -> String {
         let mut result = String::with_capacity(name.len());
         for c in name.chars() {
             if c.is_ascii_alphanumeric() || c == '_' {
@@ -1884,11 +1847,11 @@ impl Generator {
                         '\u{007F}' => "DEL",
                         '\u{FEFF}' => "BOM",
                         '\u{0080}'..='\u{FFFF}' => {
-                            write!(result, "u{:04x}", c as u32).unwrap();
+                            result.push_str(&format!("u{:04x}", c as u32));
                             break 'special_chars;
                         }
                         '\u{10000}'..='\u{10FFFF}' => {
-                            write!(result, "U{:08x}", c as u32).unwrap();
+                            result.push_str(&format!("U{:08x}", c as u32));
                             break 'special_chars;
                         }
                         '0'..='9' | 'a'..='z' | 'A'..='Z' | '_' => unreachable!(),
@@ -1904,8 +1867,7 @@ impl Generator {
         result
     }
 
-    fn sanitize_string(&self, name: StrId) -> String {
-        let name = self.str_pool.resolve(name);
+    fn sanitize_string(&self, name: &str) -> String {
         let mut result = String::with_capacity(name.len());
         for c in name.chars() {
             match c {
@@ -1920,9 +1882,11 @@ impl Generator {
                 '\r' => result += "\\r",
                 '\t' => result += "\\t",
                 '\0' => result += "\\0",
-                '\u{0001}'..='\u{001f}' => write!(result, "\\x{:02x}", c as u32).unwrap(),
-                '\u{007F}'..='\u{FFFF}' => write!(result, "\\u{:04x}", c as u32).unwrap(),
-                '\u{10000}'..='\u{10FFFF}' => write!(result, "\\U{:08x}", c as u32).unwrap(),
+                '\u{0001}'..='\u{001f}' => result += &format!("\\x{:02x}", c as u32),
+                '\u{007F}'..='\u{FFFF}' => result += &format!("\\u{:04x}", c as u32),
+                '\u{10000}'..='\u{10FFFF}' => {
+                    result.push_str(&format!("\\U{:08x}", c as u32));
+                }
                 _ => result.push(c),
             }
         }
@@ -1964,32 +1928,27 @@ impl Generator {
 /// * `default_aliases` - A map describing the global rename rules that should apply. the keys are
 ///   symbols that are *always* aliased in the same way, and the values are the aliases that are
 ///   applied to those symbols.
-/// * `str_pool` - A backing pool for `StrId`-identified strings within `syntax_grammar`,
-///   `lexical_grammar`, and `default_aliases`.
 /// * `abi_version` - The language ABI version that should be generated. Usually you want
 ///   Tree-sitter's current version, but right after making an ABI change, it may be useful to
 ///   generate code with the previous ABI.
-#[expect(
-    clippy::too_many_arguments,
-    reason = "all parameters are required for code generation"
-)]
+#[allow(clippy::too_many_arguments)]
 pub fn render_c_code(
-    name: StrId,
+    name: &str,
     tables: Tables,
     syntax_grammar: SyntaxGrammar,
     lexical_grammar: LexicalGrammar,
     default_aliases: AliasMap,
-    str_pool: StrPool,
     abi_version: usize,
     semantic_version: Option<(u8, u8, u8)>,
     supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
-) -> RenderResult<String> {
-    if !(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version) {
-        Err(RenderError::ABI(abi_version))?;
-    }
+) -> String {
+    assert!(
+        (ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version),
+        "This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {abi_version}",
+    );
 
     Generator {
-        language_name: str_pool.resolve(name).to_string(),
+        language_name: name.to_string(),
         parse_table: tables.parse_table,
         main_lex_table: tables.main_lex_table,
         keyword_lex_table: tables.keyword_lex_table,
@@ -1999,13 +1958,12 @@ pub fn render_c_code(
         lexical_grammar,
         default_aliases,
         abi_version,
-        metadata: semantic_version.map(|(major, minor, patch)| Metadata {
-            major,
-            minor,
-            patch,
+        metadata: semantic_version.map(|(major_version, minor_version, patch_version)| Metadata {
+            major_version,
+            minor_version,
+            patch_version,
         }),
         supertype_symbol_map,
-        str_pool,
         ..Default::default()
     }
     .generate()

cli/generate/src/rules.rs

@@ -0,0 +1,544 @@
+use std::{collections::HashMap, fmt};
+
+use serde::Serialize;
+use smallbitvec::SmallBitVec;
+
+use super::grammars::VariableType;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
+pub enum SymbolType {
+    External,
+    End,
+    EndOfNonTerminalExtra,
+    Terminal,
+    NonTerminal,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
+pub enum Associativity {
+    Left,
+    Right,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
+pub struct Alias {
+    pub value: String,
+    pub is_named: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default, Serialize)]
+pub enum Precedence {
+    #[default]
+    None,
+    Integer(i32),
+    Name(String),
+}
+
+pub type AliasMap = HashMap<Symbol, Alias>;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize)]
+pub struct MetadataParams {
+    pub precedence: Precedence,
+    pub dynamic_precedence: i32,
+    pub associativity: Option<Associativity>,
+    pub is_token: bool,
+    pub is_main_token: bool,
+    pub alias: Option<Alias>,
+    pub field_name: Option<String>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
+pub struct Symbol {
+    pub kind: SymbolType,
+    pub index: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
+pub enum Rule {
+    Blank,
+    String(String),
+    Pattern(String, String),
+    NamedSymbol(String),
+    Symbol(Symbol),
+    Choice(Vec<Rule>),
+    Metadata {
+        params: MetadataParams,
+        rule: Box<Rule>,
+    },
+    Repeat(Box<Rule>),
+    Seq(Vec<Rule>),
+    Reserved {
+        rule: Box<Rule>,
+        context_name: String,
+    },
+}
+
+// Because tokens are represented as small (~400 max) unsigned integers,
+// sets of tokens can be efficiently represented as bit vectors with each
+// index corresponding to a token, and each value representing whether or not
+// the token is present in the set.
+#[derive(Default, Clone, PartialEq, Eq, Hash)]
+pub struct TokenSet {
+    terminal_bits: SmallBitVec,
+    external_bits: SmallBitVec,
+    eof: bool,
+    end_of_nonterminal_extra: bool,
+}
+
+impl fmt::Debug for TokenSet {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.iter()).finish()
+    }
+}
+
+impl PartialOrd for TokenSet {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for TokenSet {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.terminal_bits
+            .iter()
+            .cmp(other.terminal_bits.iter())
+            .then_with(|| self.external_bits.iter().cmp(other.external_bits.iter()))
+            .then_with(|| self.eof.cmp(&other.eof))
+            .then_with(|| {
+                self.end_of_nonterminal_extra
+                    .cmp(&other.end_of_nonterminal_extra)
+            })
+    }
+}
+
+impl Rule {
+    pub fn field(name: String, content: Self) -> Self {
+        add_metadata(content, move |params| {
+            params.field_name = Some(name);
+        })
+    }
+
+    pub fn alias(content: Self, value: String, is_named: bool) -> Self {
+        add_metadata(content, move |params| {
+            params.alias = Some(Alias { value, is_named });
+        })
+    }
+
+    pub fn token(content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.is_token = true;
+        })
+    }
+
+    pub fn immediate_token(content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.is_token = true;
+            params.is_main_token = true;
+        })
+    }
+
+    pub fn prec(value: Precedence, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.precedence = value;
+        })
+    }
+
+    pub fn prec_left(value: Precedence, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.associativity = Some(Associativity::Left);
+            params.precedence = value;
+        })
+    }
+
+    pub fn prec_right(value: Precedence, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.associativity = Some(Associativity::Right);
+            params.precedence = value;
+        })
+    }
+
+    pub fn prec_dynamic(value: i32, content: Self) -> Self {
+        add_metadata(content, |params| {
+            params.dynamic_precedence = value;
+        })
+    }
+
+    pub fn repeat(rule: Self) -> Self {
+        Self::Repeat(Box::new(rule))
+    }
+
+    pub fn choice(rules: Vec<Self>) -> Self {
+        let mut elements = Vec::with_capacity(rules.len());
+        for rule in rules {
+            choice_helper(&mut elements, rule);
+        }
+        Self::Choice(elements)
+    }
+
+    pub const fn seq(rules: Vec<Self>) -> Self {
+        Self::Seq(rules)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        match self {
+            Self::Blank | Self::Pattern(..) | Self::NamedSymbol(_) | Self::Symbol(_) => false,
+            Self::String(string) => string.is_empty(),
+            Self::Metadata { rule, .. } | Self::Repeat(rule) | Self::Reserved { rule, .. } => {
+                rule.is_empty()
+            }
+            Self::Choice(rules) => rules.iter().any(Self::is_empty),
+            Self::Seq(rules) => rules.iter().all(Self::is_empty),
+        }
+    }
+}
+
+impl Alias {
+    #[must_use]
+    pub const fn kind(&self) -> VariableType {
+        if self.is_named {
+            VariableType::Named
+        } else {
+            VariableType::Anonymous
+        }
+    }
+}
+
+impl Precedence {
+    #[must_use]
+    pub const fn is_none(&self) -> bool {
+        matches!(self, Self::None)
+    }
+}
+
+#[cfg(test)]
+impl Rule {
+    #[must_use]
+    pub const fn terminal(index: usize) -> Self {
+        Self::Symbol(Symbol::terminal(index))
+    }
+
+    #[must_use]
+    pub const fn non_terminal(index: usize) -> Self {
+        Self::Symbol(Symbol::non_terminal(index))
+    }
+
+    #[must_use]
+    pub const fn external(index: usize) -> Self {
+        Self::Symbol(Symbol::external(index))
+    }
+
+    #[must_use]
+    pub fn named(name: &'static str) -> Self {
+        Self::NamedSymbol(name.to_string())
+    }
+
+    #[must_use]
+    pub fn string(value: &'static str) -> Self {
+        Self::String(value.to_string())
+    }
+
+    #[must_use]
+    pub fn pattern(value: &'static str, flags: &'static str) -> Self {
+        Self::Pattern(value.to_string(), flags.to_string())
+    }
+}
+
+impl Symbol {
+    #[must_use]
+    pub fn is_terminal(&self) -> bool {
+        self.kind == SymbolType::Terminal
+    }
+
+    #[must_use]
+    pub fn is_non_terminal(&self) -> bool {
+        self.kind == SymbolType::NonTerminal
+    }
+
+    #[must_use]
+    pub fn is_external(&self) -> bool {
+        self.kind == SymbolType::External
+    }
+
+    #[must_use]
+    pub fn is_eof(&self) -> bool {
+        self.kind == SymbolType::End
+    }
+
+    #[must_use]
+    pub const fn non_terminal(index: usize) -> Self {
+        Self {
+            kind: SymbolType::NonTerminal,
+            index,
+        }
+    }
+
+    #[must_use]
+    pub const fn terminal(index: usize) -> Self {
+        Self {
+            kind: SymbolType::Terminal,
+            index,
+        }
+    }
+
+    #[must_use]
+    pub const fn external(index: usize) -> Self {
+        Self {
+            kind: SymbolType::External,
+            index,
+        }
+    }
+
+    #[must_use]
+    pub const fn end() -> Self {
+        Self {
+            kind: SymbolType::End,
+            index: 0,
+        }
+    }
+
+    #[must_use]
+    pub const fn end_of_nonterminal_extra() -> Self {
+        Self {
+            kind: SymbolType::EndOfNonTerminalExtra,
+            index: 0,
+        }
+    }
+}
+
+impl From<Symbol> for Rule {
+    #[must_use]
+    fn from(symbol: Symbol) -> Self {
+        Self::Symbol(symbol)
+    }
+}
+
+impl TokenSet {
+    #[must_use]
+    pub const fn new() -> Self {
+        Self {
+            terminal_bits: SmallBitVec::new(),
+            external_bits: SmallBitVec::new(),
+            eof: false,
+            end_of_nonterminal_extra: false,
+        }
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = Symbol> + '_ {
+        self.terminal_bits
+            .iter()
+            .enumerate()
+            .filter_map(|(i, value)| {
+                if value {
+                    Some(Symbol::terminal(i))
+                } else {
+                    None
+                }
+            })
+            .chain(
+                self.external_bits
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(i, value)| {
+                        if value {
+                            Some(Symbol::external(i))
+                        } else {
+                            None
+                        }
+                    }),
+            )
+            .chain(if self.eof { Some(Symbol::end()) } else { None })
+            .chain(if self.end_of_nonterminal_extra {
+                Some(Symbol::end_of_nonterminal_extra())
+            } else {
+                None
+            })
+    }
+
+    pub fn terminals(&self) -> impl Iterator<Item = Symbol> + '_ {
+        self.terminal_bits
+            .iter()
+            .enumerate()
+            .filter_map(|(i, value)| {
+                if value {
+                    Some(Symbol::terminal(i))
+                } else {
+                    None
+                }
+            })
+    }
+
+    pub fn contains(&self, symbol: &Symbol) -> bool {
+        match symbol.kind {
+            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
+            SymbolType::Terminal => self.terminal_bits.get(symbol.index).unwrap_or(false),
+            SymbolType::External => self.external_bits.get(symbol.index).unwrap_or(false),
+            SymbolType::End => self.eof,
+            SymbolType::EndOfNonTerminalExtra => self.end_of_nonterminal_extra,
+        }
+    }
+
+    pub fn contains_terminal(&self, index: usize) -> bool {
+        self.terminal_bits.get(index).unwrap_or(false)
+    }
+
+    pub fn insert(&mut self, other: Symbol) {
+        let vec = match other.kind {
+            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
+            SymbolType::Terminal => &mut self.terminal_bits,
+            SymbolType::External => &mut self.external_bits,
+            SymbolType::End => {
+                self.eof = true;
+                return;
+            }
+            SymbolType::EndOfNonTerminalExtra => {
+                self.end_of_nonterminal_extra = true;
+                return;
+            }
+        };
+        if other.index >= vec.len() {
+            vec.resize(other.index + 1, false);
+        }
+        vec.set(other.index, true);
+    }
+
+    pub fn remove(&mut self, other: &Symbol) -> bool {
+        let vec = match other.kind {
+            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
+            SymbolType::Terminal => &mut self.terminal_bits,
+            SymbolType::External => &mut self.external_bits,
+            SymbolType::End => {
+                return if self.eof {
+                    self.eof = false;
+                    true
+                } else {
+                    false
+                }
+            }
+            SymbolType::EndOfNonTerminalExtra => {
+                return if self.end_of_nonterminal_extra {
+                    self.end_of_nonterminal_extra = false;
+                    true
+                } else {
+                    false
+                };
+            }
+        };
+        if other.index < vec.len() && vec[other.index] {
+            vec.set(other.index, false);
+            while vec.last() == Some(false) {
+                vec.pop();
+            }
+            return true;
+        }
+        false
+    }
+
+    pub fn is_empty(&self) -> bool {
+        !self.eof
+            && !self.end_of_nonterminal_extra
+            && !self.terminal_bits.iter().any(|a| a)
+            && !self.external_bits.iter().any(|a| a)
+    }
+
+    pub fn len(&self) -> usize {
+        self.eof as usize
+            + self.end_of_nonterminal_extra as usize
+            + self.terminal_bits.iter().filter(|b| *b).count()
+            + self.external_bits.iter().filter(|b| *b).count()
+    }
+
+    pub fn insert_all_terminals(&mut self, other: &Self) -> bool {
+        let mut result = false;
+        if other.terminal_bits.len() > self.terminal_bits.len() {
+            self.terminal_bits.resize(other.terminal_bits.len(), false);
+        }
+        for (i, element) in other.terminal_bits.iter().enumerate() {
+            if element {
+                result |= !self.terminal_bits[i];
+                self.terminal_bits.set(i, element);
+            }
+        }
+        result
+    }
+
+    fn insert_all_externals(&mut self, other: &Self) -> bool {
+        let mut result = false;
+        if other.external_bits.len() > self.external_bits.len() {
+            self.external_bits.resize(other.external_bits.len(), false);
+        }
+        for (i, element) in other.external_bits.iter().enumerate() {
+            if element {
+                result |= !self.external_bits[i];
+                self.external_bits.set(i, element);
+            }
+        }
+        result
+    }
+
+    pub fn insert_all(&mut self, other: &Self) -> bool {
+        let mut result = false;
+        if other.eof {
+            result |= !self.eof;
+            self.eof = true;
+        }
+        if other.end_of_nonterminal_extra {
+            result |= !self.end_of_nonterminal_extra;
+            self.end_of_nonterminal_extra = true;
+        }
+        result |= self.insert_all_terminals(other);
+        result |= self.insert_all_externals(other);
+        result
+    }
+}
+
+impl FromIterator<Symbol> for TokenSet {
+    fn from_iter<T: IntoIterator<Item = Symbol>>(iter: T) -> Self {
+        let mut result = Self::new();
+        for symbol in iter {
+            result.insert(symbol);
+        }
+        result
+    }
+}
+
+fn add_metadata<T: FnOnce(&mut MetadataParams)>(input: Rule, f: T) -> Rule {
+    match input {
+        Rule::Metadata { rule, mut params } if !params.is_token => {
+            f(&mut params);
+            Rule::Metadata { rule, params }
+        }
+        _ => {
+            let mut params = MetadataParams::default();
+            f(&mut params);
+            Rule::Metadata {
+                rule: Box::new(input),
+                params,
+            }
+        }
+    }
+}
+
+fn choice_helper(result: &mut Vec<Rule>, rule: Rule) {
+    match rule {
+        Rule::Choice(elements) => {
+            for element in elements {
+                choice_helper(result, element);
+            }
+        }
+        _ => {
+            if !result.contains(&rule) {
+                result.push(rule);
+            }
+        }
+    }
+}
+
+impl fmt::Display for Precedence {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Integer(i) => write!(f, "{i}"),
+            Self::Name(s) => write!(f, "'{s}'"),
+            Self::None => write!(f, "none"),
+        }
+    }
+}

cli/generate/src/tables.rs

@@ -0,0 +1,167 @@
+use std::collections::BTreeMap;
+
+use super::{
+    nfa::CharacterSet,
+    rules::{Alias, Symbol, TokenSet},
+};
+pub type ProductionInfoId = usize;
+pub type ParseStateId = usize;
+pub type LexStateId = usize;
+
+use std::hash::BuildHasherDefault;
+
+use indexmap::IndexMap;
+use rustc_hash::FxHasher;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum ParseAction {
+    Accept,
+    Shift {
+        state: ParseStateId,
+        is_repetition: bool,
+    },
+    ShiftExtra,
+    Recover,
+    Reduce {
+        symbol: Symbol,
+        child_count: usize,
+        dynamic_precedence: i32,
+        production_id: ProductionInfoId,
+    },
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum GotoAction {
+    Goto(ParseStateId),
+    ShiftExtra,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ParseTableEntry {
+    pub actions: Vec<ParseAction>,
+    pub reusable: bool,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ParseState {
+    pub id: ParseStateId,
+    pub terminal_entries: IndexMap<Symbol, ParseTableEntry, BuildHasherDefault<FxHasher>>,
+    pub nonterminal_entries: IndexMap<Symbol, GotoAction, BuildHasherDefault<FxHasher>>,
+    pub reserved_words: TokenSet,
+    pub lex_state_id: usize,
+    pub external_lex_state_id: usize,
+    pub core_id: usize,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct FieldLocation {
+    pub index: usize,
+    pub inherited: bool,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct ProductionInfo {
+    pub alias_sequence: Vec<Option<Alias>>,
+    pub field_map: BTreeMap<String, Vec<FieldLocation>>,
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct ParseTable {
+    pub states: Vec<ParseState>,
+    pub symbols: Vec<Symbol>,
+    pub production_infos: Vec<ProductionInfo>,
+    pub max_aliased_production_length: usize,
+    pub external_lex_states: Vec<TokenSet>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct AdvanceAction {
+    pub state: LexStateId,
+    pub in_main_token: bool,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
+pub struct LexState {
+    pub accept_action: Option<Symbol>,
+    pub eof_action: Option<AdvanceAction>,
+    pub advance_actions: Vec<(CharacterSet, AdvanceAction)>,
+}
+
+#[derive(Debug, PartialEq, Eq, Default)]
+pub struct LexTable {
+    pub states: Vec<LexState>,
+}
+
+impl ParseTableEntry {
+    #[must_use]
+    pub const fn new() -> Self {
+        Self {
+            reusable: true,
+            actions: Vec::new(),
+        }
+    }
+}
+
+impl ParseState {
+    pub fn is_end_of_non_terminal_extra(&self) -> bool {
+        self.terminal_entries
+            .contains_key(&Symbol::end_of_nonterminal_extra())
+    }
+
+    pub fn referenced_states(&self) -> impl Iterator<Item = ParseStateId> + '_ {
+        self.terminal_entries
+            .iter()
+            .flat_map(|(_, entry)| {
+                entry.actions.iter().filter_map(|action| match action {
+                    ParseAction::Shift { state, .. } => Some(*state),
+                    _ => None,
+                })
+            })
+            .chain(self.nonterminal_entries.iter().filter_map(|(_, action)| {
+                if let GotoAction::Goto(state) = action {
+                    Some(*state)
+                } else {
+                    None
+                }
+            }))
+    }
+
+    pub fn update_referenced_states<F>(&mut self, mut f: F)
+    where
+        F: FnMut(usize, &Self) -> usize,
+    {
+        let mut updates = Vec::new();
+        for (symbol, entry) in &self.terminal_entries {
+            for (i, action) in entry.actions.iter().enumerate() {
+                if let ParseAction::Shift { state, .. } = action {
+                    let result = f(*state, self);
+                    if result != *state {
+                        updates.push((*symbol, i, result));
+                    }
+                }
+            }
+        }
+        for (symbol, action) in &self.nonterminal_entries {
+            if let GotoAction::Goto(other_state) = action {
+                let result = f(*other_state, self);
+                if result != *other_state {
+                    updates.push((*symbol, 0, result));
+                }
+            }
+        }
+        for (symbol, action_index, new_state) in updates {
+            if symbol.is_non_terminal() {
+                self.nonterminal_entries
+                    .insert(symbol, GotoAction::Goto(new_state));
+            } else {
+                let entry = self.terminal_entries.get_mut(&symbol).unwrap();
+                if let ParseAction::Shift { is_repetition, .. } = entry.actions[action_index] {
+                    entry.actions[action_index] = ParseAction::Shift {
+                        state: new_state,
+                        is_repetition,
+                    };
+                }
+            }
+        }
+    }
+}

cli/generate/src/templates/array.h

@@ -50,104 +50,69 @@ extern "C" {
 /// memory allocated for the array's contents.
 #define array_clear(self) ((self)->size = 0)
 
-#ifdef __cplusplus
-#define _array__cast(self, expr) (decltype((self)->contents))(expr)
-#else
-#define _array__cast(self, expr) (expr)
-#endif
-
 /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
 /// less than the array's current capacity, this function has no effect.
-#define array_reserve(self, new_capacity)                 \
-  ((self)->contents = _array__cast(self, _array__reserve( \
-    (void *)(self)->contents, &(self)->capacity,          \
-    array_elem_size(self), new_capacity))                 \
-  )
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
 
 /// Free any memory allocated for this array. Note that this does not free any
 /// memory allocated for the array's contents.
-#define array_delete(self)                           \
-  do {                                               \
-    if ((self)->contents) ts_free((self)->contents); \
-    (self)->contents = NULL;                         \
-    (self)->size = 0;                                \
-    (self)->capacity = 0;                            \
-  } while (0)
+#define array_delete(self) _array__delete((Array *)(self))
 
 /// Push a new `element` onto the end of the array.
-#define array_push(self, element)                                 \
-  do {                                                            \
-    (self)->contents = _array__cast(self, _array__grow(           \
-      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
-      1, array_elem_size(self)                                    \
-    ));                                                           \
-   (self)->contents[(self)->size++] = (element);                  \
-  } while(0)
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
 
 /// Increase the array's size by `count` elements.
 /// New elements are zero-initialized.
-#define array_grow_by(self, count)                                               \
-  do {                                                                           \
-    if ((count) == 0) break;                                                     \
-    (self)->contents = _array__cast(self, _array__grow(                          \
-      (self)->contents, (self)->size, &(self)->capacity,                         \
-      count, array_elem_size(self)                                               \
-    ));                                                                          \
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
     memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
-    (self)->size += (count);                                                     \
+    (self)->size += (count); \
   } while (0)
 
 /// Append all elements from one array to the end of another.
-#define array_push_all(self, other) \
+#define array_push_all(self, other)                                       \
   array_extend((self), (other)->size, (other)->contents)
 
 /// Append `count` elements to the end of the array, reading their values from the
 /// `contents` pointer.
-#define array_extend(self, count, other_contents)                 \
-  ((self)->contents = _array__cast(self, _array__splice(          \
-    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
-    array_elem_size(self), (self)->size, 0, count, other_contents \
-  )))
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
 
 /// Remove `old_count` elements from the array starting at the given `index`. At
 /// the same index, insert `new_count` new elements, reading their values from the
 /// `new_contents` pointer.
-#define array_splice(self, _index, old_count, new_count, new_contents) \
-  ((self)->contents = _array__cast(self, _array__splice(              \
-    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
-    array_elem_size(self), _index, old_count, new_count, new_contents  \
-  )))
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
 
 /// Insert one `element` into the array at the given `index`.
-#define array_insert(self, _index, element)                     \
-  ((self)->contents = _array__cast(self, _array__splice(        \
-    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
-    array_elem_size(self), _index, 0, 1, &(element)             \
-  )))
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
 
 /// Remove one element from the array at the given `index`.
 #define array_erase(self, _index) \
-  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
+  _array__erase((Array *)(self), array_elem_size(self), _index)
 
 /// Pop the last element off the array, returning the element by value.
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 /// Assign the contents of one array to another, reallocating if necessary.
-#define array_assign(self, other)                                   \
-  ((self)->contents = _array__cast(self, _array__assign(            \
-    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
-    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
-  )))
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
 
 /// Swap one array with another
-#define array_swap(self, other)                                     \
-  do {                                                              \
-    void *_array_swap_tmp = (void *)(self)->contents;               \
-    (self)->contents = (other)->contents;                           \
-    (other)->contents = _array__cast(other, _array_swap_tmp);       \
-    _array__swap(&(self)->size, &(self)->capacity,                  \
-                 &(other)->size, &(other)->capacity);               \
-  } while (0)
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
 
 /// Get the size of the array contents
 #define array_elem_size(self) (sizeof *(self)->contents)
@@ -192,90 +157,82 @@ extern "C" {
 
 // Private
 
-// Pointers to individual `Array` fields (rather than the entire `Array` itself)
-// are passed to the various `_array__*` functions below to address strict aliasing
-// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
-//
-// The `Array` type itself was not altered as a solution in order to avoid breakage
-// with existing consumers (in particular, parsers with external scanners).
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
 
 /// This is not what you're looking for, see `array_erase`.
-static inline void _array__erase(void* self_contents, uint32_t *size,
-                                size_t element_size, uint32_t index) {
-  assert(index < *size);
-  char *contents = (char *)self_contents;
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
   memmove(contents + index * element_size, contents + (index + 1) * element_size,
-          (*size - index - 1) * element_size);
-  (*size)--;
+          (self->size - index - 1) * element_size);
+  self->size--;
 }
 
 /// This is not what you're looking for, see `array_reserve`.
-static inline void *_array__reserve(void *contents, uint32_t *capacity,
-                                  size_t element_size, uint32_t new_capacity) {
-  void *new_contents = contents;
-  if (new_capacity > *capacity) {
-    if (contents) {
-      new_contents = ts_realloc(contents, new_capacity * element_size);
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
     } else {
-      new_contents = ts_malloc(new_capacity * element_size);
+      self->contents = ts_malloc(new_capacity * element_size);
     }
-    *capacity = new_capacity;
+    self->capacity = new_capacity;
   }
-  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_assign`.
-static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
-                                 const void *other_contents, uint32_t other_size, size_t element_size) {
-  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
-  *self_size = other_size;
-  memcpy(new_contents, other_contents, *self_size * element_size);
-  return new_contents;
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
 }
 
 /// This is not what you're looking for, see `array_swap`.
-static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
-                               uint32_t *other_size, uint32_t *other_capacity) {
-  uint32_t tmp_size = *self_size;
-  uint32_t tmp_capacity = *self_capacity;
-  *self_size = *other_size;
-  *self_capacity = *other_capacity;
-  *other_size = tmp_size;
-  *other_capacity = tmp_capacity;
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
 }
 
 /// This is not what you're looking for, see `array_push` or `array_grow_by`.
-static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
-                               uint32_t count, size_t element_size) {
-  void *new_contents = contents;
-  uint32_t new_size = size + count;
-  if (new_size > *capacity) {
-    uint32_t new_capacity = *capacity * 2;
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
-    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
+    _array__reserve(self, element_size, new_capacity);
   }
-  return new_contents;
 }
 
 /// This is not what you're looking for, see `array_splice`.
-static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
-                                 size_t element_size,
+static inline void _array__splice(Array *self, size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
-  uint32_t new_size = *size + new_count - old_count;
+  uint32_t new_size = self->size + new_count - old_count;
   uint32_t old_end = index + old_count;
   uint32_t new_end = index + new_count;
-  assert(old_end <= *size);
+  assert(old_end <= self->size);
 
-  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
+  _array__reserve(self, element_size, new_size);
 
-  char *contents = (char *)new_contents;
-  if (*size > old_end) {
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
     memmove(
       contents + new_end * element_size,
       contents + old_end * element_size,
-      (*size - old_end) * element_size
+      (self->size - old_end) * element_size
     );
   }
   if (new_count > 0) {
@@ -293,9 +250,7 @@ static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t
       );
     }
   }
-  *size += new_count - old_count;
-
-  return new_contents;
+  self->size += new_count - old_count;
 }
 
 /// A binary search routine, based on Rust's `std::slice::binary_search_by`.

cli/loader/Cargo.toml

@@ -5,40 +5,40 @@ description = "Locates, builds, and loads tree-sitter grammars at runtime"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
+readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
-documentation = "https://docs.rs/tree-sitter-loader"
 license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 
 [package.metadata.docs.rs]
 all-features = true
-rustdoc-args = [ "--cfg", "docsrs" ]
-
-[lib]
-path = "src/loader.rs"
+rustdoc-args = ["--cfg", "docsrs"]
 
 [lints]
 workspace = true
 
 [features]
-default = [ "tree-sitter-highlight", "tree-sitter-tags" ]
-wasm = [ "tree-sitter/wasm" ]
+wasm = ["tree-sitter/wasm"]
+default = ["tree-sitter-highlight", "tree-sitter-tags"]
 
 [dependencies]
+anyhow.workspace = true
 cc.workspace = true
 etcetera.workspace = true
+fs4.workspace = true
+indoc.workspace = true
 libloading.workspace = true
-log.workspace = true
 once_cell.workspace = true
+path-slash.workspace = true
 regex.workspace = true
 semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tempfile.workspace = true
-thiserror.workspace = true
+url.workspace = true
 
 tree-sitter = { workspace = true }
-tree-sitter-highlight = { optional = true, workspace = true }
-tree-sitter-tags = { optional = true, workspace = true }
+tree-sitter-highlight = { workspace = true, optional = true }
+tree-sitter-tags = { workspace = true, optional = true }

cli/loader/build.rs

@@ -7,4 +7,7 @@ fn main() {
         "cargo:rustc-env=BUILD_HOST={}",
         std::env::var("HOST").unwrap()
     );
+
+    let emscripten_version = std::fs::read_to_string("emscripten-version").unwrap();
+    println!("cargo:rustc-env=EMSCRIPTEN_VERSION={emscripten_version}");
 }

cli/loader/emscripten-version

@@ -0,0 +1 @@
+4.0.1

cli/npm/dsl.d.ts

@@ -3,22 +3,17 @@ type BlankRule = { type: 'BLANK' };
 type ChoiceRule = { type: 'CHOICE'; members: Rule[] };
 type FieldRule = { type: 'FIELD'; name: string; content: Rule };
 type ImmediateTokenRule = { type: 'IMMEDIATE_TOKEN'; content: Rule };
-type PatternRule = { type: 'PATTERN'; value: string; flags?: string };
-type PrecedenceValue = string | number;
+type PatternRule = { type: 'PATTERN'; value: string };
 type PrecDynamicRule = { type: 'PREC_DYNAMIC'; content: Rule; value: number };
-type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: PrecedenceValue };
-type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: PrecedenceValue };
-type PrecRule = { type: 'PREC'; content: Rule; value: PrecedenceValue };
+type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: number };
+type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number };
+type PrecRule = { type: 'PREC'; content: Rule; value: number };
 type Repeat1Rule = { type: 'REPEAT1'; content: Rule };
 type RepeatRule = { type: 'REPEAT'; content: Rule };
-type ReservedRule = { type: 'RESERVED'; content: Rule; context_name: string };
 type SeqRule = { type: 'SEQ'; members: Rule[] };
 type StringRule = { type: 'STRING'; value: string };
 type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
-type PrecedenceEntry = StringRule | SymbolRule<string>;
 type TokenRule = { type: 'TOKEN'; content: Rule };
-type EOFRule = { type: 'EOF' };
-
 
 type Rule =
   | AliasRule
@@ -33,17 +28,17 @@ type Rule =
   | PrecRule
   | Repeat1Rule
   | RepeatRule
-  | ReservedRule
   | SeqRule
   | StringRule
   | SymbolRule<string>
-  | TokenRule
-  | EOFRule;
+  | TokenRule;
 
-declare class RustRegex {
+class RustRegex {
   value: string;
 
-  constructor(pattern: string);
+  constructor(pattern: string) {
+    this.value = pattern;
+  }
 }
 
 type RuleOrLiteral = Rule | RegExp | RustRegex | string;
@@ -91,8 +86,8 @@ interface Grammar<
    */
   precedences?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: PrecedenceEntry[][],
-  ) => (string | PrecedenceEntry)[][],
+    previous: Rule[][],
+  ) => RuleOrLiteral[][],
 
   /**
    * An array of arrays of rule names. Each inner array represents a set of
@@ -106,8 +101,8 @@ interface Grammar<
    */
   conflicts?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: SymbolRule<string>[][],
-  ) => SymbolRule<string>[][];
+    previous: Rule[][],
+  ) => RuleOrLiteral[][];
 
   /**
    * An array of token names which can be returned by an _external scanner_.
@@ -132,11 +127,9 @@ interface Grammar<
    * specify extras: `$ => []` in your grammar.
    *
    *  @param $ grammar rules
-   *  @param previous array of extras from the base grammar
    */
   extras?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[],
   ) => RuleOrLiteral[];
 
   /**
@@ -149,8 +142,8 @@ interface Grammar<
    */
   inline?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: SymbolRule<string>[],
-  ) => SymbolRule<string>[];
+    previous: Rule[],
+  ) => RuleOrLiteral[];
 
   /**
    * A list of hidden rule names that should be considered supertypes in the
@@ -162,8 +155,8 @@ interface Grammar<
    */
   supertypes?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: SymbolRule<string>[],
-  ) => SymbolRule<string>[];
+    previous: Rule[],
+  ) => RuleOrLiteral[];
 
   /**
    * The name of a token that will match keywords for the purpose of the
@@ -173,47 +166,13 @@ interface Grammar<
    *
    * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar#keyword-extraction
    */
-  word?: (
-    $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-  ) => SymbolRule<string>;
-
-
-  /**
-   * Mapping of names to reserved word sets. The first reserved word set is the
-   * global word set, meaning it applies to every rule in every parse state.
-   * The other word sets can be used with the `reserved` function. Each callback
-   * receives the base grammar's reserved word set of the same name as its second
-   * argument, or `undefined` if no matching set exists.
-   */
-  reserved?: Record<
-    string,
-    (
-      $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-      previous: Rule[] | undefined,
-    ) => RuleOrLiteral[]
-  >;
+  word?: ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral;
 }
 
-/**
- * Return type of grammar(). The runtime evaluates and normalizes the grammar
- * beneath a "grammar" key. Optional input fields become required output fields
- * with default values when not provided.
- */
 type GrammarSchema<RuleName extends string> = {
-  grammar: {
-    name: string;
-    /** Base grammar name when extending; undefined for root grammars. */
-    inherits: string | undefined;
-    rules: Record<RuleName, Rule>;
-    precedences: PrecedenceEntry[][];
-    conflicts: string[][];
-    externals: Rule[];
-    extras: Rule[];
-    inline: string[];
-    supertypes: string[];
-    word: string | undefined;
-    reserved: Record<string, Rule[]>;
-  };
+  [K in keyof Grammar<RuleName>]: K extends 'rules'
+  ? Record<RuleName, Rule>
+  : Grammar<RuleName>[K];
 };
 
 /**
@@ -292,7 +251,7 @@ declare function optional(rule: RuleOrLiteral): ChoiceRule;
  * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
  */
 declare const prec: {
-  (value: string | number, rule: RuleOrLiteral): PrecRule;
+  (value: String | number, rule: RuleOrLiteral): PrecRule;
 
   /**
    * Marks the given rule as left-associative (and optionally applies a
@@ -308,7 +267,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   left(rule: RuleOrLiteral): PrecLeftRule;
-  left(value: string | number, rule: RuleOrLiteral): PrecLeftRule;
+  left(value: String | number, rule: RuleOrLiteral): PrecLeftRule;
 
   /**
    * Marks the given rule as right-associative (and optionally applies a
@@ -324,7 +283,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   right(rule: RuleOrLiteral): PrecRightRule;
-  right(value: string | number, rule: RuleOrLiteral): PrecRightRule;
+  right(value: String | number, rule: RuleOrLiteral): PrecRightRule;
 
   /**
    * Marks the given rule with a numerical precedence which will be used to
@@ -341,7 +300,7 @@ declare const prec: {
    *
    * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
    */
-  dynamic(value: number, rule: RuleOrLiteral): PrecDynamicRule;
+  dynamic(value: String | number, rule: RuleOrLiteral): PrecDynamicRule;
 };
 
 /**
@@ -361,15 +320,6 @@ declare function repeat(rule: RuleOrLiteral): RepeatRule;
  */
 declare function repeat1(rule: RuleOrLiteral): Repeat1Rule;
 
-/**
- * Overrides the global reserved word set for a given rule. The word set name
- * should be defined in the `reserved` field in the grammar.
- *
- * @param wordset name of the reserved word set
- * @param rule rule that will use the reserved word set
- */
-declare function reserved(wordset: string, rule: RuleOrLiteral): ReservedRule;
-
 /**
  * Creates a rule that matches any number of other rules, one after another.
  * It is analogous to simply writing multiple symbols next to each other
@@ -388,7 +338,7 @@ declare function sym<Name extends string>(name: Name): SymbolRule<Name>;
 
 /**
  * Marks the given rule as producing only a single token. Tree-sitter's
- * default is to treat each string or RegExp literal in the grammar as a
+ * default is to treat each String or RegExp literal in the grammar as a
  * separate token. Each token is matched separately by the lexer and
  * returned as its own leaf node in the tree. The token function allows
  * you to express a complex rule using the DSL functions (rather
@@ -412,19 +362,6 @@ declare const token: {
   immediate(rule: RuleOrLiteral): ImmediateTokenRule;
 };
 
-/**
- * Matches the end of input. May only appear as the final symbol of a
- * (possibly nested) sequence; a production ending in `eof()` reduces only
- * when the lookahead is end-of-input, rather than shifting a token.
- *
- * Choice branches that continue past `eof()` are dropped as unreachable,
- * and `eof()` is not allowed inside `token()`.
- *
- * Useful when a rule should match either an explicit terminator (e.g. a
- * newline) or the end of the file.
- */
-declare function eof(): EOFRule;
-
 /**
  * Creates a new language grammar with the provided schema.
  *

cli/npm/install.js

@@ -6,8 +6,7 @@ const http = require('http');
 const https = require('https');
 const packageJSON = require('./package.json');
 
-https.globalAgent.keepAlive = false;
-
+// Look to a results table in https://github.com/tree-sitter/tree-sitter/issues/2196
 const matrix = {
   platform: {
     'darwin': {

cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.28.0",
+  "version": "0.25.1",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"
@@ -14,20 +14,20 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
-    "directory": "crates/cli/npm"
+    "url": "https://github.com/tree-sitter/tree-sitter.git"
   },
   "description": "CLI for generating fast incremental parsers",
   "keywords": [
     "parser",
     "lexer"
   ],
+  "main": "lib/api/index.js",
   "engines": {
     "node": ">=12.0.0"
   },
   "scripts": {
     "install": "node install.js",
-    "prepack": "cp ../../../LICENSE ../README.md .",
+    "prepack": "cp ../../LICENSE ../README.md .",
     "postpack": "rm LICENSE README.md"
   },
   "bin": {

cli/src/fuzz/allocations.rs

@@ -2,21 +2,19 @@ use std::{
     collections::HashMap,
     os::raw::c_void,
     sync::{
-        Mutex,
         atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},
+        Mutex,
     },
 };
 
 #[ctor::ctor]
 unsafe fn initialize_allocation_recording() {
-    unsafe {
-        tree_sitter::set_allocator(Some(tree_sitter::Allocator {
-            malloc: ts_record_malloc,
-            calloc: ts_record_calloc,
-            realloc: ts_record_realloc,
-            free: ts_record_free,
-        }));
-    }
+    tree_sitter::set_allocator(
+        Some(ts_record_malloc),
+        Some(ts_record_calloc),
+        Some(ts_record_realloc),
+        Some(ts_record_free),
+    );
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -35,18 +33,14 @@ thread_local! {
     static RECORDER: AllocationRecorder = AllocationRecorder::default();
 }
 
-unsafe extern "C" {
+extern "C" {
     fn malloc(size: usize) -> *mut c_void;
     fn calloc(count: usize, size: usize) -> *mut c_void;
     fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
     fn free(ptr: *mut c_void);
 }
 
-pub fn record<T>(f: impl FnOnce() -> T) -> T {
-    record_checked(f).unwrap()
-}
-
-pub fn record_checked<T>(f: impl FnOnce() -> T) -> Result<T, String> {
+pub fn record<T>(f: impl FnOnce() -> T) -> Result<T, String> {
     RECORDER.with(|recorder| {
         recorder.enabled.store(true, SeqCst);
         recorder.allocation_count.store(0, SeqCst);
@@ -99,57 +93,30 @@ fn record_dealloc(ptr: *mut c_void) {
     });
 }
 
-/// # Safety
-///
-/// The caller must ensure that the returned pointer is eventually
-/// freed by calling `ts_record_free`.
-#[must_use]
-pub unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
-    unsafe {
-        let result = malloc(size);
+unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
+    let result = malloc(size);
+    record_alloc(result);
+    result
+}
+
+unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
+    let result = calloc(count, size);
+    record_alloc(result);
+    result
+}
+
+unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
+    let result = realloc(ptr, size);
+    if ptr.is_null() {
         record_alloc(result);
-        result
-    }
-}
-
-/// # Safety
-///
-/// The caller must ensure that the returned pointer is eventually
-/// freed by calling `ts_record_free`.
-#[must_use]
-pub unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
-    unsafe {
-        let result = calloc(count, size);
-        record_alloc(result);
-        result
-    }
-}
-
-/// # Safety
-///
-/// The caller must ensure that the returned pointer is eventually
-/// freed by calling `ts_record_free`.
-#[must_use]
-pub unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
-    unsafe {
-        let result = realloc(ptr, size);
-        if ptr.is_null() {
-            record_alloc(result);
-        } else if !core::ptr::eq(ptr, result) {
-            record_dealloc(ptr);
-            record_alloc(result);
-        }
-        result
-    }
-}
-
-/// # Safety
-///
-/// The caller must ensure that `ptr` was allocated by a previous call
-/// to `ts_record_malloc`, `ts_record_calloc`, or `ts_record_realloc`.
-pub unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
-    unsafe {
+    } else if ptr != result {
         record_dealloc(ptr);
-        free(ptr);
+        record_alloc(result);
     }
+    result
+}
+
+unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
+    record_dealloc(ptr);
+    free(ptr);
 }

cli/src/fuzz/corpus_test.rs

@@ -1,22 +1,10 @@
 use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
 
-use super::{LOG_ENABLED, LOG_GRAPH_ENABLED, scope_sequence::ScopeSequence};
+use super::{scope_sequence::ScopeSequence, LOG_ENABLED, LOG_GRAPH_ENABLED};
 use crate::util;
 
-struct SizeCheckFrame<'a> {
-    node: Node<'a>,
-    end_byte: usize,
-    end_point: Point,
-    child_count: u32,
-    child_index: u32,
-    last_child_end_byte: usize,
-    last_child_end_point: Point,
-    some_child_has_changes: bool,
-    actual_named_child_count: usize,
-}
-
-impl SizeCheckFrame<'_> {
-    fn new<'a>(node: Node<'a>, line_offsets: &[usize]) -> SizeCheckFrame<'a> {
+pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
+    fn check(node: Node, line_offsets: &[usize]) {
         let start_byte = node.start_byte();
         let end_byte = node.end_byte();
         let start_point = node.start_position();
@@ -30,21 +18,37 @@ impl SizeCheckFrame<'_> {
         );
         assert_eq!(end_byte, line_offsets[end_point.row] + end_point.column);
 
-        SizeCheckFrame {
-            node,
-            end_byte,
-            end_point,
-            child_count: node.child_count(),
-            child_index: 0,
-            last_child_end_byte: start_byte,
-            last_child_end_point: start_point,
-            some_child_has_changes: false,
-            actual_named_child_count: 0,
+        let mut last_child_end_byte = start_byte;
+        let mut last_child_end_point = start_point;
+        let mut some_child_has_changes = false;
+        let mut actual_named_child_count = 0;
+        for i in 0..node.child_count() {
+            let child = node.child(i).unwrap();
+            assert!(child.start_byte() >= last_child_end_byte);
+            assert!(child.start_position() >= last_child_end_point);
+            check(child, line_offsets);
+            if child.has_changes() {
+                some_child_has_changes = true;
+            }
+            if child.is_named() {
+                actual_named_child_count += 1;
+            }
+            last_child_end_byte = child.end_byte();
+            last_child_end_point = child.end_position();
+        }
+
+        assert_eq!(actual_named_child_count, node.named_child_count());
+
+        if node.child_count() > 0 {
+            assert!(end_byte >= last_child_end_byte);
+            assert!(end_point >= last_child_end_point);
+        }
+
+        if some_child_has_changes {
+            assert!(node.has_changes());
         }
     }
-}
 
-pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
     let mut line_offsets = vec![0];
     for (i, c) in input.iter().enumerate() {
         if *c == b'\n' {
@@ -52,41 +56,7 @@ pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
         }
     }
 
-    let mut stack: Vec<SizeCheckFrame> = vec![SizeCheckFrame::new(tree.root_node(), &line_offsets)];
-    while let Some(top) = stack.last_mut() {
-        if top.child_index < top.child_count {
-            let i = top.child_index;
-            let child = top.node.child(i).unwrap();
-
-            assert!(child.start_byte() >= top.last_child_end_byte);
-            assert!(child.start_position() >= top.last_child_end_point);
-            if child.has_changes() {
-                top.some_child_has_changes = true;
-            }
-            if child.is_named() {
-                top.actual_named_child_count += 1;
-            }
-            top.last_child_end_byte = child.end_byte();
-            top.last_child_end_point = child.end_position();
-            top.child_index += 1;
-
-            stack.push(SizeCheckFrame::new(child, &line_offsets));
-            continue;
-        }
-
-        let frame = stack.pop().unwrap();
-        assert_eq!(
-            frame.actual_named_child_count,
-            frame.node.named_child_count()
-        );
-        if frame.child_count > 0 {
-            assert!(frame.end_byte >= frame.last_child_end_byte);
-            assert!(frame.end_point >= frame.last_child_end_point);
-        }
-        if frame.some_child_has_changes {
-            assert!(frame.node.has_changes());
-        }
-    }
+    check(tree.root_node(), &line_offsets);
 }
 
 pub fn check_changed_ranges(old_tree: &Tree, new_tree: &Tree, input: &[u8]) -> Result<(), String> {

cli/src/fuzz/mod.rs

@@ -1,12 +1,6 @@
-use std::{
-    collections::HashMap,
-    env, fs,
-    path::{Path, PathBuf},
-    sync::LazyLock,
-};
+use std::{collections::HashMap, env, fs, path::Path, sync::LazyLock};
 
-use log::{error, info};
-use rand::RngExt;
+use rand::Rng;
 use regex::Regex;
 use tree_sitter::{Language, Parser};
 
@@ -25,7 +19,7 @@ use crate::{
         random::Rand,
     },
     parse::perform_edit,
-    test::{DiffKey, TestDiff, TestEntry, TestExpectation, parse_tests, render_test_output},
+    test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry},
 };
 
 pub static LOG_ENABLED: LazyLock<bool> = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok());
@@ -44,13 +38,11 @@ pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
 
 pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
 
-pub const DEFAULT_EDIT_COUNT: usize = 3;
 pub static EDIT_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(DEFAULT_EDIT_COUNT));
+    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
 
-pub const DEFAULT_ITERATION_COUNT: usize = 10;
 pub static ITERATION_COUNT: LazyLock<usize> =
-    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(DEFAULT_ITERATION_COUNT));
+    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
 
 fn int_env_var(name: &'static str) -> Option<usize> {
     env::var(name).ok().and_then(|e| e.parse().ok())
@@ -63,16 +55,14 @@ fn regex_env_var(name: &'static str) -> Option<Regex> {
 #[must_use]
 pub fn new_seed() -> usize {
     int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
-        let mut rng = rand::rng();
-        let seed = rng.random_range(0..=usize::MAX);
-        eprintln!("fuzz seed: {seed}");
-        seed
+        let mut rng = rand::thread_rng();
+        rng.gen::<usize>()
     })
 }
 
 pub struct FuzzOptions {
     pub skipped: Option<Vec<String>>,
-    pub subdir: Option<PathBuf>,
+    pub subdir: Option<String>,
     pub edits: usize,
     pub iterations: usize,
     pub include: Option<Regex>,
@@ -97,7 +87,9 @@ pub fn fuzz_language_corpus(
                         .iter()
                         .any(|lang| lang.as_ref() == language_name)
             }
-            TestEntry::Group { children, .. } => {
+            TestEntry::Group {
+                ref mut children, ..
+            } => {
                 children.retain_mut(|child| retain(child, language_name));
                 !children.is_empty()
             }
@@ -109,16 +101,12 @@ pub fn fuzz_language_corpus(
     let corpus_dir = grammar_dir.join(subdir).join("test").join("corpus");
 
     if !corpus_dir.exists() || !corpus_dir.is_dir() {
-        error!(
-            "No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file."
-        );
+        eprintln!("No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file.");
         return;
     }
 
     if std::fs::read_dir(&corpus_dir).unwrap().count() == 0 {
-        error!(
-            "No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory."
-        );
+        eprintln!("No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory.");
         return;
     }
 
@@ -144,7 +132,7 @@ pub fn fuzz_language_corpus(
         .take()
         .unwrap_or_default()
         .into_iter()
-        .chain(tests.iter().filter(|t| t.skip()).map(get_test_name))
+        .chain(tests.iter().filter(|x| x.skip).map(get_test_name))
         .map(|x| (x, 0))
         .collect::<HashMap<String, usize>>();
 
@@ -154,7 +142,7 @@ pub fn fuzz_language_corpus(
     let dump_edits = env::var("TREE_SITTER_DUMP_EDITS").is_ok();
 
     if log_seed {
-        info!("  start seed: {start_seed}");
+        println!("  start seed: {start_seed}");
     }
 
     println!();
@@ -168,12 +156,35 @@ pub fn fuzz_language_corpus(
 
         println!("  {test_index}. {test_name}");
 
-        let passed = allocations::record_checked(|| {
-            let check_output = !test.error();
-            test.check_initial_parse(language, &test_name, check_output)
+        let passed = allocations::record(|| {
+            let mut log_session = None;
+            let mut parser = get_parser(&mut log_session, "log.html");
+            parser.set_language(language).unwrap();
+            set_included_ranges(&mut parser, &test.input, test.template_delimiters);
+
+            let tree = parser.parse(&test.input, None).unwrap();
+
+            if test.error {
+                return true;
+            }
+
+            let mut actual_output = tree.root_node().to_sexp();
+            if !test.has_fields {
+                actual_output = strip_sexp_fields(&actual_output);
+            }
+
+            if actual_output != test.output {
+                println!("Incorrect initial parse for {test_name}");
+                print_diff_key();
+                print_diff(&actual_output, &test.output, true);
+                println!();
+                return false;
+            }
+
+            true
         })
         .unwrap_or_else(|e| {
-            error!("{e}");
+            eprintln!("Error: {e}");
             false
         });
 
@@ -189,7 +200,7 @@ pub fn fuzz_language_corpus(
 
         for trial in 0..options.iterations {
             let seed = start_seed + trial;
-            let passed = allocations::record_checked(|| {
+            let passed = allocations::record(|| {
                 let mut rand = Rand::new(seed);
                 let mut log_session = None;
                 let mut parser = get_parser(&mut log_session, "log.html");
@@ -198,20 +209,19 @@ pub fn fuzz_language_corpus(
                 let mut input = test.input.clone();
 
                 if options.log_graphs {
-                    info!("{}\n", String::from_utf8_lossy(&input));
+                    eprintln!("{}\n", String::from_utf8_lossy(&input));
                 }
 
                 // Perform a random series of edits and reparse.
-                let edit_count = rand.unsigned(options.edits);
-                let mut undo_stack = Vec::with_capacity(edit_count);
-                for _ in 0..=edit_count {
+                let mut undo_stack = Vec::new();
+                for _ in 0..=rand.unsigned(*EDIT_COUNT) {
                     let edit = get_random_edit(&mut rand, &input);
                     undo_stack.push(invert_edit(&input, &edit));
                     perform_edit(&mut tree, &mut input, &edit).unwrap();
                 }
 
                 if log_seed {
-                    info!("   {test_index}.{trial:<2} seed: {seed}");
+                    println!("   {test_index}.{trial:<2} seed: {seed}");
                 }
 
                 if dump_edits {
@@ -225,7 +235,7 @@ pub fn fuzz_language_corpus(
                 }
 
                 if options.log_graphs {
-                    info!("{}\n", String::from_utf8_lossy(&input));
+                    eprintln!("{}\n", String::from_utf8_lossy(&input));
                 }
 
                 set_included_ranges(&mut parser, &input, test.template_delimiters);
@@ -234,7 +244,7 @@ pub fn fuzz_language_corpus(
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
+                    println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
                     return false;
                 }
 
@@ -243,19 +253,22 @@ pub fn fuzz_language_corpus(
                     perform_edit(&mut tree2, &mut input, &edit).unwrap();
                 }
                 if options.log_graphs {
-                    info!("{}\n", String::from_utf8_lossy(&input));
+                    eprintln!("{}\n", String::from_utf8_lossy(&input));
                 }
 
                 set_included_ranges(&mut parser, &test.input, test.template_delimiters);
                 let tree3 = parser.parse(&input, Some(&tree2)).unwrap();
 
                 // Verify that the final tree matches the expectation from the corpus.
-                let actual_output = render_test_output(&input, &tree3, test.cst, test.has_fields).unwrap();
+                let mut actual_output = tree3.root_node().to_sexp();
+                if !test.has_fields {
+                    actual_output = strip_sexp_fields(&actual_output);
+                }
 
-                if actual_output != test.output && !test.error() {
+                if actual_output != test.output && !test.error {
                     println!("Incorrect parse for {test_name} - seed {seed}");
-                    DiffKey::print();
-                    println!("{}", TestDiff::new(&actual_output, &test.output));
+                    print_diff_key();
+                    print_diff(&actual_output, &test.output, true);
                     println!();
                     return false;
                 }
@@ -263,13 +276,13 @@ pub fn fuzz_language_corpus(
                 // Check that the edited tree is consistent.
                 check_consistent_sizes(&tree3, &input);
                 if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) {
-                    error!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
+                    println!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 
                 true
             }).unwrap_or_else(|e| {
-                error!("{e}");
+                eprintln!("Error: {e}");
                 false
             });
 
@@ -281,17 +294,17 @@ pub fn fuzz_language_corpus(
     }
 
     if failure_count != 0 {
-        info!("{failure_count} {language_name} corpus tests failed fuzzing");
+        eprintln!("{failure_count} {language_name} corpus tests failed fuzzing");
     }
 
     skipped.retain(|_, v| *v == 0);
 
     if !skipped.is_empty() {
-        info!("Non matchable skip definitions:");
+        println!("Non matchable skip definitions:");
         for k in skipped.keys() {
-            info!("  {k}");
+            println!("  {k}");
         }
-        panic!("Non matchable skip definitions need to be removed");
+        panic!("Non matchable skip definitions needs to be removed");
     }
 }
 
@@ -300,56 +313,12 @@ pub struct FlattenedTest {
     pub input: Vec<u8>,
     pub output: String,
     pub languages: Vec<Box<str>>,
-    pub expectation: TestExpectation,
+    pub error: bool,
+    pub skip: bool,
     pub has_fields: bool,
-    pub cst: bool,
     pub template_delimiters: Option<(&'static str, &'static str)>,
 }
 
-impl FlattenedTest {
-    #[must_use]
-    fn skip(&self) -> bool {
-        self.expectation == TestExpectation::Skip
-    }
-
-    #[must_use]
-    fn error(&self) -> bool {
-        self.expectation == TestExpectation::Error
-    }
-
-    #[must_use]
-    pub(crate) fn check_initial_parse(
-        &self,
-        language: &Language,
-        display_name: &str,
-        check_output: bool,
-    ) -> bool {
-        let mut log_session = None;
-        let mut parser = get_parser(&mut log_session, "log.html");
-        parser.set_language(language).unwrap();
-        set_included_ranges(&mut parser, &self.input, self.template_delimiters);
-
-        let tree = parser.parse(&self.input, None).unwrap();
-
-        if !check_output {
-            return true;
-        }
-
-        let actual_output =
-            render_test_output(&self.input, &tree, self.cst, self.has_fields).unwrap();
-
-        if actual_output == self.output {
-            true
-        } else {
-            println!("Incorrect initial parse for {display_name}");
-            DiffKey::print();
-            println!("{}", TestDiff::new(&actual_output, &self.output));
-            println!();
-            false
-        }
-    }
-}
-
 #[must_use]
 pub fn flatten_tests(
     test: TestEntry,
@@ -382,20 +351,20 @@ pub fn flatten_tests(
                     if !include.is_match(&name) {
                         return;
                     }
-                } else if let Some(exclude) = exclude
-                    && exclude.is_match(&name)
-                {
-                    return;
+                } else if let Some(exclude) = exclude {
+                    if exclude.is_match(&name) {
+                        return;
+                    }
                 }
 
                 result.push(FlattenedTest {
                     name,
                     input,
                     output,
-                    languages: attributes.languages,
-                    expectation: attributes.expectation,
                     has_fields,
-                    cst: attributes.cst,
+                    languages: attributes.languages,
+                    error: attributes.error,
+                    skip: attributes.skip,
                     template_delimiters: None,
                 });
             }

cli/src/fuzz/random.rs

@@ -1,4 +1,7 @@
-use rand::{RngExt, SeedableRng, distr::Alphanumeric, rngs::StdRng};
+use rand::{
+    distributions::Alphanumeric,
+    prelude::{Rng, SeedableRng, StdRng},
+};
 
 const OPERATORS: &[char] = &[
     '+', '-', '<', '>', '(', ')', '*', '/', '&', '|', '!', ',', '.', '%',
@@ -13,12 +16,12 @@ impl Rand {
     }
 
     pub fn unsigned(&mut self, max: usize) -> usize {
-        self.0.random_range(0..=max)
+        self.0.gen_range(0..=max)
     }
 
     pub fn words(&mut self, max_count: usize) -> Vec<u8> {
+        let mut result = Vec::new();
         let word_count = self.unsigned(max_count);
-        let mut result = Vec::with_capacity(2 * word_count);
         for i in 0..word_count {
             if i > 0 {
                 if self.unsigned(5) == 0 {

cli/src/fuzz/scope_sequence.rs

@@ -1,13 +1,13 @@
 use tree_sitter::{Point, Range, Tree};
 
 #[derive(Debug)]
-pub struct ScopeSequence<'a>(Vec<ScopeStack<'a>>);
+pub struct ScopeSequence(Vec<ScopeStack>);
 
-type ScopeStack<'a> = Vec<&'a str>;
+type ScopeStack = Vec<&'static str>;
 
-impl<'a> ScopeSequence<'a> {
+impl ScopeSequence {
     #[must_use]
-    pub fn new(tree: &'a Tree) -> Self {
+    pub fn new(tree: &Tree) -> Self {
         let mut result = Self(Vec::new());
         let mut scope_stack = Vec::new();
 
@@ -49,7 +49,7 @@ impl<'a> ScopeSequence<'a> {
         for i in 0..(self.0.len().max(other.0.len())) {
             let stack = &self.0.get(i);
             let other_stack = &other.0.get(i);
-            if *stack != *other_stack && !b"\r\n".contains(&text[i]) {
+            if *stack != *other_stack && ![b'\r', b'\n'].contains(&text[i]) {
                 let containing_range = known_changed_ranges
                     .iter()
                     .find(|range| range.start_point <= position && position < range.end_point);