build(deps): bump rquickjs to v0.13.0

Also use `IoError` more consistently throughout the rest of the project
where its straightforward to do so.

BREAKING CHANGE: Changes public error types for config, generate, and
loader crates.

.dockerignore

@@ -1,2 +0,0 @@
-target
-.git

.editorconfig

@@ -10,6 +10,9 @@ insert_final_newline = true
 [*.rs]
 indent_size = 4
 
+[*.{zig,zon}]
+indent_size = 4
+
 [Makefile]
 indent_style = tab
 indent_size = 8

.envrc

@@ -0,0 +1,4 @@
+source_up_if_exists
+if [ -z "${IN_NIX_SHELL:-}" ]; then
+  use flake .
+fi

.gitattributes

@@ -3,5 +3,4 @@
 /lib/src/unicode/*.h linguist-vendored
 /lib/src/unicode/LICENSE linguist-vendored
 
-/cli/src/generate/prepare_grammar/*.json -diff
 Cargo.lock -diff

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,7 +1,15 @@
 name: Bug Report
 description: Report a problem
-labels: [bug]
+type: Bug
 body:
+  - type: checkboxes
+    attributes:
+      label: AI Policy
+      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
+      options:
+        - label: I have read the AI Policy and this issue complies with it.
+          required: true
+
   - type: textarea
     attributes:
       label: "Problem"

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,7 +1,15 @@
 name: Feature request
 description: Request an enhancement
-labels: [enhancement]
+type: Feature
 body:
+  - type: checkboxes
+    attributes:
+      label: AI Policy
+      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
+      options:
+        - label: I have read the AI Policy and this issue complies with it.
+          required: true
+
   - type: markdown
     attributes:
       value: |

.github/actions/cache/action.yml

@@ -10,17 +10,20 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
       id: cache
       with:
         path: |
           test/fixtures/grammars
           target/release/tree-sitter-*.wasm
         key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles(
-          'cli/generate/src/**',
+          'crates/generate/src/**',
           'lib/src/parser.h',
           'lib/src/array.h',
           'lib/src/alloc.h',
-          'xtask/src/*',
+          'lib/src/wasm-stdlib/external_scanner_stdlib.h',
+          'crates/loader/wasi-sdk-version',
+          'crates/loader/binaryen-version',
           'test/fixtures/grammars/*/**/src/*.c',
+          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}

.github/cliff.toml

@@ -43,16 +43,16 @@ commit_preprocessors = [
 ]
 # regex for parsing and grouping commits
 commit_parsers = [
-  { message = "!:", group = "<!-- 0 -->Breaking" },
-  { message = "^feat", group = "<!-- 1 -->Features" },
-  { message = "^fix", group = "<!-- 2 -->Bug Fixes" },
-  { message = "^perf", group = "<!-- 3 -->Performance" },
-  { message = "^doc", group = "<!-- 4 -->Documentation" },
-  { message = "^refactor", group = "<!-- 5 -->Refactor" },
-  { message = "^test", group = "<!-- 6 -->Testing" },
-  { message = "^build", group = "<!-- 7 -->Build System and CI" },
-  { message = "^ci", group = "<!-- 7 -->Build System and CI" },
-  { message = ".*", group = "<!-- 8 -->Other" },
+  { group = "<!-- 0 -->Breaking", message = "!:" },
+  { group = "<!-- 1 -->Features", message = "^feat" },
+  { group = "<!-- 2 -->Bug Fixes", message = "^fix" },
+  { group = "<!-- 3 -->Performance", message = "^perf" },
+  { group = "<!-- 4 -->Documentation", message = "^doc" },
+  { group = "<!-- 5 -->Refactor", message = "^refactor" },
+  { group = "<!-- 6 -->Testing", message = "^test" },
+  { group = "<!-- 7 -->Build System and CI", message = "^build" },
+  { group = "<!-- 7 -->Build System and CI", message = "^ci" },
+  { group = "<!-- 8 -->Other", message = ".*" },
 ]
 # filter out the commits that are not matched by commit parsers
 filter_commits = false

.github/dependabot.yml

@@ -4,6 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 3
     commit-message:
       prefix: "build(deps)"
     labels:
@@ -12,10 +14,16 @@ updates:
     groups:
       cargo:
         patterns: ["*"]
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major", "version-update:semver-minor"]
+
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 3
     commit-message:
       prefix: "ci"
     labels:
@@ -24,3 +32,22 @@ updates:
     groups:
       actions:
         patterns: ["*"]
+
+  - package-ecosystem: "npm"
+    versioning-strategy: increase
+    directories:
+      - "/crates/npm"
+      - "/crates/eslint"
+      - "/lib/binding_web"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 3
+    commit-message:
+      prefix: "build(deps)"
+    labels:
+      - "dependencies"
+      - "npm"
+    groups:
+      npm:
+        patterns: ["*"]

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+### AI Policy
+
+- [ ] I have read the [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy) and this PR complies with it.
+- [ ] If AI tools were used: I have disclosed the tool and extent of usage below.
+
+<!-- If you used AI tools, state which tool and how it was used. Delete this section if not applicable. -->

.github/scripts/close_spam.js

@@ -0,0 +1,29 @@
+module.exports = async ({ github, context }) => {
+  let target = context.payload.issue;
+  if (target) {
+    await github.rest.issues.update({
+      ...context.repo,
+      issue_number: target.number,
+      state: "closed",
+      state_reason: "not_planned",
+      title: "[spam]",
+      body: "",
+      type: null,
+    });
+  } else {
+    target = context.payload.pull_request;
+    await github.rest.pulls.update({
+      ...context.repo,
+      pull_number: target.number,
+      state: "closed",
+      title: "[spam]",
+      body: "",
+    });
+  }
+
+  await github.rest.issues.lock({
+    ...context.repo,
+    issue_number: target.number,
+    lock_reason: "spam",
+  });
+};

.github/scripts/cross.sh

@@ -1,3 +0,0 @@
-#!/bin/bash -eu
-
-exec docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/scripts/make.sh

@@ -1,9 +0,0 @@
-#!/bin/bash -eu
-
-tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
-
-if [[ $BUILD_CMD == cross ]]; then
-  cross.sh make CC="$CC" AR="$AR" "$@"
-else
-  exec make "$@"
-fi

.github/scripts/reviewers_remove.js

@@ -1,16 +0,0 @@
-module.exports = async ({ github, context }) => {
-  const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
-    owner: context.repo.owner,
-    repo: context.repo.repo,
-    pull_number: context.issue.number,
-  });
-
-  const reviewers = requestedReviewers.data.users.map((e) => e.login);
-
-  github.rest.pulls.removeRequestedReviewers({
-    owner: context.repo.owner,
-    repo: context.repo.repo,
-    pull_number: context.issue.number,
-    reviewers: reviewers,
-  });
-};

.github/scripts/tree-sitter.sh

@@ -1,9 +0,0 @@
-#!/bin/bash -eu
-
-tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
-
-if [[ $BUILD_CMD == cross ]]; then
-  cross.sh "$CROSS_RUNNER" "$tree_sitter" "$@"
-else
-  exec "$tree_sitter" "$@"
-fi

.github/scripts/wasm_stdlib.js

@@ -0,0 +1,35 @@
+module.exports = async ({ github, context, core }) => {
+  if (context.eventName !== 'pull_request') return;
+
+  const prNumber = context.payload.pull_request.number;
+  const owner = context.repo.owner;
+  const repo = context.repo.repo;
+
+  const files = await github.paginate(github.rest.pulls.listFiles, {
+    owner,
+    repo,
+    pull_number: prNumber,
+    per_page: 100
+  });
+
+  const changedFiles = files.map(file => file.filename);
+
+  const wasmStdLibSources = [
+    'lib/src/wasm-stdlib/external_scanner_allocator.c',
+    'lib/src/wasm-stdlib/imports.txt',
+    'lib/src/wasm-stdlib/libc.c',
+    'lib/src/wasm-stdlib/stdio.c'
+  ];
+  const dirChanged = changedFiles.some(file =>
+    wasmStdLibSources.includes(file) ||
+    file.startsWith('lib/src/wasm-stdlib/libc/ctype/') ||
+    file.startsWith('lib/src/wasm-stdlib/libc/string/')
+  );
+
+  if (!dirChanged) return;
+
+  const wasmStdLibHeader = 'lib/src/wasm-stdlib/external_scanner_stdlib.h';
+  const requiredChanged = changedFiles.includes(wasmStdLibHeader);
+
+  if (!requiredChanged) core.setFailed(`Changes detected in the Wasm stdlib sources but ${wasmStdLibHeader} was not modified.`);
+};

.github/workflows/backport.yml

@@ -14,17 +14,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
+        with:
+          persist-credentials: true
+          ref: ${{ github.event.pull_request.base.ref }}
 
       - name: Create app token
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ vars.BACKPORT_APP }}
           private-key: ${{ secrets.BACKPORT_KEY }}
 
       - name: Create backport PR
-        uses: korthout/backport-action@v3
+        uses: korthout/backport-action@v4.6.0
         with:
           pull_title: "${pull_title}"
           label_pattern: "^ci:backport ([^ ]+)$"

.github/workflows/bindgen.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1

.github/workflows/build.yml

@@ -1,10 +1,5 @@
 name: Build & Test
 
-env:
-  CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-D warnings"
-  CROSS_DEBUG: 1
-
 on:
   workflow_call:
     inputs:
@@ -31,38 +26,46 @@ jobs:
           - windows-x86
           - macos-arm64
           - macos-x64
+          - illumos-x64
+          - wasm32
 
         include:
           # When adding a new `target`:
           # 1. Define a new platform alias above
-          # 2. Add a new record to the matrix map in `cli/npm/install.js`
-          - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu   , os: ubuntu-latest    , use-cross: true }
-          - { platform: linux-arm         , target: arm-unknown-linux-gnueabi   , os: ubuntu-latest    , use-cross: true }
-          - { platform: linux-x64         , target: x86_64-unknown-linux-gnu    , os: ubuntu-22.04     , features: wasm  }
-          - { platform: linux-x86         , target: i686-unknown-linux-gnu      , os: ubuntu-latest    , use-cross: true }
-          - { platform: linux-powerpc64   , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest    , use-cross: true }
-          - { platform: windows-arm64     , target: aarch64-pc-windows-msvc     , os: windows-latest                     }
-          - { platform: windows-x64       , target: x86_64-pc-windows-msvc      , os: windows-latest   , features: wasm  }
-          - { platform: windows-x86       , target: i686-pc-windows-msvc        , os: windows-latest                     }
-          - { platform: macos-arm64       , target: aarch64-apple-darwin        , os: macos-latest     , features: wasm  }
-          - { platform: macos-x64         , target: x86_64-apple-darwin         , os: macos-13         , features: wasm  }
+          # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
+          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
+          - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
+          - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
+          - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
+          - { platform: linux-x86       , target: i686-unknown-linux-gnu        , os: ubuntu-24.04     }
+          - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu   , os: ubuntu-24.04     }
+          - { platform: windows-arm64   , target: aarch64-pc-windows-msvc       , os: windows-11-arm   }
+          - { platform: windows-x64     , target: x86_64-pc-windows-msvc        , os: windows-2025     }
+          - { platform: windows-x86     , target: i686-pc-windows-msvc          , os: windows-2025     }
+          - { platform: macos-arm64     , target: aarch64-apple-darwin          , os: macos-15         }
+          - { platform: macos-x64       , target: x86_64-apple-darwin           , os: macos-15-intel   }
+          - { platform: wasm32          , target: wasm32-unknown-unknown        , os: ubuntu-24.04     }
 
-          # Cross compilers for C library
-          - { platform: linux-arm64       , cc: aarch64-linux-gnu-gcc           , ar: aarch64-linux-gnu-ar   }
-          - { platform: linux-arm         , cc: arm-linux-gnueabi-gcc           , ar: arm-linux-gnueabi-ar   }
-          - { platform: linux-x86         , cc: i686-linux-gnu-gcc              , ar: i686-linux-gnu-ar      }
-          - { platform: linux-powerpc64   , cc: powerpc64-linux-gnu-gcc         , ar: powerpc64-linux-gnu-ar }
+          # illumos is not supported OOTB, it runs in a vm
+          - { platform: illumos-x64     , target: x86_64-unknown-illumos        , os: ubuntu-24.04     , vm: true , no-run: true }
 
-          # Prevent race condition (see #2041)
-          - { platform: windows-x64   , rust-test-threads: 1 }
-          - { platform: windows-x86   , rust-test-threads: 1 }
+          # Extra features
+          - { platform: linux-arm64     , features: wasm }
+          - { platform: linux-x64       , features: wasm , run-wasm-test: true }
+          - { platform: macos-arm64     , features: wasm , run-wasm-test: true }
+          - { platform: macos-x64       , features: wasm }
 
-          # Can't natively run CLI on Github runner's host
-          - { platform: windows-arm64 , no-run: true }
+          # Cross-compilation
+          - { platform: linux-arm       , cross: true }
+          - { platform: linux-x86       , cross: true }
+          - { platform: linux-powerpc64 , cross: true }
+
+          # Compile-only
+          - { platform: wasm32          , no-run: true }
 
     env:
-      BUILD_CMD: cargo
-      SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
+      CARGO_TERM_COLOR: always
+      RUSTFLAGS: -D warnings
 
     defaults:
       run:
@@ -70,122 +73,186 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7.0.1
 
-    - name: Read Emscripten version
-      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<cli/loader/emscripten-version)" >> $GITHUB_ENV
+    - name: Set up cross-compilation
+      if: matrix.cross
+      run: |
+        for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do
+          camel_target=${target//-/_}; target_cc=${target/-unknown/}
+          printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
+          printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
+          printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
+        done >> $GITHUB_ENV
+        {
+          printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
+          printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
+        } >> $GITHUB_ENV
+
+    - name: Get emscripten version
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
+
+    - name: Cache Emscripten SDK
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/cache@v6
+      with:
+        path: emsdk
+        key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
 
     - name: Install Emscripten
-      if: ${{ !matrix.no-run && !matrix.use-cross }}
-      uses: mymindstorm/setup-emsdk@v14
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      run: |
+        if [[ ! -d emsdk ]]; then
+          git clone --depth 1 https://github.com/emscripten-core/emsdk.git
+        fi
+        cd emsdk
+        ./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
+        ./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
+        echo "$PWD" >> "$GITHUB_PATH"
+        echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
+        echo "EMSDK=$PWD" >> "$GITHUB_ENV"
+        echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
+
+    - name: Set up Node.js
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/setup-node@v7.0.0
       with:
-        version: ${{ env.EMSCRIPTEN_VERSION }}
+        node-version: 24
+        cache: npm
+        cache-dependency-path: lib/binding_web/package-lock.json
 
     - name: Set up Rust
+      if: ${{ !matrix.vm }}
       uses: actions-rust-lang/setup-rust-toolchain@v1
       with:
         target: ${{ matrix.target }}
 
-    - name: Install cross
-      if: ${{ matrix.use-cross }}
+    - name: Install Rust Wasm test target
+      if: matrix.run-wasm-test
+      run: rustup toolchain install nightly --profile minimal --component rust-src
+
+    - name: Install cross-compilation toolchain
+      if: matrix.cross
       run: |
-        if [ ! -x "$(command -v cross)" ]; then
-          # TODO: Remove 'RUSTFLAGS=""' once https://github.com/cross-rs/cross/issues/1561 is resolved
-          RUSTFLAGS="" cargo install cross --git https://github.com/cross-rs/cross
+        sudo apt-get update -qy
+        if [[ $PLATFORM == linux-arm ]]; then
+          sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user
+        elif [[ $PLATFORM == linux-x86 ]]; then
+          sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu
+        elif [[ $PLATFORM == linux-powerpc64 ]]; then
+          sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user
         fi
-
-    - name: Configure cross
-      if: ${{ matrix.use-cross }}
-      run: |
-        printf '%s\n' > Cross.toml \
-          '[target.${{ matrix.target }}]'                                   \
-          'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"'            \
-          '[build]'                                                         \
-          'pre-build = ['                                                   \
-          '  "dpkg --add-architecture $CROSS_DEB_ARCH",'                    \
-          '  "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",'  \
-          '  "apt-get update && apt-get -y install libssl-dev nodejs"'      \
-          ']'
-        cat - Cross.toml <<< 'Cross.toml:'
-        printf '%s\n' >> $GITHUB_ENV \
-          "CROSS_CONFIG=$PWD/Cross.toml" \
-          "CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge"
-
-    - name: Set up environment
       env:
-        RUST_TEST_THREADS: ${{ matrix.rust-test-threads }}
-        USE_CROSS: ${{ matrix.use-cross }}
-        TARGET: ${{ matrix.target }}
-        CC: ${{ matrix.cc }}
-        AR: ${{ matrix.ar }}
+        PLATFORM: ${{ matrix.platform }}
+
+    - name: Install MinGW and Clang (Windows x64 MSYS2)
+      if: matrix.platform == 'windows-x64'
+      uses: msys2/setup-msys2@v2
+      with:
+        install: |
+          mingw-w64-x86_64-toolchain
+          mingw-w64-x86_64-clang
+          mingw-w64-x86_64-make
+          mingw-w64-x86_64-cmake
+
+    - name: Build C library (Windows x64 MSYS2 CMake)
+      if: matrix.platform == 'windows-x64'
+      shell: msys2 {0}
       run: |
-        PATH="$PWD/.github/scripts:$PATH"
-        printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH
+        cmake -G Ninja -S . -B build/static \
+          -DBUILD_SHARED_LIBS=OFF \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM \
+          -DCMAKE_C_COMPILER=clang
+        cmake --build build/static
 
-        printf '%s\n' >> $GITHUB_ENV \
-          'TREE_SITTER=tree-sitter.sh' \
-          "TARGET=$TARGET" \
-          "ROOT=$PWD"
+        cmake -G Ninja -S . -B build/shared \
+          -DBUILD_SHARED_LIBS=ON \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM \
+          -DCMAKE_C_COMPILER=clang
+        cmake --build build/shared
+        rm -rf \
+          build/{static,shared} \
+          "${CMAKE_PREFIX_PATH}/artifacts" \
+          target/wasmtime-${WASMTIME_VERSION}
+      env:
+        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
 
-        [[ -n $RUST_TEST_THREADS ]] && \
-          printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV
-
-        [[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV
-        [[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV
-
-        if [[ $USE_CROSS == true ]]; then
-          printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV
-          runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'")
-          [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV
-        fi
-
-    - name: Build wasmtime library
-      if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }}
+    - name: Download wasmtime C API
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       run: |
         mkdir -p target
         WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
                            jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
-        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
-        cd target/wasmtime-${WASMTIME_VERSION}
-        cmake -S crates/c-api -B target/c-api \
-          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
-          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
-          -DWASMTIME_FEATURE_CRANELIFT=ON \
-          -DWASMTIME_TARGET='${{ matrix.target }}'
-        cmake --build target/c-api && cmake --install target/c-api
-        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
+        case '${{ matrix.target }}' in
+          x86_64-unknown-linux-gnu)  WT_TARGET=x86_64-linux ;;
+          aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
+          x86_64-apple-darwin)       WT_TARGET=x86_64-macos ;;
+          aarch64-apple-darwin)      WT_TARGET=aarch64-macos ;;
+        esac
+        curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
+          | tar xJf - -C target
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
       env:
         WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
 
     - name: Build C library (make)
-      if: ${{ runner.os != 'Windows' }}
-      run: make.sh -j CFLAGS="$CFLAGS"
+      if: runner.os != 'Windows' && !matrix.vm
+      run: |
+        if [[ $PLATFORM == linux-arm ]]; then
+          CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar
+        elif [[ $PLATFORM == linux-x86 ]]; then
+          CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar
+        elif [[ $PLATFORM == linux-powerpc64 ]]; then
+          CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar
+        else
+          CC=gcc; AR=ar
+        fi
+        make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
       env:
-        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+        PLATFORM: ${{ matrix.platform }}
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
     - name: Build C library (CMake)
-      if: ${{ !matrix.use-cross }}
+      if: "!matrix.cross && !matrix.vm"
       run: |
-        cmake -S lib -B build/static \
+        cmake -S . -B build/static \
           -DBUILD_SHARED_LIBS=OFF \
           -DCMAKE_BUILD_TYPE=Debug \
           -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
           -DTREE_SITTER_FEATURE_WASM=$WASM
         cmake --build build/static --verbose
 
-        cmake -S lib -B build/shared \
+        cmake -S . -B build/shared \
           -DBUILD_SHARED_LIBS=ON \
           -DCMAKE_BUILD_TYPE=Debug \
           -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
           -DTREE_SITTER_FEATURE_WASM=$WASM
         cmake --build build/shared --verbose
       env:
-        CC: ${{ contains(matrix.target, 'linux') && 'clang' || '' }}
-        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
+        CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
+        WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
 
-    - name: Build wasm library
-      # No reason to build on the same Github runner hosts many times
-      if: ${{ !matrix.no-run && !matrix.use-cross }}
+    - name: Build C library and Rust crate (illumos gmake)
+      if: matrix.platform == 'illumos-x64'
+      uses: vmactions/omnios-vm@v1.3.6
+      with:
+        release: r151056-build
+        copyback: false
+        prepare: |
+          pkg install -q build-essential || [ $? -eq 4 ]
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
+        run: |
+          . "$HOME/.cargo/env"
+          gmake -j
+          cargo build -p tree-sitter
+
+    - name: Build Wasm library
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       shell: bash
       run: |
         cd lib/binding_web
@@ -195,64 +262,77 @@ jobs:
         npm run build
         npm run build:debug
 
+    - name: Check no_std builds
+      if: inputs.run-test && !matrix.no-run
+      working-directory: lib
+      shell: bash
+      run: cargo check --no-default-features --target='${{ matrix.target }}'
+
     - name: Build target
-      run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }}
+      if: "!inputs.run-test && !matrix.vm"
+      run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
+      env:
+        PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
 
     - name: Cache fixtures
       id: cache
-      if: ${{ !matrix.no-run && inputs.run-test }}
+      if: inputs.run-test && !matrix.no-run
       uses: ./.github/actions/cache
 
     - name: Fetch fixtures
-      if: ${{ !matrix.no-run && inputs.run-test }}
-      run: $BUILD_CMD run -p xtask -- fetch-fixtures
+      if: inputs.run-test && !matrix.no-run
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures
 
     - name: Generate fixtures
-      if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }}
-      run: $BUILD_CMD run -p xtask -- generate-fixtures
+      if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true'
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
 
     - name: Generate Wasm fixtures
-      if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }}
-      run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
 
     - name: Run main tests
-      if: ${{ !matrix.no-run && inputs.run-test }}
-      run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }}
+      if: inputs.run-test && !matrix.no-run
+      run: cargo test --workspace --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
 
-    - name: Run wasm tests
-      if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }}
-      run: $BUILD_CMD run -p xtask -- test-wasm
+    - name: Run Wasm tests
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
 
-    - name: Run benchmarks
-      # Cross-compiled benchmarks are pointless
-      if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }}
-      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
+    - name: Run Rust Wasm web test
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-rust-wasm-web
 
     - name: Upload CLI artifact
-      uses: actions/upload-artifact@v4
+      if: "!inputs.run-test && !matrix.no-run"
+      uses: actions/upload-artifact@v7
       with:
         name: tree-sitter.${{ matrix.platform }}
-        path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
         if-no-files-found: error
         retention-days: 7
 
     - name: Upload Wasm artifacts
-      if: ${{ matrix.platform == 'linux-x64' }}
-      uses: actions/upload-artifact@v4
+      if: "!inputs.run-test && matrix.platform == 'linux-x64'"
+      uses: actions/upload-artifact@v7
       with:
         name: tree-sitter.wasm
         path: |
-          lib/binding_web/tree-sitter.js
-          lib/binding_web/tree-sitter.js.map
-          lib/binding_web/tree-sitter.cjs
-          lib/binding_web/tree-sitter.cjs.map
-          lib/binding_web/tree-sitter.wasm
-          lib/binding_web/tree-sitter.wasm.map
-          lib/binding_web/debug/tree-sitter.cjs
-          lib/binding_web/debug/tree-sitter.cjs.map
-          lib/binding_web/debug/tree-sitter.js
-          lib/binding_web/debug/tree-sitter.js.map
-          lib/binding_web/debug/tree-sitter.wasm
-          lib/binding_web/debug/tree-sitter.wasm.map
+          lib/binding_web/web-tree-sitter.js
+          lib/binding_web/web-tree-sitter.js.map
+          lib/binding_web/web-tree-sitter.cjs
+          lib/binding_web/web-tree-sitter.cjs.map
+          lib/binding_web/web-tree-sitter.wasm
+          lib/binding_web/web-tree-sitter.wasm.map
+          lib/binding_web/debug/web-tree-sitter.cjs
+          lib/binding_web/debug/web-tree-sitter.cjs.map
+          lib/binding_web/debug/web-tree-sitter.js
+          lib/binding_web/debug/web-tree-sitter.js.map
+          lib/binding_web/debug/web-tree-sitter.wasm
+          lib/binding_web/debug/web-tree-sitter.wasm.map
+          lib/binding_web/lib/*.c
+          lib/binding_web/lib/*.h
+          lib/binding_web/lib/*.ts
+          lib/binding_web/src/*.ts
         if-no-files-found: error
         retention-days: 7

.github/workflows/ci.yml

@@ -1,21 +1,14 @@
 name: CI
 
 on:
-  pull_request:
-    paths-ignore:
-      - docs/**
-      - "**/README.md"
-      - CONTRIBUTING.md
-      - LICENSE
-      - cli/src/templates
   push:
-    branches: [master]
-    paths-ignore:
-      - docs/**
-      - "**/README.md"
-      - CONTRIBUTING.md
-      - LICENSE
-      - cli/src/templates
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
+  pull_request:
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -26,26 +19,33 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: stable
-
-      - name: Set up nightly Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: nightly
           components: clippy, rustfmt
 
-      - name: Lint files
-        run: |
-          make lint
-          make lint-web
+      - name: Lint Rust files
+        run: make lint
+
+      - name: Install Taplo
+        uses: taiki-e/install-action@v2
+        with:
+          tool: taplo@0.10.0
+
+      - name: Lint TOML files
+        run: make lint-toml
+
+      - name: Lint web files
+        run: make lint-web
 
   sanitize:
     uses: ./.github/workflows/sanitize.yml
 
   build:
     uses: ./.github/workflows/build.yml
+
+  check-wasm-stdlib:
+    uses: ./.github/workflows/wasm_stdlib.yml

.github/workflows/crate_versions.yml

@@ -0,0 +1,22 @@
+name: Crate Versions Check
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+  workflow_dispatch:
+
+jobs:
+  check-crates:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Check crates against crates.io
+        uses: katyo/publish-crates@v2
+        with:
+          dry-run: true

.github/workflows/docs.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches: [master]
     paths: [docs/**]
+  workflow_dispatch:
 
 jobs:
   deploy-docs:
@@ -15,35 +16,29 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+        uses: actions/checkout@v7.0.1
 
       - name: Install mdbook
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
-          url=$(gh api repos/rust-lang/mdbook/releases/latest --jq "$jq_expr")
+          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.5.4 --jq "$jq_expr")
           mkdir mdbook
           curl -sSL "$url" | tar -xz -C mdbook
           printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
 
-      - name: Install mdbook-admonish
-        run: cargo install mdbook-admonish
-
       - name: Build Book
         run: mdbook build docs
 
       - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@v6
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v5
         with:
           path: docs/book
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v5

.github/workflows/emscripten.yml

@@ -1,30 +0,0 @@
-name: Update Emscripten
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-permissions:
-  contents: write
-  pull-requests: read
-
-jobs:
-  update-emscripten:
-    if: github.actor == 'dependabot[bot]'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Set up stable Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Run emscripten update xtask
-        run: |
-          git config --global user.name "dependabot[bot]"
-          git config --global user.email "49699333+dependabot[bot]@users.noreply.github.com"
-          cargo xtask upgrade-emscripten
-
-      - name: Push updated version
-        run: git push origin HEAD:$GITHUB_HEAD_REF

.github/workflows/nvim_ts.yml

@@ -3,7 +3,11 @@ name: nvim-treesitter parser tests
 on:
   pull_request:
     paths:
-      - 'cli/**'
+      - 'lib/**'
+      - 'crates/cli/**'
+      - 'crates/config/**'
+      - 'crates/generate/**'
+      - 'crates/loader/**'
       - '.github/workflows/nvim_ts.yml'
   workflow_dispatch:
 
@@ -13,7 +17,7 @@ concurrency:
 
 jobs:
   check_compilation:
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -22,45 +26,71 @@ jobs:
     name: ${{ matrix.os }} - ${{ matrix.type }}
     runs-on: ${{ matrix.os }}
     env:
-      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }}
+      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim.exe' || 'nvim' }}
+      NVIM_TAG: stable
+      NVIM_DIR: neovim
       NVIM_TS_DIR: nvim-treesitter
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7.0.1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build --profile optimize
 
-      - uses: actions/checkout@v4
+      - name: Clone Neovim
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: neovim/neovim
+          ref: ${{ env.NVIM_TAG }}
+          path: ${{ env.NVIM_DIR }}
+
+      - if: runner.os != 'Windows'
+        name: Setup environment (Posix)
+        run: |
+          echo ${{ github.workspace }}/target/optimize >> "$GITHUB_PATH"
+          echo ${{ github.workspace }}/neovim/build/bin >> "$GITHUB_PATH"
+          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$GITHUB_ENV"
+
+      - if: runner.os == 'Windows'
+        name: Setup environment (why can't you just be normal?!)
+        run: |
+          ${{ env.NVIM_DIR }}/.github/scripts/env.ps1
+          echo ${{ github.workspace }}/target/optimize >> "$env:GITHUB_PATH"
+          echo ${{ github.workspace }}/neovim/build/bin >> "$env:GITHUB_PATH"
+          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$env:GITHUB_ENV"
+
+      - name: Build Neovim
+        working-directory: ${{ env.NVIM_DIR }}
+        run: |
+          cmake -S cmake.deps -B .deps -G Ninja -D CMAKE_BUILD_TYPE=Release -D TREESITTER_URL=https://github.com/tree-sitter/tree-sitter/archive/${{ github.event.pull_request.head.sha }}.tar.gz -D DEPS_IGNORE_SHA=TRUE
+          cmake --build .deps --config Release
+          cmake -B build -G Ninja -D CMAKE_BUILD_TYPE=Release
+          cmake --build build --config Release
+
+      - name: Clone nvim-treesitter
+        uses: actions/checkout@v7.0.1
         with:
           repository: nvim-treesitter/nvim-treesitter
           path: ${{ env.NVIM_TS_DIR }}
-          ref: main
-
-      - if: runner.os != 'Windows'
-        run: echo ${{ github.workspace }}/target/release >> $GITHUB_PATH
-
-      - if: runner.os == 'Windows'
-        run: echo ${{ github.workspace }}/target/release >> $env:GITHUB_PATH
-
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - run: cargo build --release
-      - uses: ilammy/msvc-dev-cmd@v1
-
-      - name: Install and prepare Neovim
-        run: bash ./scripts/ci-install.sh
-        working-directory: ${{ env.NVIM_TS_DIR }}
 
       - if: matrix.type == 'generate'
         name: Generate and compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=2
+        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=10
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash
 
       - if: matrix.type == 'build'
         name: Compile parsers
-        run: $NVIM -l ./scripts/install-parsers.lua
+        run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash
 
       - if: "!cancelled()"
-        name: Check query files
+        name: Test parsers
+        run: $NVIM -l ./scripts/check-parsers.lua
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: "!cancelled()"
+        name: Test queries
         run: $NVIM -l ./scripts/check-queries.lua
         working-directory: ${{ env.NVIM_TS_DIR }}
         shell: bash

.github/workflows/release.yml

@@ -1,7 +1,14 @@
 name: Release
 
 on:
+  schedule:
+    - cron: '5 5 * * *'
   workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Tag name for release'
+        required: false
+        default: nightly
   push:
     tags:
       - v[0-9]+.[0-9]+.[0-9]+
@@ -17,13 +24,26 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     permissions:
+      id-token: write
+      attestations: write
       contents: write
     steps:
+      - if: github.event_name == 'workflow_dispatch'
+        env:
+          TAG_NAME: ${{ github.event.inputs.tag_name }}
+        run: echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_ENV
+
+      - if: github.event_name == 'schedule'
+        run: echo 'TAG_NAME=nightly' >> $GITHUB_ENV
+
+      - if: github.event_name == 'push'
+        run: echo "TAG_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
+
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           path: artifacts
 
@@ -33,84 +53,93 @@ jobs:
 
       - name: Prepare release artifacts
         run: |
-          mkdir -p target
-          mv artifacts/tree-sitter.wasm/* target/
+          mkdir -p target web
+          mv artifacts/tree-sitter.wasm/* web/
 
-          # Rename files
-          mv target/tree-sitter.js        target/web-tree-sitter.js
-          mv target/tree-sitter.js.map    target/web-tree-sitter.js.map
-          mv target/tree-sitter.cjs       target/web-tree-sitter.cjs
-          mv target/tree-sitter.cjs.map   target/web-tree-sitter.cjs.map
-          mv target/tree-sitter.wasm      target/web-tree-sitter.wasm
-          mv target/tree-sitter.wasm.map  target/web-tree-sitter.wasm.map
-
-          mv target/debug/tree-sitter.js        target/web-tree-sitter-debug.js
-          mv target/debug/tree-sitter.js.map    target/web-tree-sitter-debug.js.map
-          mv target/debug/tree-sitter.cjs       target/web-tree-sitter-debug.cjs
-          mv target/debug/tree-sitter.cjs.map   target/web-tree-sitter-debug.cjs.map
-          mv target/debug/tree-sitter.wasm      target/web-tree-sitter-debug.wasm
-          mv target/debug/tree-sitter.wasm.map  target/web-tree-sitter-debug.wasm.map
-          rm -rf target/debug
+          tar -czf target/web-tree-sitter.tar.gz -C web .
 
           rm -r artifacts/tree-sitter.wasm
+
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
+            chmod +x $exe
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
+            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
 
+      - name: Generate attestations
+        uses: actions/attest-build-provenance@v4
+        with:
+          subject-path: |
+            target/tree-sitter-*.gz
+            target/tree-sitter-cli-*.zip
+            target/web-tree-sitter.tar.gz
+
+      - if: env.TAG_NAME == 'nightly'
+        run: |
+          echo 'PRERELEASE=--prerelease' >> $GITHUB_ENV
+          gh release delete nightly --yes || true
+          git push https://${GITHUB_ACTOR}:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY} :nightly || true
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Create release
         run: |-
-          gh release create ${{ github.ref_name }} \
+          gh release create ${{ env.TAG_NAME }} $PRERELEASE \
             target/tree-sitter-*.gz \
-            target/web-tree-sitter.js \
-            target/web-tree-sitter.js.map \
-            target/web-tree-sitter.cjs \
-            target/web-tree-sitter.cjs.map \
-            target/web-tree-sitter.wasm \
-            target/web-tree-sitter.wasm.map \
-            target/web-tree-sitter-debug.js \
-            target/web-tree-sitter-debug.js.map \
-            target/web-tree-sitter-debug.cjs \
-            target/web-tree-sitter-debug.cjs.map \
-            target/web-tree-sitter-debug.wasm \
-            target/web-tree-sitter-debug.wasm.map
+            target/tree-sitter-cli-*.zip \
+            target/web-tree-sitter.tar.gz
         env:
           GH_TOKEN: ${{ github.token }}
 
   crates_io:
     name: Publish packages to Crates.io
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
+    environment: crates
+    permissions:
+      id-token: write
+      contents: read
     needs: release
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Set up Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
 
+      - name: Set up registry token
+        id: auth
+        uses: rust-lang/crates-io-auth-action@v1
+
       - name: Publish crates to Crates.io
         uses: katyo/publish-crates@v2
         with:
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          registry-token: ${{ steps.auth.outputs.token }}
 
   npm:
     name: Publish packages to npmjs.com
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
+    environment: npm
+    permissions:
+      id-token: write
+      contents: read
     needs: release
     strategy:
       fail-fast: false
       matrix:
-        directory: [cli/npm, lib/binding_web]
+        directory: [crates/cli/npm, lib/binding_web]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Set up Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v7.0.0
         with:
-          node-version: 20
+          node-version: 24
           registry-url: https://registry.npmjs.org
 
       - name: Set up Rust
@@ -125,9 +154,8 @@ jobs:
           npm run build:debug
           CJS=true npm run build
           CJS=true npm run build:debug
+          npm run build:dts
 
       - name: Publish to npmjs.com
         working-directory: ${{ matrix.directory }}
         run: npm publish
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/response.yml

@@ -17,13 +17,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout script
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts/close_unresponsive.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v7
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/close_unresponsive.js')
@@ -35,13 +35,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout script
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts/remove_response_label.js
           sparse-checkout-cone-mode: false
 
       - name: Run script
-        uses: actions/github-script@v7
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/remove_response_label.js')

.github/workflows/reviewers_remove.yml

@@ -11,15 +11,21 @@ jobs:
   remove-reviewers:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout script
-        uses: actions/checkout@v4
-        with:
-          sparse-checkout: .github/scripts/reviewers_remove.js
-          sparse-checkout-cone-mode: false
-
-      - name: Run script
-        uses: actions/github-script@v7
+      - name: Remove reviewers
+        uses: actions/github-script@v9
         with:
           script: |
-            const script = require('./.github/scripts/reviewers_remove.js')
-            await script({github, context})
+            const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+
+            const reviewers = requestedReviewers.data.users.map((e) => e.login);
+
+            github.rest.pulls.removeRequestedReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+              reviewers: reviewers,
+            });

.github/workflows/sanitize.yml

@@ -15,7 +15,7 @@ jobs:
       TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Install UBSAN library
         run: sudo apt-get update -y && sudo apt-get install -y libubsan1

.github/workflows/spam.yml

@@ -0,0 +1,29 @@
+name: Close as spam
+
+on:
+  issues:
+    types: [labeled]
+  pull_request_target:
+    types: [labeled]
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  spam:
+    runs-on: ubuntu-latest
+    if: github.event.label.name == 'spam'
+    steps:
+      - name: Checkout script
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts/close_spam.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const script = require('./.github/scripts/close_spam.js')
+            await script({github, context})

.github/workflows/wasm_exports.yml

@@ -1,23 +1,24 @@
-name: Check WASM Exports
+name: Check Wasm Exports
 
 on:
   pull_request:
     paths:
       - lib/include/tree_sitter/api.h
       - lib/binding_web/**
+      - xtask/src/**
   push:
     branches: [master]
     paths:
       - lib/include/tree_sitter/api.h
       - lib/binding_rust/bindings.rs
-      - lib/CMakeLists.txt
+      - CMakeLists.txt
 
 jobs:
   check-wasm-exports:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7.0.1
 
       - name: Set up stable Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -30,11 +31,11 @@ jobs:
       - name: Build C library (make)
         run: make -j CFLAGS="$CFLAGS"
         env:
-          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 
-      - name: Build WASM Library
+      - name: Build Wasm Library
         working-directory: lib/binding_web
         run: npm ci && npm run build:debug
 
-      - name: Check WASM exports
+      - name: Check Wasm exports
         run: cargo xtask check-wasm-exports

.github/workflows/wasm_stdlib.yml

@@ -0,0 +1,19 @@
+name: Check Wasm Stdlib build
+
+on:
+  workflow_call:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Check directory changes
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
+            const script = require(scriptPath);
+            return script({ github, context, core });

.gitignore

@@ -1,10 +1,12 @@
 log*.html
+.direnv
 
 .idea
 *.xcodeproj
 .vscode
 .cache
 .zig-cache
+.direnv
 
 profile*
 fuzz-results
@@ -24,6 +26,7 @@ docs/assets/js/tree-sitter.js
 *.dylib
 *.so
 *.so.[0-9]*
+*.dll
 *.o
 *.obj
 *.exp
@@ -33,3 +36,5 @@ docs/assets/js/tree-sitter.js
 .build
 build
 zig-*
+
+/result

.taplo.toml

@@ -0,0 +1,19 @@
+[formatting]
+column_width = 100
+compact_arrays = false
+reorder_inline_tables = true
+reorder_keys = true
+
+[[rule]]
+include = [ "**/Cargo.toml" ]
+keys = [ "package" ]
+
+[rule.formatting]
+reorder_keys = false
+
+[[rule]]
+include = [ "**/Cargo.toml" ]
+keys = [ "profile" ]
+
+[rule.formatting]
+reorder_keys = false

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.25.1"
+        VERSION "0.28.0"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
@@ -11,15 +11,15 @@ option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF)
 option(AMALGAMATED "Build using an amalgamated source" OFF)
 
 if(AMALGAMATED)
-  set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c")
+  set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c")
 else()
-  file(GLOB TS_SOURCE_FILES src/*.c)
-  list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c")
+  file(GLOB TS_SOURCE_FILES lib/src/*.c)
+  list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c")
 endif()
 
 add_library(tree-sitter ${TS_SOURCE_FILES})
 
-target_include_directories(tree-sitter PRIVATE src src/wasm include)
+target_include_directories(tree-sitter PRIVATE lib/src lib/src/wasm PUBLIC lib/include)
 
 if(MSVC)
   target_compile_options(tree-sitter PRIVATE
@@ -33,7 +33,8 @@ if(MSVC)
 else()
   target_compile_options(tree-sitter PRIVATE
                          -Wall -Wextra -Wshadow -Wpedantic
-                         -Werror=incompatible-pointer-types)
+                         -Werror=incompatible-pointer-types
+                         -Werror=strict-aliasing -Wstrict-aliasing=2)
 endif()
 
 if(TREE_SITTER_FEATURE_WASM)
@@ -81,15 +82,15 @@ set_target_properties(tree-sitter
                       SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
                       DEFINE_SYMBOL "")
 
-target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE)
-
-configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
 
 include(GNUInstallDirs)
 
-install(FILES include/tree_sitter/api.h
+configure_file(lib/tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
+
+install(FILES lib/include/tree_sitter/api.h
         DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter")
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc"
-        DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig")
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 install(TARGETS tree-sitter
         LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")

Cargo.toml

@@ -1,37 +1,35 @@
 [workspace]
-default-members = ["cli"]
+default-members = [ "crates/cli" ]
 members = [
-  "cli",
-  "cli/config",
-  "cli/loader",
+  "crates/cli",
+  "crates/config",
+  "crates/generate",
+  "crates/highlight",
+  "crates/loader",
+  "crates/tags",
+  "crates/xtask",
+  "crates/language",
   "lib",
-  "lib/language",
-  "tags",
-  "highlight",
-  "xtask",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.25.1"
-authors = [
-  "Max Brunsfeld <maxbrunsfeld@gmail.com>",
-  "Amaan Qureshi <amaanq12@gmail.com>",
-]
-edition = "2021"
-rust-version = "1.82"
+authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>" ]
+categories = [ "command-line-utilities", "parsing" ]
+edition = "2024"
 homepage = "https://tree-sitter.github.io/tree-sitter"
-repository = "https://github.com/tree-sitter/tree-sitter"
+keywords = [ "incremental", "parsing" ]
 license = "MIT"
-keywords = ["incremental", "parsing"]
-categories = ["command-line-utilities", "parsing"]
+repository = "https://github.com/tree-sitter/tree-sitter"
+rust-version = "1.90"
+version = "0.28.0"
 
 [workspace.lints.clippy]
-dbg_macro = "deny"
-todo = "deny"
-pedantic = { level = "warn", priority = -1 }
-nursery = { level = "warn", priority = -1 }
 cargo = { level = "warn", priority = -1 }
+dbg_macro = "deny"
+nursery = { level = "warn", priority = -1 }
+pedantic = { level = "warn", priority = -1 }
+todo = "deny"
 
 # The lints below are a specific subset of the pedantic+nursery lints
 # that we explicitly allow in the tree-sitter codebase because they either:
@@ -40,47 +38,31 @@ cargo = { level = "warn", priority = -1 }
 # 2. Are unnecessary, or
 # 3. Worsen the code
 
-branches_sharing_code = "allow"
-cast_lossless = "allow"
 cast_possible_truncation = "allow"
 cast_possible_wrap = "allow"
 cast_precision_loss = "allow"
 cast_sign_loss = "allow"
-checked_conversions = "allow"
-cognitive_complexity = "allow"
-collection_is_never_read = "allow"
-fallible_impl_from = "allow"
-fn_params_excessive_bools = "allow"
-inline_always = "allow"
 if_not_else = "allow"
 items_after_statements = "allow"
-match_wildcard_for_single_variants = "allow"
 missing_errors_doc = "allow"
 missing_panics_doc = "allow"
-module_name_repetitions = "allow"
 multiple_crate_versions = "allow"
 option_if_let_else = "allow"
-or_fun_call = "allow"
-range_plus_one = "allow"
-redundant_clone = "allow"
-redundant_closure_for_method_calls = "allow"
-ref_option = "allow"
 similar_names = "allow"
 string_lit_as_bytes = "allow"
 struct_excessive_bools = "allow"
-struct_field_names = "allow"
-transmute_undefined_repr = "allow"
 too_many_lines = "allow"
-unnecessary_wraps = "allow"
-unused_self = "allow"
-used_underscore_items = "allow"
+tuple_array_conversions = "allow"
+
+[workspace.lints.rust]
+mismatched_lifetime_syntaxes = "allow"
 
 [profile.optimize]
 inherits = "release"
-strip = true         # Automatically strip symbols from the binary.
+codegen-units = 1    # Maximum size reduction optimizations.
 lto = true           # Link-time optimization.
 opt-level = 3        # Optimization level 3.
-codegen-units = 1    # Maximum size reduction optimizations.
+strip = true         # Automatically strip symbols from the binary.
 
 [profile.size]
 inherits = "optimize"
@@ -88,70 +70,69 @@ opt-level = "s"       # Optimize for size.
 
 [profile.release-dev]
 inherits = "release"
-lto = false
+codegen-units = 256
 debug = true
 debug-assertions = true
-overflow-checks = true
 incremental = true
-codegen-units = 256
+lto = false
+overflow-checks = true
 
 [workspace.dependencies]
 ansi_colours = "1.2.3"
-anstyle = "1.0.10"
-anyhow = "1.0.95"
-bstr = "1.11.3"
-cc = "1.2.10"
-clap = { version = "4.5.27", features = [
+anstyle = "1.0.14"
+anyhow = "1.0.102"
+bstr = "1.12.1"
+cc = "1.2.63"
+clap = { features = [
   "cargo",
   "derive",
   "env",
   "help",
+  "string",
   "unstable-styles",
-] }
-clap_complete = "4.5.42"
-clap_complete_nushell = "4.5.5"
-ctor = "0.2.9"
-ctrlc = { version = "3.4.5", features = ["termination"] }
-dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
-etcetera = "0.8.0"
-filetime = "0.2.25"
-fs4 = "0.12.0"
-git2 = "0.20.0"
-glob = "0.3.2"
+], version = "4.5.58" }
+clap_complete = "4.6.3"
+clap_complete_nushell = "4.5.10"
+crc32fast = "1.5.0"
+ctor = "0.6.3"
+ctrlc = { features = [ "termination" ], version = "3.5.2" }
+dialoguer = { features = [ "fuzzy-select" ], version = "0.12.0" }
+etcetera = "0.11.0"
+glob = "0.3.3"
+hashbrown = { default-features = false, version = "0.17.1" }
 heck = "0.5.0"
 html-escape = "0.2.13"
-indexmap = "2.7.1"
-indoc = "2.0.5"
-libloading = "0.8.6"
-log = { version = "0.4.25", features = ["std"] }
-memchr = "2.7.4"
-once_cell = "1.20.2"
-path-slash = "0.2.1"
+indexmap = "2.13.0"
+indoc = "2.0.7"
+libloading = "0.9.0"
+log = { features = [ "std" ], version = "0.4.30" }
+memchr = "2.8.1"
+once_cell = "1.21.3"
 pretty_assertions = "1.4.1"
-rand = "0.8.5"
-regex = "1.11.1"
-regex-syntax = "0.8.5"
-rustc-hash = "2.1.0"
-semver = { version = "1.0.25", features = ["serde"] }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_derive = "1.0.217"
-serde_json = { version = "1.0.137", features = ["preserve_order"] }
+rand = "0.10.1"
+regex = "1.12.3"
+regex-syntax = "0.8.9"
+rustc-hash = "2.1.1"
+schemars = "1.2.1"
+semver = { features = [ "serde" ], version = "1.0.27" }
+serde = { features = [ "derive" ], version = "1.0.228" }
+serde_json = { features = [ "preserve_order" ], version = "1.0.150" }
 similar = "2.7.0"
-smallbitvec = "2.5.3"
 streaming-iterator = "0.1.9"
-tempfile = "3.15.0"
-thiserror = "2.0.11"
+tempfile = "3.25.0"
+thiserror = "2.0.18"
 tiny_http = "0.12.0"
-toml = "0.8.19"
-unindent = "0.2.3"
-url = { version = "2.5.4", features = ["serde"] }
+topological-sort = "0.2.2"
+unindent = "0.2.4"
 walkdir = "2.5.0"
-wasmparser = "0.224.0"
-webbrowser = "1.0.3"
+wasmparser = "0.244.0"
+webbrowser = "1.2.1"
 
-tree-sitter = { version = "0.25.1", path = "./lib" }
-tree-sitter-generate = { version = "0.25.1", path = "./cli/generate" }
-tree-sitter-loader = { version = "0.25.1", path = "./cli/loader" }
-tree-sitter-config = { version = "0.25.1", path = "./cli/config" }
-tree-sitter-highlight = { version = "0.25.1", path = "./highlight" }
-tree-sitter-tags = { version = "0.25.1", path = "./tags" }
+tree-sitter = { path = "./lib", version = "0.28.0" }
+tree-sitter-config = { path = "./crates/config", version = "0.28.0" }
+tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.28.0" }
+tree-sitter-highlight = { path = "./crates/highlight", version = "0.28.0" }
+tree-sitter-loader = { path = "./crates/loader", version = "0.28.0" }
+tree-sitter-tags = { path = "./crates/tags", version = "0.28.0" }
+
+tree-sitter-language = { path = "./crates/language", version = "0.1.8" }

Dockerfile

@@ -1,10 +0,0 @@
-FROM rust:1.76-buster
-
-WORKDIR /app
-
-RUN apt-get update
-RUN apt-get install -y nodejs
-
-COPY . .
-
-CMD cargo test --all-features

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2024 Max Brunsfeld
+Copyright (c) 2018 Max Brunsfeld
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,8 +1,4 @@
-ifeq ($(OS),Windows_NT)
-$(error Windows is not supported)
-endif
-
-VERSION := 0.25.1
+VERSION := 0.28.0
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -10,6 +6,7 @@ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 PREFIX ?= /usr/local
 INCLUDEDIR ?= $(PREFIX)/include
 LIBDIR ?= $(PREFIX)/lib
+BINDIR ?= $(PREFIX)/bin
 PCLIBDIR ?= $(LIBDIR)/pkgconfig
 
 # collect sources
@@ -25,9 +22,9 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -35,20 +32,25 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
 SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
 
 # OS-specific bits
-ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),)
+MACHINE := $(shell $(CC) -dumpmachine)
+
+ifneq ($(findstring darwin,$(MACHINE)),)
 	SOEXT = dylib
 	SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
 	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
 	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER)
+else ifneq ($(findstring mingw32,$(MACHINE)),)
+	SOEXT = dll
+	LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a
 else
 	SOEXT = so
 	SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
 	SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
 	LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER)
-endif
 ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
 	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
 endif
+endif
 
 all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc
 
@@ -61,6 +63,10 @@ ifneq ($(STRIP),)
 	$(STRIP) $@
 endif
 
+ifneq ($(findstring mingw32,$(MACHINE)),)
+libtree-sitter.dll.a: libtree-sitter.$(SOEXT)
+endif
+
 tree-sitter.pc: lib/tree-sitter.pc.in
 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
 		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
@@ -69,17 +75,27 @@ tree-sitter.pc: lib/tree-sitter.pc.in
 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
 
+shared: libtree-sitter.$(SOEXT)
+
+static: libtree-sitter.a
+
 clean:
-	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT)
+	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a
 
 install: all
 	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
 	install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h
 	install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
 	install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a
+ifneq ($(findstring mingw32,$(MACHINE)),)
+	install -d '$(DESTDIR)$(BINDIR)'
+	install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll
+	install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a
+else
 	install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER)
-	ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR)
-	ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT)
+	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR)
+	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT)
+endif
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \
@@ -88,8 +104,9 @@ uninstall:
 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \
 		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \
 		'$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
+	rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
 
-.PHONY: all install uninstall clean
+.PHONY: all shared static install uninstall clean
 
 
 ##### Dev targets #####
@@ -105,16 +122,20 @@ test-wasm:
 
 lint:
 	cargo update --workspace --locked --quiet
-	cargo check --workspace --all-targets
-	cargo +nightly fmt --all --check
-	cargo +nightly clippy --workspace --all-targets -- -D warnings
+	cargo fmt --all --check
+	cargo clippy --workspace --all-targets -- -D warnings
 
 lint-web:
 	npm --prefix lib/binding_web ci
 	npm --prefix lib/binding_web run lint
 
+lint-toml:
+	taplo check
+	taplo format --check --diff
+
 format:
-	cargo +nightly fmt --all
+	cargo fmt --all
+	taplo format
 
 changelog:
 	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)

Package.swift

@@ -1,33 +0,0 @@
-// swift-tools-version: 5.8
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
-import PackageDescription
-
-let package = Package(
-    name: "TreeSitter",
-    products: [
-        // Products define the executables and libraries a package produces, and make them visible to other packages.
-        .library(
-            name: "TreeSitter",
-            targets: ["TreeSitter"]),
-    ],
-    targets: [
-        .target(name: "TreeSitter",
-                path: "lib",
-                exclude: [
-                        "src/unicode/ICU_SHA",
-                        "src/unicode/README.md",
-                        "src/unicode/LICENSE",
-                        "src/wasm/stdlib-symbols.txt",
-                        "src/lib.c",
-                ],
-                sources: ["src"],
-                publicHeadersPath: "include",
-                cSettings: [
-                        .headerSearchPath("src"),
-                        .define("_POSIX_C_SOURCE", to: "200112L"),
-                        .define("_DEFAULT_SOURCE"),
-                ]),
-    ],
-    cLanguageStandard: .c11
-)

README.md

@@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca
 ## Links
 - [Documentation](https://tree-sitter.github.io)
 - [Rust binding](lib/binding_rust/README.md)
-- [WASM binding](lib/binding_web/README.md)
-- [Command-line interface](cli/README.md)
+- [Wasm binding](lib/binding_web/README.md)
+- [Command-line interface](crates/cli/README.md)
 
 [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord
 [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix

build.zig

@@ -1,116 +1,147 @@
 const std = @import("std");
 
 pub fn build(b: *std.Build) !void {
-  const target = b.standardTargetOptions(.{});
-  const optimize = b.standardOptimizeOption(.{});
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
 
-  const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
-  const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
-  const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
+    var threaded: std.Io.Threaded = .init(b.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
 
-  const lib: *std.Build.Step.Compile = if (!shared) b.addStaticLibrary(.{
-    .name = "tree-sitter",
-    .target = target,
-    .optimize = optimize,
-    .link_libc = true,
-  }) else b.addSharedLibrary(.{
-    .name = "tree-sitter",
-    .pic = true,
-    .target = target,
-    .optimize = optimize,
-    .link_libc = true,
-  });
+    const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
+    const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
+    const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
 
-  if (amalgamated) {
-    lib.addCSourceFile(.{
-      .file = b.path("lib/src/lib.c"),
-      .flags = &.{"-std=c11"},
+    var tree_sitter = b.createModule(.{
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .pic = if (shared) true else null,
     });
-  } else {
-    lib.addCSourceFiles(.{
-      .root = b.path("lib/src"),
-      .files = try findSourceFiles(b),
-      .flags = &.{"-std=c11"},
+    const lib: *std.Build.Step.Compile = b.addLibrary(.{
+        .name = "tree-sitter",
+        .linkage = if (shared) .dynamic else .static,
+        .root_module = tree_sitter,
     });
-  }
 
-  lib.addIncludePath(b.path("lib/include"));
-  lib.addIncludePath(b.path("lib/src"));
-  lib.addIncludePath(b.path("lib/src/wasm"));
-
-  lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
-  lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
-
-  if (wasm) {
-    if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
-      lib.root_module.addCMacro("TREE_SITTER_FEATURE_WASM", "");
-      lib.addSystemIncludePath(wasmtime.path("include"));
-      lib.addLibraryPath(wasmtime.path("lib"));
-      lib.linkSystemLibrary("wasmtime");
+    if (amalgamated) {
+        tree_sitter.addCSourceFile(.{
+            .file = b.path("lib/src/lib.c"),
+            .flags = &.{"-std=c11"},
+        });
+    } else {
+        const files = try findSourceFiles(b, io);
+        defer b.allocator.free(files);
+        tree_sitter.addCSourceFiles(.{
+            .root = b.path("lib/src"),
+            .files = files,
+            .flags = &.{"-std=c11"},
+        });
     }
-  }
 
-  lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
+    tree_sitter.addIncludePath(b.path("lib/include"));
+    tree_sitter.addIncludePath(b.path("lib/src"));
+    tree_sitter.addIncludePath(b.path("lib/src/wasm"));
 
-  b.installArtifact(lib);
-}
+    tree_sitter.addCMacro("_POSIX_C_SOURCE", "200112L");
+    tree_sitter.addCMacro("_DEFAULT_SOURCE", "");
+    tree_sitter.addCMacro("_BSD_SOURCE", "");
+    tree_sitter.addCMacro("_DARWIN_C_SOURCE", "");
 
-fn wasmtimeDep(target: std.Target) []const u8 {
-  const arch = target.cpu.arch;
-  const os = target.os.tag;
-  const abi = target.abi;
-  return switch (os) {
-    .linux => switch (arch) {
-      .x86_64 => switch (abi) {
-        .gnu => "wasmtime_c_api_x86_64_linux",
-        .musl => "wasmtime_c_api_x86_64_musl",
-        .android => "wasmtime_c_api_x86_64_android",
-        else => null
-      },
-      .aarch64 => switch (abi) {
-        .gnu => "wasmtime_c_api_aarch64_linux",
-        .android => "wasmtime_c_api_aarch64_android",
-        else => null
-      },
-      .s390x => "wasmtime_c_api_s390x_linux",
-      .riscv64 => "wasmtime_c_api_riscv64gc_linux",
-      else => null
-    },
-    .windows => switch (arch) {
-      .x86_64 => switch (abi) {
-        .gnu => "wasmtime_c_api_x86_64_mingw",
-        .msvc => "wasmtime_c_api_x86_64_windows",
-        else => null
-      },
-      else => null
-    },
-    .macos => switch (arch) {
-      .x86_64 => "wasmtime_c_api_x86_64_macos",
-      .aarch64 => "wasmtime_c_api_aarch64_macos",
-      else => null
-    },
-    else => null
-  } orelse std.debug.panic(
-    "Unsupported target for wasmtime: {s}-{s}-{s}",
-    .{ @tagName(arch), @tagName(os), @tagName(abi) }
-  );
-}
-
-fn findSourceFiles(b: *std.Build) ![]const []const u8 {
-  var sources = std.ArrayList([]const u8).init(b.allocator);
-
-  var dir = try b.build_root.handle.openDir("lib/src", .{ .iterate = true });
-  var iter = dir.iterate();
-  defer dir.close();
-
-  while (try iter.next()) |entry| {
-    if (entry.kind != .file) continue;
-    const file = entry.name;
-    const ext = std.fs.path.extension(file);
-    if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
-      try sources.append(b.dupe(file));
+    if (wasm) {
+        if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
+            tree_sitter.addCMacro("TREE_SITTER_FEATURE_WASM", "");
+            tree_sitter.addSystemIncludePath(wasmtime.path("include"));
+            tree_sitter.addLibraryPath(wasmtime.path("lib"));
+            if (shared) tree_sitter.linkSystemLibrary("wasmtime", .{});
+        }
     }
-  }
 
-  return sources.items;
+    lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
+
+    b.installArtifact(lib);
+}
+
+/// Get the name of the wasmtime dependency for this target.
+pub fn wasmtimeDep(target: std.Target) []const u8 {
+    const arch = target.cpu.arch;
+    const os = target.os.tag;
+    const abi = target.abi;
+    return @as(?[]const u8, switch (os) {
+        .linux => switch (arch) {
+            .x86_64 => switch (abi) {
+                .gnu => "wasmtime_c_api_x86_64_linux",
+                .musl => "wasmtime_c_api_x86_64_musl",
+                .android => "wasmtime_c_api_x86_64_android",
+                else => null,
+            },
+            .aarch64 => switch (abi) {
+                .gnu => "wasmtime_c_api_aarch64_linux",
+                .musl => "wasmtime_c_api_aarch64_musl",
+                .android => "wasmtime_c_api_aarch64_android",
+                else => null,
+            },
+            .x86 => switch (abi) {
+                .gnu => "wasmtime_c_api_i686_linux",
+                else => null,
+            },
+            .arm => switch (abi) {
+                .gnueabi => "wasmtime_c_api_armv7_linux",
+                else => null,
+            },
+            .s390x => switch (abi) {
+                .gnu => "wasmtime_c_api_s390x_linux",
+                else => null,
+            },
+            .riscv64 => switch (abi) {
+                .gnu => "wasmtime_c_api_riscv64gc_linux",
+                else => null,
+            },
+            else => null,
+        },
+        .windows => switch (arch) {
+            .x86_64 => switch (abi) {
+                .gnu => "wasmtime_c_api_x86_64_mingw",
+                .msvc => "wasmtime_c_api_x86_64_windows",
+                else => null,
+            },
+            .aarch64 => switch (abi) {
+                .msvc => "wasmtime_c_api_aarch64_windows",
+                else => null,
+            },
+            .x86 => switch (abi) {
+                .msvc => "wasmtime_c_api_i686_windows",
+                else => null,
+            },
+            else => null,
+        },
+        .macos => switch (arch) {
+            .x86_64 => "wasmtime_c_api_x86_64_macos",
+            .aarch64 => "wasmtime_c_api_aarch64_macos",
+            else => null,
+        },
+        else => null,
+    }) orelse std.debug.panic(
+        "Unsupported target for wasmtime: {s}-{s}-{s}",
+        .{ @tagName(arch), @tagName(os), @tagName(abi) },
+    );
+}
+
+fn findSourceFiles(b: *std.Build, io: std.Io) ![]const []const u8 {
+    var sources: std.ArrayListUnmanaged([]const u8) = .empty;
+
+    var dir = try b.build_root.handle.openDir(io, "lib/src", .{ .iterate = true });
+    var iter = dir.iterate();
+    defer dir.close(io);
+
+    while (try iter.next(io)) |entry| {
+        if (entry.kind != .file) continue;
+        const file = entry.name;
+        const ext = std.fs.path.extension(file);
+        if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
+            try sources.append(b.allocator, b.dupe(file));
+        }
+    }
+
+    return sources.toOwnedSlice(b.allocator);
 }

build.zig.zon

@@ -1,69 +1,96 @@
 .{
-  .name = "tree-sitter",
-  .version = "0.25.1",
-  .paths = .{
-    "build.zig",
-    "build.zig.zon",
-    "lib/src",
-    "lib/include",
-    "README.md",
-    "LICENSE",
-  },
-  .dependencies = .{
-    .wasmtime_c_api_aarch64_android = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-android-c-api.tar.xz",
-      .hash = "12204c77979ad8291c6e395d695a824fb053ffdfeb2cc21de95fffb09f77d77188d1",
-      .lazy = true,
+    .name = .tree_sitter,
+    .fingerprint = 0x841224b447ac0d4f,
+    .version = "0.28.0",
+    .minimum_zig_version = "0.16.0",
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "lib/src",
+        "lib/include",
+        "README.md",
+        "LICENSE",
     },
-    .wasmtime_c_api_aarch64_linux = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-linux-c-api.tar.xz",
-      .hash = "12203a8e3d823490186fb1e230d54f575148713088e914926305ee5678790b731bba",
-      .lazy = true,
+    .dependencies = .{
+        .wasmtime_c_api_aarch64_android = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAIp_mQVzQOITXcYcWxYLJkvB1W1SvLlrdiU2G7fj",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAMztsgU5Aj4oI3MRHXJVe5rW72op-kT_78I3kZVM",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_macos = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AANZxOwT27sdrKxDDGGKsiwtcZlHy204xAWNIgDBH",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_musl = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAJL1zQW9yxC98uc60lSuVUHhH77QHTto0zwIQnBj",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-windows-c-api.zip",
+            .hash = "N-V-__8AAHRCtQU93hJcRFOgVcof3IQpRV9stT2Pp54wpJc2",
+            .lazy = true,
+        },
+        .wasmtime_c_api_armv7_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-armv7-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAJaW6gT8QdULOU0jxX4a_DOCA5YD6cxWBC8IqhQF",
+            .lazy = true,
+        },
+        .wasmtime_c_api_i686_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-linux-c-api.tar.xz",
+            .hash = "N-V-__8AANguMgVX4XMhdOVkdj4yfFKXrG8RTgZDs3nQB8J8",
+            .lazy = true,
+        },
+        .wasmtime_c_api_i686_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-windows-c-api.zip",
+            .hash = "N-V-__8AANY9ggXg4rK2_1o3EIlrCq124l5RfykPv-DPforq",
+            .lazy = true,
+        },
+        .wasmtime_c_api_riscv64gc_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-riscv64gc-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAPDtCAdQ0dD9Rs-qWl-kPr2c7L3PVsUVjwy12Iz1",
+            .lazy = true,
+        },
+        .wasmtime_c_api_s390x_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-s390x-linux-c-api.tar.xz",
+            .hash = "N-V-__8AANA3BwY1ZOoGCWCR_tTY9G1vfIX128RGVxufN3ov",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_android = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAF4AIgY0ltjevj1ybGfvMU1ErPRnNve5X1TmvCru",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAIR0cAbjf3DkrTbu81Oq_zociz-0lCpb5DR0lIC9",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_macos = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAFJ4lgRgCBnYdz8-Yfc4hLve45Hv-0RICAEuCp4s",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_mingw = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-mingw-c-api.zip",
+            .hash = "N-V-__8AAMxZxQZUpp1cU8J5zgLiMNq4e4dy0hcchlQFy03J",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_musl = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAN5pWgZrZBt8VYWkN82WjyFe3DkGcE7uLn3jpt38",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-windows-c-api.zip",
+            .hash = "N-V-__8AAEIJkgaVHFETgakbognNUpFELuV17vpjw6NjsWhQ",
+            .lazy = true,
+        },
     },
-    .wasmtime_c_api_aarch64_macos = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-macos-c-api.tar.xz",
-      .hash = "122043e8b19079b855b12674b9e3d4a28dc5c399c43b62fbeb8bdf0fdb4ef2d1d38c",
-      .lazy = true,
-    },
-    .wasmtime_c_api_riscv64gc_linux = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-riscv64gc-linux-c-api.tar.xz",
-      .hash = "12209d07031cf33271bf4b0c63df407b535cd5d65c6402bd6f80d99de439d6feb89b",
-      .lazy = true,
-    },
-    .wasmtime_c_api_s390x_linux = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-s390x-linux-c-api.tar.xz",
-      .hash = "122033f7d9b04f429063d9b2d9ac75a7a00fce02c425e578208f54ddc40edaa1e355",
-      .lazy = true,
-    },
-    .wasmtime_c_api_x86_64_android = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-android-c-api.tar.xz",
-      .hash = "122093cb33df8e09e70b2d1dc09897a0388915b942918389b10bf23f9684bdb6f047",
-      .lazy = true,
-    },
-    .wasmtime_c_api_x86_64_linux = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-linux-c-api.tar.xz",
-      .hash = "12209210346e94bf6ef8e249fa5d3f1a84f95050ed19665ac8422a15b5f2246d83af",
-      .lazy = true,
-    },
-    .wasmtime_c_api_x86_64_macos = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-macos-c-api.tar.xz",
-      .hash = "12208f875dd3a89092485762f3b184707b3cccae85a84e2ffd38c138cc3a3fd90447",
-      .lazy = true,
-    },
-    .wasmtime_c_api_x86_64_mingw = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-mingw-c-api.zip",
-      .hash = "1220bea757df3a777b6ec6322fc498e4ece20d466eedc5e2a3610b338849553cd94d",
-      .lazy = true,
-    },
-    .wasmtime_c_api_x86_64_musl = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-musl-c-api.tar.xz",
-      .hash = "1220c9596e6a63edcc3234c0611d0cbac724bf30ac9a0fbaf402c7da649b278b1322",
-      .lazy = true,
-    },
-    .wasmtime_c_api_x86_64_windows = .{
-      .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-windows-c-api.zip",
-      .hash = "1220440ccb01d72989cf1a47728897a35fc8dd31673cce598f2d62c58e2c3228b0ed",
-      .lazy = true,
-    },
-  }
 }

cli/Cargo.toml

@@ -1,81 +0,0 @@
-[package]
-name = "tree-sitter-cli"
-version.workspace = true
-description = "CLI tool for developing, testing, and using Tree-sitter parsers"
-authors.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-readme = "README.md"
-homepage.workspace = true
-repository.workspace = true
-license.workspace = true
-keywords.workspace = true
-categories.workspace = true
-include = ["build.rs", "README.md", "benches/*", "src/**"]
-
-[lints]
-workspace = true
-
-[[bin]]
-name = "tree-sitter"
-path = "src/main.rs"
-doc = false
-
-[[bench]]
-name = "benchmark"
-harness = false
-
-[features]
-wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
-
-[dependencies]
-ansi_colours.workspace = true
-anstyle.workspace = true
-anyhow.workspace = true
-bstr.workspace = true
-clap.workspace = true
-clap_complete.workspace = true
-clap_complete_nushell.workspace = true
-ctor.workspace = true
-ctrlc.workspace = true
-dialoguer.workspace = true
-filetime.workspace = true
-glob.workspace = true
-heck.workspace = true
-html-escape.workspace = true
-indexmap.workspace = true
-indoc.workspace = true
-log.workspace = true
-memchr.workspace = true
-rand.workspace = true
-regex.workspace = true
-regex-syntax.workspace = true
-rustc-hash.workspace = true
-semver.workspace = true
-serde.workspace = true
-serde_derive.workspace = true
-serde_json.workspace = true
-similar.workspace = true
-smallbitvec.workspace = true
-streaming-iterator.workspace = true
-tiny_http.workspace = true
-url.workspace = true
-walkdir.workspace = true
-wasmparser.workspace = true
-webbrowser.workspace = true
-
-tree-sitter.workspace = true
-tree-sitter-generate.workspace = true
-tree-sitter-config.workspace = true
-tree-sitter-highlight.workspace = true
-tree-sitter-loader.workspace = true
-tree-sitter-tags.workspace = true
-
-[dev-dependencies]
-encoding_rs = "0.8.35"
-widestring = "1.1.0"
-tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
-
-tempfile.workspace = true
-pretty_assertions.workspace = true
-unindent.workspace = true

cli/generate/build.rs

@@ -1,32 +0,0 @@
-use std::{env, path::PathBuf, process::Command};
-
-fn main() {
-    if let Some(git_sha) = read_git_sha() {
-        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
-    }
-}
-
-// This is copied from the build.rs in parent directory. This should be updated if the
-// parent build.rs gets fixes.
-fn read_git_sha() -> Option<String> {
-    let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
-
-    if !crate_path
-        .parent()?
-        .parent()
-        .is_some_and(|p| p.join(".git").exists())
-    {
-        return None;
-    }
-
-    Command::new("git")
-        .args(["rev-parse", "HEAD"])
-        .current_dir(crate_path)
-        .output()
-        .map_or(None, |output| {
-            if !output.status.success() {
-                return None;
-            }
-            Some(String::from_utf8_lossy(&output.stdout).to_string())
-        })
-}

cli/generate/src/build_tables/coincident_tokens.rs

@@ -1,79 +0,0 @@
-use std::fmt;
-
-use crate::{
-    grammars::LexicalGrammar,
-    rules::Symbol,
-    tables::{ParseStateId, ParseTable},
-};
-
-pub struct CoincidentTokenIndex<'a> {
-    entries: Vec<Vec<ParseStateId>>,
-    grammar: &'a LexicalGrammar,
-    n: usize,
-}
-
-impl<'a> CoincidentTokenIndex<'a> {
-    pub fn new(table: &ParseTable, lexical_grammar: &'a LexicalGrammar) -> Self {
-        let n = lexical_grammar.variables.len();
-        let mut result = Self {
-            n,
-            grammar: lexical_grammar,
-            entries: vec![Vec::new(); n * n],
-        };
-        for (i, state) in table.states.iter().enumerate() {
-            for symbol in state.terminal_entries.keys() {
-                if symbol.is_terminal() {
-                    for other_symbol in state.terminal_entries.keys() {
-                        if other_symbol.is_terminal() {
-                            let index = result.index(symbol.index, other_symbol.index);
-                            if result.entries[index].last().copied() != Some(i) {
-                                result.entries[index].push(i);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        result
-    }
-
-    pub fn states_with(&self, a: Symbol, b: Symbol) -> &[ParseStateId] {
-        &self.entries[self.index(a.index, b.index)]
-    }
-
-    pub fn contains(&self, a: Symbol, b: Symbol) -> bool {
-        !self.entries[self.index(a.index, b.index)].is_empty()
-    }
-
-    #[must_use]
-    const fn index(&self, a: usize, b: usize) -> usize {
-        if a < b {
-            a * self.n + b
-        } else {
-            b * self.n + a
-        }
-    }
-}
-
-impl fmt::Debug for CoincidentTokenIndex<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "CoincidentTokenIndex {{")?;
-
-        writeln!(f, "  entries: {{")?;
-        for i in 0..self.n {
-            writeln!(f, "    {}: {{", self.grammar.variables[i].name)?;
-            for j in 0..self.n {
-                writeln!(
-                    f,
-                    "      {}: {:?},",
-                    self.grammar.variables[j].name,
-                    self.entries[self.index(i, j)].len()
-                )?;
-            }
-            writeln!(f, "    }},")?;
-        }
-        write!(f, "  }},")?;
-        write!(f, "}}")?;
-        Ok(())
-    }
-}

cli/generate/src/build_tables/item.rs

@@ -1,462 +0,0 @@
-use std::{
-    cmp::Ordering,
-    fmt,
-    hash::{Hash, Hasher},
-    sync::LazyLock,
-};
-
-use crate::{
-    grammars::{
-        LexicalGrammar, Production, ProductionStep, ReservedWordSetId, SyntaxGrammar,
-        NO_RESERVED_WORDS,
-    },
-    rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
-};
-
-static START_PRODUCTION: LazyLock<Production> = LazyLock::new(|| Production {
-    dynamic_precedence: 0,
-    steps: vec![ProductionStep {
-        symbol: Symbol {
-            index: 0,
-            kind: SymbolType::NonTerminal,
-        },
-        precedence: Precedence::None,
-        associativity: None,
-        alias: None,
-        field_name: None,
-        reserved_word_set_id: NO_RESERVED_WORDS,
-    }],
-});
-
-/// A [`ParseItem`] represents an in-progress match of a single production in a grammar.
-#[derive(Clone, Copy, Debug)]
-pub struct ParseItem<'a> {
-    /// The index of the parent rule within the grammar.
-    pub variable_index: u32,
-    /// The number of symbols that have already been matched.
-    pub step_index: u32,
-    /// The production being matched.
-    pub production: &'a Production,
-    /// A boolean indicating whether any of the already-matched children were
-    /// hidden nodes and had fields. Ordinarily, a parse item's behavior is not
-    /// affected by the symbols of its preceding children; it only needs to
-    /// keep track of their fields and aliases.
-    ///
-    /// Take for example these two items:
-    ///   X -> a b • c
-    ///   X -> a g • c
-    ///
-    /// They can be considered equivalent, for the purposes of parse table
-    /// generation, because they entail the same actions. But if this flag is
-    /// true, then the item's set of inherited fields may depend on the specific
-    /// symbols of its preceding children.
-    pub has_preceding_inherited_fields: bool,
-}
-
-/// A [`ParseItemSet`] represents a set of in-progress matches of productions in a
-/// grammar, and for each in-progress match, a set of "lookaheads" - tokens that
-/// are allowed to *follow* the in-progress rule. This object corresponds directly
-/// to a state in the final parse table.
-#[derive(Clone, Debug, PartialEq, Eq, Default)]
-pub struct ParseItemSet<'a> {
-    pub entries: Vec<ParseItemSetEntry<'a>>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ParseItemSetEntry<'a> {
-    pub item: ParseItem<'a>,
-    pub lookaheads: TokenSet,
-    pub following_reserved_word_set: ReservedWordSetId,
-}
-
-/// A [`ParseItemSetCore`] is like a [`ParseItemSet`], but without the lookahead
-/// information. Parse states with the same core are candidates for merging.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ParseItemSetCore<'a> {
-    pub entries: Vec<ParseItem<'a>>,
-}
-
-pub struct ParseItemDisplay<'a>(
-    pub &'a ParseItem<'a>,
-    pub &'a SyntaxGrammar,
-    pub &'a LexicalGrammar,
-);
-
-pub struct TokenSetDisplay<'a>(
-    pub &'a TokenSet,
-    pub &'a SyntaxGrammar,
-    pub &'a LexicalGrammar,
-);
-
-pub struct ParseItemSetDisplay<'a>(
-    pub &'a ParseItemSet<'a>,
-    pub &'a SyntaxGrammar,
-    pub &'a LexicalGrammar,
-);
-
-impl<'a> ParseItem<'a> {
-    pub fn start() -> Self {
-        ParseItem {
-            variable_index: u32::MAX,
-            production: &START_PRODUCTION,
-            step_index: 0,
-            has_preceding_inherited_fields: false,
-        }
-    }
-
-    pub fn step(&self) -> Option<&'a ProductionStep> {
-        self.production.steps.get(self.step_index as usize)
-    }
-
-    pub fn symbol(&self) -> Option<Symbol> {
-        self.step().map(|step| step.symbol)
-    }
-
-    pub fn associativity(&self) -> Option<Associativity> {
-        self.prev_step().and_then(|step| step.associativity)
-    }
-
-    pub fn precedence(&self) -> &Precedence {
-        self.prev_step()
-            .map_or(&Precedence::None, |step| &step.precedence)
-    }
-
-    pub fn prev_step(&self) -> Option<&'a ProductionStep> {
-        if self.step_index > 0 {
-            Some(&self.production.steps[self.step_index as usize - 1])
-        } else {
-            None
-        }
-    }
-
-    #[must_use]
-    pub fn is_done(&self) -> bool {
-        self.step_index as usize == self.production.steps.len()
-    }
-
-    #[must_use]
-    pub const fn is_augmented(&self) -> bool {
-        self.variable_index == u32::MAX
-    }
-
-    /// Create an item like this one, but advanced by one step.
-    #[must_use]
-    pub const fn successor(&self) -> Self {
-        ParseItem {
-            variable_index: self.variable_index,
-            production: self.production,
-            step_index: self.step_index + 1,
-            has_preceding_inherited_fields: self.has_preceding_inherited_fields,
-        }
-    }
-
-    /// Create an item identical to this one, but with a different production.
-    /// This is used when dynamically "inlining" certain symbols in a production.
-    pub const fn substitute_production(&self, production: &'a Production) -> Self {
-        let mut result = *self;
-        result.production = production;
-        result
-    }
-}
-
-impl<'a> ParseItemSet<'a> {
-    pub fn insert(&mut self, item: ParseItem<'a>) -> &mut ParseItemSetEntry<'a> {
-        match self.entries.binary_search_by(|e| e.item.cmp(&item)) {
-            Err(i) => {
-                self.entries.insert(
-                    i,
-                    ParseItemSetEntry {
-                        item,
-                        lookaheads: TokenSet::new(),
-                        following_reserved_word_set: ReservedWordSetId::default(),
-                    },
-                );
-                &mut self.entries[i]
-            }
-            Ok(i) => &mut self.entries[i],
-        }
-    }
-
-    pub fn core(&self) -> ParseItemSetCore<'a> {
-        ParseItemSetCore {
-            entries: self.entries.iter().map(|e| e.item).collect(),
-        }
-    }
-}
-
-impl fmt::Display for ParseItemDisplay<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        if self.0.is_augmented() {
-            write!(f, "START →")?;
-        } else {
-            write!(
-                f,
-                "{} →",
-                self.1.variables[self.0.variable_index as usize].name
-            )?;
-        }
-
-        for (i, step) in self.0.production.steps.iter().enumerate() {
-            if i == self.0.step_index as usize {
-                write!(f, " •")?;
-                if !step.precedence.is_none()
-                    || step.associativity.is_some()
-                    || step.reserved_word_set_id != ReservedWordSetId::default()
-                {
-                    write!(f, " (")?;
-                    if step.precedence.is_none() {
-                        write!(f, " {}", step.precedence)?;
-                    }
-                    if let Some(associativity) = step.associativity {
-                        write!(f, " {associativity:?}")?;
-                    }
-                    if step.reserved_word_set_id != ReservedWordSetId::default() {
-                        write!(f, "reserved: {}", step.reserved_word_set_id)?;
-                    }
-                    write!(f, " )")?;
-                }
-            }
-
-            write!(f, " ")?;
-            if step.symbol.is_terminal() {
-                if let Some(variable) = self.2.variables.get(step.symbol.index) {
-                    write!(f, "{}", variable.name)?;
-                } else {
-                    write!(f, "terminal-{}", step.symbol.index)?;
-                }
-            } else if step.symbol.is_external() {
-                write!(f, "{}", self.1.external_tokens[step.symbol.index].name)?;
-            } else {
-                write!(f, "{}", self.1.variables[step.symbol.index].name)?;
-            }
-
-            if let Some(alias) = &step.alias {
-                write!(f, "@{}", alias.value)?;
-            }
-        }
-
-        if self.0.is_done() {
-            write!(f, " •")?;
-            if let Some(step) = self.0.production.steps.last() {
-                if let Some(associativity) = step.associativity {
-                    if step.precedence.is_none() {
-                        write!(f, " ({associativity:?})")?;
-                    } else {
-                        write!(f, " ({} {associativity:?})", step.precedence)?;
-                    }
-                } else if !step.precedence.is_none() {
-                    write!(f, " ({})", step.precedence)?;
-                }
-            }
-        }
-
-        Ok(())
-    }
-}
-
-const fn escape_invisible(c: char) -> Option<&'static str> {
-    Some(match c {
-        '\n' => "\\n",
-        '\r' => "\\r",
-        '\t' => "\\t",
-        '\0' => "\\0",
-        '\\' => "\\\\",
-        '\x0b' => "\\v",
-        '\x0c' => "\\f",
-        _ => return None,
-    })
-}
-
-fn display_variable_name(source: &str) -> String {
-    source
-        .chars()
-        .fold(String::with_capacity(source.len()), |mut acc, c| {
-            if let Some(esc) = escape_invisible(c) {
-                acc.push_str(esc);
-            } else {
-                acc.push(c);
-            }
-            acc
-        })
-}
-
-impl fmt::Display for TokenSetDisplay<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        write!(f, "[")?;
-        for (i, symbol) in self.0.iter().enumerate() {
-            if i > 0 {
-                write!(f, ", ")?;
-            }
-
-            if symbol.is_terminal() {
-                if let Some(variable) = self.2.variables.get(symbol.index) {
-                    write!(f, "{}", display_variable_name(&variable.name))?;
-                } else {
-                    write!(f, "terminal-{}", symbol.index)?;
-                }
-            } else if symbol.is_external() {
-                write!(f, "{}", self.1.external_tokens[symbol.index].name)?;
-            } else {
-                write!(f, "{}", self.1.variables[symbol.index].name)?;
-            }
-        }
-        write!(f, "]")?;
-        Ok(())
-    }
-}
-
-impl fmt::Display for ParseItemSetDisplay<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        for entry in &self.0.entries {
-            write!(
-                f,
-                "{}\t{}",
-                ParseItemDisplay(&entry.item, self.1, self.2),
-                TokenSetDisplay(&entry.lookaheads, self.1, self.2),
-            )?;
-            if entry.following_reserved_word_set != ReservedWordSetId::default() {
-                write!(
-                    f,
-                    "\treserved word set: {}",
-                    entry.following_reserved_word_set
-                )?;
-            }
-            writeln!(f)?;
-        }
-        Ok(())
-    }
-}
-
-impl Hash for ParseItem<'_> {
-    fn hash<H: Hasher>(&self, hasher: &mut H) {
-        hasher.write_u32(self.variable_index);
-        hasher.write_u32(self.step_index);
-        hasher.write_i32(self.production.dynamic_precedence);
-        hasher.write_usize(self.production.steps.len());
-        hasher.write_i32(i32::from(self.has_preceding_inherited_fields));
-        self.precedence().hash(hasher);
-        self.associativity().hash(hasher);
-
-        // The already-matched children don't play any role in the parse state for
-        // this item, unless any of the following are true:
-        //   * the children have fields
-        //   * the children have aliases
-        //   * the children are hidden and represent rules that have fields.
-        // See the docs for `has_preceding_inherited_fields`.
-        for step in &self.production.steps[0..self.step_index as usize] {
-            step.alias.hash(hasher);
-            step.field_name.hash(hasher);
-            if self.has_preceding_inherited_fields {
-                step.symbol.hash(hasher);
-            }
-        }
-        for step in &self.production.steps[self.step_index as usize..] {
-            step.hash(hasher);
-        }
-    }
-}
-
-impl PartialEq for ParseItem<'_> {
-    fn eq(&self, other: &Self) -> bool {
-        if self.variable_index != other.variable_index
-            || self.step_index != other.step_index
-            || self.production.dynamic_precedence != other.production.dynamic_precedence
-            || self.production.steps.len() != other.production.steps.len()
-            || self.precedence() != other.precedence()
-            || self.associativity() != other.associativity()
-            || self.has_preceding_inherited_fields != other.has_preceding_inherited_fields
-        {
-            return false;
-        }
-
-        for (i, step) in self.production.steps.iter().enumerate() {
-            // See the previous comment (in the `Hash::hash` impl) regarding comparisons
-            // of parse items' already-completed steps.
-            if i < self.step_index as usize {
-                if step.alias != other.production.steps[i].alias {
-                    return false;
-                }
-                if step.field_name != other.production.steps[i].field_name {
-                    return false;
-                }
-                if self.has_preceding_inherited_fields
-                    && step.symbol != other.production.steps[i].symbol
-                {
-                    return false;
-                }
-            } else if *step != other.production.steps[i] {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
-impl Ord for ParseItem<'_> {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.step_index
-            .cmp(&other.step_index)
-            .then_with(|| self.variable_index.cmp(&other.variable_index))
-            .then_with(|| {
-                self.production
-                    .dynamic_precedence
-                    .cmp(&other.production.dynamic_precedence)
-            })
-            .then_with(|| {
-                self.production
-                    .steps
-                    .len()
-                    .cmp(&other.production.steps.len())
-            })
-            .then_with(|| self.precedence().cmp(other.precedence()))
-            .then_with(|| self.associativity().cmp(&other.associativity()))
-            .then_with(|| {
-                for (i, step) in self.production.steps.iter().enumerate() {
-                    // See the previous comment (in the `Hash::hash` impl) regarding comparisons
-                    // of parse items' already-completed steps.
-                    let o = if i < self.step_index as usize {
-                        step.alias
-                            .cmp(&other.production.steps[i].alias)
-                            .then_with(|| {
-                                step.field_name.cmp(&other.production.steps[i].field_name)
-                            })
-                    } else {
-                        step.cmp(&other.production.steps[i])
-                    };
-                    if o != Ordering::Equal {
-                        return o;
-                    }
-                }
-                Ordering::Equal
-            })
-    }
-}
-
-impl PartialOrd for ParseItem<'_> {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Eq for ParseItem<'_> {}
-
-impl Hash for ParseItemSet<'_> {
-    fn hash<H: Hasher>(&self, hasher: &mut H) {
-        hasher.write_usize(self.entries.len());
-        for entry in &self.entries {
-            entry.item.hash(hasher);
-            entry.lookaheads.hash(hasher);
-            entry.following_reserved_word_set.hash(hasher);
-        }
-    }
-}
-
-impl Hash for ParseItemSetCore<'_> {
-    fn hash<H: Hasher>(&self, hasher: &mut H) {
-        hasher.write_usize(self.entries.len());
-        for item in &self.entries {
-            item.hash(hasher);
-        }
-    }
-}

cli/generate/src/build_tables/item_set_builder.rs

@@ -1,397 +0,0 @@
-use std::{
-    collections::{HashMap, HashSet},
-    fmt,
-};
-
-use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, ParseItemSetEntry, TokenSetDisplay};
-use crate::{
-    grammars::{InlinedProductionMap, LexicalGrammar, ReservedWordSetId, SyntaxGrammar},
-    rules::{Symbol, SymbolType, TokenSet},
-};
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-struct TransitiveClosureAddition<'a> {
-    item: ParseItem<'a>,
-    info: FollowSetInfo,
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-struct FollowSetInfo {
-    lookaheads: TokenSet,
-    reserved_lookaheads: ReservedWordSetId,
-    propagates_lookaheads: bool,
-}
-
-pub struct ParseItemSetBuilder<'a> {
-    syntax_grammar: &'a SyntaxGrammar,
-    lexical_grammar: &'a LexicalGrammar,
-    first_sets: HashMap<Symbol, TokenSet>,
-    reserved_first_sets: HashMap<Symbol, ReservedWordSetId>,
-    last_sets: HashMap<Symbol, TokenSet>,
-    inlines: &'a InlinedProductionMap,
-    transitive_closure_additions: Vec<Vec<TransitiveClosureAddition<'a>>>,
-}
-
-fn find_or_push<T: Eq>(vector: &mut Vec<T>, value: T) {
-    if !vector.contains(&value) {
-        vector.push(value);
-    }
-}
-
-impl<'a> ParseItemSetBuilder<'a> {
-    pub fn new(
-        syntax_grammar: &'a SyntaxGrammar,
-        lexical_grammar: &'a LexicalGrammar,
-        inlines: &'a InlinedProductionMap,
-    ) -> Self {
-        let mut result = Self {
-            syntax_grammar,
-            lexical_grammar,
-            first_sets: HashMap::new(),
-            reserved_first_sets: HashMap::new(),
-            last_sets: HashMap::new(),
-            inlines,
-            transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()],
-        };
-
-        // For each grammar symbol, populate the FIRST and LAST sets: the set of
-        // terminals that appear at the beginning and end that symbol's productions,
-        // respectively.
-        // For a terminal symbol, the FIRST and LAST sets just consist of the
-        // terminal itself.
-        for i in 0..lexical_grammar.variables.len() {
-            let symbol = Symbol::terminal(i);
-            let mut set = TokenSet::new();
-            set.insert(symbol);
-            result.first_sets.insert(symbol, set.clone());
-            result.last_sets.insert(symbol, set);
-            result
-                .reserved_first_sets
-                .insert(symbol, ReservedWordSetId::default());
-        }
-
-        for i in 0..syntax_grammar.external_tokens.len() {
-            let symbol = Symbol::external(i);
-            let mut set = TokenSet::new();
-            set.insert(symbol);
-            result.first_sets.insert(symbol, set.clone());
-            result.last_sets.insert(symbol, set);
-            result
-                .reserved_first_sets
-                .insert(symbol, ReservedWordSetId::default());
-        }
-
-        // The FIRST set of a non-terminal `i` is the union of the the FIRST sets
-        // of all the symbols that appear at the beginnings of i's productions. Some
-        // of these symbols may themselves be non-terminals, so this is a recursive
-        // definition.
-        //
-        // Rather than computing these sets using recursion, we use an explicit stack
-        // called `symbols_to_process`.
-        let mut symbols_to_process = Vec::new();
-        let mut processed_non_terminals = HashSet::new();
-        for i in 0..syntax_grammar.variables.len() {
-            let symbol = Symbol::non_terminal(i);
-            let first_set = result.first_sets.entry(symbol).or_default();
-            let reserved_first_set = result.reserved_first_sets.entry(symbol).or_default();
-
-            processed_non_terminals.clear();
-            symbols_to_process.clear();
-            symbols_to_process.push(symbol);
-            while let Some(sym) = symbols_to_process.pop() {
-                for production in &syntax_grammar.variables[sym.index].productions {
-                    if let Some(step) = production.steps.first() {
-                        if step.symbol.is_terminal() || step.symbol.is_external() {
-                            first_set.insert(step.symbol);
-                        } else if processed_non_terminals.insert(step.symbol) {
-                            symbols_to_process.push(step.symbol);
-                        }
-                        *reserved_first_set = (*reserved_first_set).max(step.reserved_word_set_id);
-                    }
-                }
-            }
-
-            // The LAST set is defined in a similar way to the FIRST set.
-            let last_set = result.last_sets.entry(symbol).or_default();
-            processed_non_terminals.clear();
-            symbols_to_process.clear();
-            symbols_to_process.push(symbol);
-            while let Some(sym) = symbols_to_process.pop() {
-                for production in &syntax_grammar.variables[sym.index].productions {
-                    if let Some(step) = production.steps.last() {
-                        if step.symbol.is_terminal() || step.symbol.is_external() {
-                            last_set.insert(step.symbol);
-                        } else if processed_non_terminals.insert(step.symbol) {
-                            symbols_to_process.push(step.symbol);
-                        }
-                    }
-                }
-            }
-        }
-
-        // To compute an item set's transitive closure, we find each item in the set
-        // whose next symbol is a non-terminal, and we add new items to the set for
-        // each of that symbol's productions. These productions might themselves begin
-        // with non-terminals, so the process continues recursively. In this process,
-        // the total set of entries that get added depends only on two things:
-        //
-        //   * the non-terminal symbol that occurs next in each item
-        //
-        //   * the set of terminals that can follow that non-terminal symbol in the item
-        //
-        // So we can avoid a lot of duplicated recursive work by precomputing, for each
-        // non-terminal symbol `i`, a final list of *additions* that must be made to an
-        // item set when symbol `i` occurs as the next symbol in one if its core items.
-        // The structure of a precomputed *addition* is as follows:
-        //
-        //   * `item` - the new item that must be added as part of the expansion of the symbol `i`.
-        //
-        //   * `lookaheads` - the set of possible lookahead tokens that can always come after `item`
-        //     in an expansion of symbol `i`.
-        //
-        //   * `reserved_lookaheads` - the set of reserved lookahead lookahead tokens that can
-        //     always come after `item` in the expansion of symbol `i`.
-        //
-        //   * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the
-        //     *end* of the expansion of symbol `i`, so that i's own current lookahead tokens can
-        //     occur after `item`.
-        //
-        // Rather than computing these additions recursively, we use an explicit stack.
-        let empty_lookaheads = TokenSet::new();
-        let mut stack = Vec::new();
-        let mut follow_set_info_by_non_terminal = HashMap::<usize, FollowSetInfo>::new();
-        for i in 0..syntax_grammar.variables.len() {
-            // First, build up a map whose keys are all of the non-terminals that can
-            // appear at the beginning of non-terminal `i`, and whose values store
-            // information about the tokens that can follow those non-terminals.
-            stack.clear();
-            stack.push((i, &empty_lookaheads, ReservedWordSetId::default(), true));
-            follow_set_info_by_non_terminal.clear();
-            while let Some((sym_ix, lookaheads, reserved_word_set_id, propagates_lookaheads)) =
-                stack.pop()
-            {
-                let mut did_add = false;
-                let info = follow_set_info_by_non_terminal.entry(sym_ix).or_default();
-                did_add |= info.lookaheads.insert_all(lookaheads);
-                if reserved_word_set_id > info.reserved_lookaheads {
-                    info.reserved_lookaheads = reserved_word_set_id;
-                    did_add = true;
-                }
-                did_add |= propagates_lookaheads && !info.propagates_lookaheads;
-                info.propagates_lookaheads |= propagates_lookaheads;
-                if !did_add {
-                    continue;
-                }
-
-                for production in &syntax_grammar.variables[sym_ix].productions {
-                    if let Some(symbol) = production.first_symbol() {
-                        if symbol.is_non_terminal() {
-                            if let Some(next_step) = production.steps.get(1) {
-                                stack.push((
-                                    symbol.index,
-                                    &result.first_sets[&next_step.symbol],
-                                    result.reserved_first_sets[&next_step.symbol],
-                                    false,
-                                ));
-                            } else {
-                                stack.push((
-                                    symbol.index,
-                                    lookaheads,
-                                    reserved_word_set_id,
-                                    propagates_lookaheads,
-                                ));
-                            }
-                        }
-                    }
-                }
-            }
-
-            // Store all of those non-terminals' productions, along with their associated
-            // lookahead info, as *additions* associated with non-terminal `i`.
-            let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
-            for (&variable_index, follow_set_info) in &follow_set_info_by_non_terminal {
-                let variable = &syntax_grammar.variables[variable_index];
-                let non_terminal = Symbol::non_terminal(variable_index);
-                let variable_index = variable_index as u32;
-                if syntax_grammar.variables_to_inline.contains(&non_terminal) {
-                    continue;
-                }
-                for production in &variable.productions {
-                    let item = ParseItem {
-                        variable_index,
-                        production,
-                        step_index: 0,
-                        has_preceding_inherited_fields: false,
-                    };
-
-                    if let Some(inlined_productions) =
-                        inlines.inlined_productions(item.production, item.step_index)
-                    {
-                        for production in inlined_productions {
-                            find_or_push(
-                                additions_for_non_terminal,
-                                TransitiveClosureAddition {
-                                    item: item.substitute_production(production),
-                                    info: follow_set_info.clone(),
-                                },
-                            );
-                        }
-                    } else {
-                        find_or_push(
-                            additions_for_non_terminal,
-                            TransitiveClosureAddition {
-                                item,
-                                info: follow_set_info.clone(),
-                            },
-                        );
-                    }
-                }
-            }
-        }
-
-        result
-    }
-
-    pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
-        let mut result = ParseItemSet::default();
-        for entry in &item_set.entries {
-            if let Some(productions) = self
-                .inlines
-                .inlined_productions(entry.item.production, entry.item.step_index)
-            {
-                for production in productions {
-                    self.add_item(
-                        &mut result,
-                        &ParseItemSetEntry {
-                            item: entry.item.substitute_production(production),
-                            lookaheads: entry.lookaheads.clone(),
-                            following_reserved_word_set: entry.following_reserved_word_set,
-                        },
-                    );
-                }
-            } else {
-                self.add_item(&mut result, entry);
-            }
-        }
-        result
-    }
-
-    pub fn first_set(&self, symbol: &Symbol) -> &TokenSet {
-        &self.first_sets[symbol]
-    }
-
-    pub fn reserved_first_set(&self, symbol: &Symbol) -> Option<&TokenSet> {
-        let id = *self.reserved_first_sets.get(symbol)?;
-        Some(&self.syntax_grammar.reserved_word_sets[id.0])
-    }
-
-    pub fn last_set(&self, symbol: &Symbol) -> &TokenSet {
-        &self.last_sets[symbol]
-    }
-
-    fn add_item(&self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) {
-        if let Some(step) = entry.item.step() {
-            if step.symbol.is_non_terminal() {
-                let next_step = entry.item.successor().step();
-
-                // Determine which tokens can follow this non-terminal.
-                let (following_tokens, following_reserved_tokens) =
-                    if let Some(next_step) = next_step {
-                        (
-                            self.first_sets.get(&next_step.symbol).unwrap(),
-                            *self.reserved_first_sets.get(&next_step.symbol).unwrap(),
-                        )
-                    } else {
-                        (&entry.lookaheads, entry.following_reserved_word_set)
-                    };
-
-                // Use the pre-computed *additions* to expand the non-terminal.
-                for addition in &self.transitive_closure_additions[step.symbol.index] {
-                    let entry = set.insert(addition.item);
-                    entry.lookaheads.insert_all(&addition.info.lookaheads);
-
-                    if let Some(word_token) = self.syntax_grammar.word_token {
-                        if addition.info.lookaheads.contains(&word_token) {
-                            entry.following_reserved_word_set = entry
-                                .following_reserved_word_set
-                                .max(addition.info.reserved_lookaheads);
-                        }
-                    }
-
-                    if addition.info.propagates_lookaheads {
-                        entry.lookaheads.insert_all(following_tokens);
-
-                        if let Some(word_token) = self.syntax_grammar.word_token {
-                            if following_tokens.contains(&word_token) {
-                                entry.following_reserved_word_set = entry
-                                    .following_reserved_word_set
-                                    .max(following_reserved_tokens);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        let e = set.insert(entry.item);
-        e.lookaheads.insert_all(&entry.lookaheads);
-        e.following_reserved_word_set = e
-            .following_reserved_word_set
-            .max(entry.following_reserved_word_set);
-    }
-}
-
-impl fmt::Debug for ParseItemSetBuilder<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "ParseItemSetBuilder {{")?;
-
-        writeln!(f, "  first_sets: {{")?;
-        for (symbol, first_set) in &self.first_sets {
-            let name = match symbol.kind {
-                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
-                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
-                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
-                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
-            };
-            writeln!(
-                f,
-                "    first({name:?}): {}",
-                TokenSetDisplay(first_set, self.syntax_grammar, self.lexical_grammar)
-            )?;
-        }
-        writeln!(f, "  }}")?;
-
-        writeln!(f, "  last_sets: {{")?;
-        for (symbol, last_set) in &self.last_sets {
-            let name = match symbol.kind {
-                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
-                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
-                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
-                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
-            };
-            writeln!(
-                f,
-                "    last({name:?}): {}",
-                TokenSetDisplay(last_set, self.syntax_grammar, self.lexical_grammar)
-            )?;
-        }
-        writeln!(f, "  }}")?;
-
-        writeln!(f, "  additions: {{")?;
-        for (i, variable) in self.syntax_grammar.variables.iter().enumerate() {
-            writeln!(f, "    {}: {{", variable.name)?;
-            for addition in &self.transitive_closure_additions[i] {
-                writeln!(
-                    f,
-                    "      {}",
-                    ParseItemDisplay(&addition.item, self.syntax_grammar, self.lexical_grammar)
-                )?;
-            }
-            writeln!(f, "    }},")?;
-        }
-        write!(f, "  }},")?;
-
-        write!(f, "}}")?;
-        Ok(())
-    }
-}

cli/generate/src/build_tables/minimize_parse_table.rs

@@ -1,492 +0,0 @@
-use std::{
-    collections::{HashMap, HashSet},
-    mem,
-};
-
-use log::info;
-
-use super::token_conflicts::TokenConflictMap;
-use crate::{
-    dedup::split_state_id_groups,
-    grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
-    rules::{AliasMap, Symbol, TokenSet},
-    tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
-};
-
-pub fn minimize_parse_table(
-    parse_table: &mut ParseTable,
-    syntax_grammar: &SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-    simple_aliases: &AliasMap,
-    token_conflict_map: &TokenConflictMap,
-    keywords: &TokenSet,
-) {
-    let mut minimizer = Minimizer {
-        parse_table,
-        syntax_grammar,
-        lexical_grammar,
-        token_conflict_map,
-        keywords,
-        simple_aliases,
-    };
-    minimizer.merge_compatible_states();
-    minimizer.remove_unit_reductions();
-    minimizer.remove_unused_states();
-    minimizer.reorder_states_by_descending_size();
-}
-
-struct Minimizer<'a> {
-    parse_table: &'a mut ParseTable,
-    syntax_grammar: &'a SyntaxGrammar,
-    lexical_grammar: &'a LexicalGrammar,
-    token_conflict_map: &'a TokenConflictMap<'a>,
-    keywords: &'a TokenSet,
-    simple_aliases: &'a AliasMap,
-}
-
-impl Minimizer<'_> {
-    fn remove_unit_reductions(&mut self) {
-        let mut aliased_symbols = HashSet::new();
-        for variable in &self.syntax_grammar.variables {
-            for production in &variable.productions {
-                for step in &production.steps {
-                    if step.alias.is_some() {
-                        aliased_symbols.insert(step.symbol);
-                    }
-                }
-            }
-        }
-
-        let mut unit_reduction_symbols_by_state = HashMap::new();
-        for (i, state) in self.parse_table.states.iter().enumerate() {
-            let mut only_unit_reductions = true;
-            let mut unit_reduction_symbol = None;
-            for (_, entry) in &state.terminal_entries {
-                for action in &entry.actions {
-                    match action {
-                        ParseAction::ShiftExtra => continue,
-                        ParseAction::Reduce {
-                            child_count: 1,
-                            production_id: 0,
-                            symbol,
-                            ..
-                        } if !self.simple_aliases.contains_key(symbol)
-                            && !self.syntax_grammar.supertype_symbols.contains(symbol)
-                            && !self.syntax_grammar.extra_symbols.contains(symbol)
-                            && !aliased_symbols.contains(symbol)
-                            && self.syntax_grammar.variables[symbol.index].kind
-                                != VariableType::Named
-                            && (unit_reduction_symbol.is_none()
-                                || unit_reduction_symbol == Some(symbol)) =>
-                        {
-                            unit_reduction_symbol = Some(symbol);
-                            continue;
-                        }
-                        _ => {}
-                    }
-                    only_unit_reductions = false;
-                    break;
-                }
-
-                if !only_unit_reductions {
-                    break;
-                }
-            }
-
-            if let Some(symbol) = unit_reduction_symbol {
-                if only_unit_reductions {
-                    unit_reduction_symbols_by_state.insert(i, *symbol);
-                }
-            }
-        }
-
-        for state in &mut self.parse_table.states {
-            let mut done = false;
-            while !done {
-                done = true;
-                state.update_referenced_states(|other_state_id, state| {
-                    unit_reduction_symbols_by_state.get(&other_state_id).map_or(
-                        other_state_id,
-                        |symbol| {
-                            done = false;
-                            match state.nonterminal_entries.get(symbol) {
-                                Some(GotoAction::Goto(state_id)) => *state_id,
-                                _ => other_state_id,
-                            }
-                        },
-                    )
-                });
-            }
-        }
-    }
-
-    fn merge_compatible_states(&mut self) {
-        let core_count = 1 + self
-            .parse_table
-            .states
-            .iter()
-            .map(|state| state.core_id)
-            .max()
-            .unwrap();
-
-        // Initially group the states by their parse item set core.
-        let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
-        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
-        for (i, state) in self.parse_table.states.iter().enumerate() {
-            state_ids_by_group_id[state.core_id].push(i);
-            group_ids_by_state_id.push(state.core_id);
-        }
-
-        split_state_id_groups(
-            &self.parse_table.states,
-            &mut state_ids_by_group_id,
-            &mut group_ids_by_state_id,
-            0,
-            |left, right, groups| self.states_conflict(left, right, groups),
-        );
-
-        while split_state_id_groups(
-            &self.parse_table.states,
-            &mut state_ids_by_group_id,
-            &mut group_ids_by_state_id,
-            0,
-            |left, right, groups| self.state_successors_differ(left, right, groups),
-        ) {}
-
-        let error_group_index = state_ids_by_group_id
-            .iter()
-            .position(|g| g.contains(&0))
-            .unwrap();
-        let start_group_index = state_ids_by_group_id
-            .iter()
-            .position(|g| g.contains(&1))
-            .unwrap();
-        state_ids_by_group_id.swap(error_group_index, 0);
-        state_ids_by_group_id.swap(start_group_index, 1);
-
-        // Create a list of new parse states: one state for each group of old states.
-        let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
-        for state_ids in &state_ids_by_group_id {
-            // Initialize the new state based on the first old state in the group.
-            let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0]]);
-
-            // Extend the new state with all of the actions from the other old states
-            // in the group.
-            for state_id in &state_ids[1..] {
-                let other_parse_state = mem::take(&mut self.parse_table.states[*state_id]);
-
-                parse_state
-                    .terminal_entries
-                    .extend(other_parse_state.terminal_entries);
-                parse_state
-                    .nonterminal_entries
-                    .extend(other_parse_state.nonterminal_entries);
-                parse_state
-                    .reserved_words
-                    .insert_all(&other_parse_state.reserved_words);
-                for symbol in parse_state.terminal_entries.keys() {
-                    parse_state.reserved_words.remove(symbol);
-                }
-            }
-
-            // Update the new state's outgoing references using the new grouping.
-            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
-            new_states.push(parse_state);
-        }
-
-        self.parse_table.states = new_states;
-    }
-
-    fn states_conflict(
-        &self,
-        left_state: &ParseState,
-        right_state: &ParseState,
-        group_ids_by_state_id: &[ParseStateId],
-    ) -> bool {
-        for (token, left_entry) in &left_state.terminal_entries {
-            if let Some(right_entry) = right_state.terminal_entries.get(token) {
-                if self.entries_conflict(
-                    left_state.id,
-                    right_state.id,
-                    token,
-                    left_entry,
-                    right_entry,
-                    group_ids_by_state_id,
-                ) {
-                    return true;
-                }
-            } else if self.token_conflicts(left_state.id, right_state.id, right_state, *token) {
-                return true;
-            }
-        }
-
-        for token in right_state.terminal_entries.keys() {
-            if !left_state.terminal_entries.contains_key(token)
-                && self.token_conflicts(left_state.id, right_state.id, left_state, *token)
-            {
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn state_successors_differ(
-        &self,
-        state1: &ParseState,
-        state2: &ParseState,
-        group_ids_by_state_id: &[ParseStateId],
-    ) -> bool {
-        for (token, entry1) in &state1.terminal_entries {
-            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap() {
-                if let Some(entry2) = state2.terminal_entries.get(token) {
-                    if let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap() {
-                        let group1 = group_ids_by_state_id[*s1];
-                        let group2 = group_ids_by_state_id[*s2];
-                        if group1 != group2 {
-                            info!(
-                                "split states {} {} - successors for {} are split: {s1} {s2}",
-                                state1.id,
-                                state2.id,
-                                self.symbol_name(token),
-                            );
-                            return true;
-                        }
-                    }
-                }
-            }
-        }
-
-        for (symbol, s1) in &state1.nonterminal_entries {
-            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
-                match (s1, s2) {
-                    (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => {}
-                    (GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
-                        let group1 = group_ids_by_state_id[*s1];
-                        let group2 = group_ids_by_state_id[*s2];
-                        if group1 != group2 {
-                            info!(
-                                "split states {} {} - successors for {} are split: {s1} {s2}",
-                                state1.id,
-                                state2.id,
-                                self.symbol_name(symbol),
-                            );
-                            return true;
-                        }
-                    }
-                    _ => return true,
-                }
-            }
-        }
-
-        false
-    }
-
-    fn entries_conflict(
-        &self,
-        state_id1: ParseStateId,
-        state_id2: ParseStateId,
-        token: &Symbol,
-        entry1: &ParseTableEntry,
-        entry2: &ParseTableEntry,
-        group_ids_by_state_id: &[ParseStateId],
-    ) -> bool {
-        // To be compatible, entries need to have the same actions.
-        let actions1 = &entry1.actions;
-        let actions2 = &entry2.actions;
-        if actions1.len() != actions2.len() {
-            info!(
-                "split states {state_id1} {state_id2} - differing action counts for token {}",
-                self.symbol_name(token)
-            );
-            return true;
-        }
-
-        for (i, action1) in actions1.iter().enumerate() {
-            let action2 = &actions2[i];
-
-            // Two shift actions are equivalent if their destinations are in the same group.
-            if let (
-                ParseAction::Shift {
-                    state: s1,
-                    is_repetition: is_repetition1,
-                },
-                ParseAction::Shift {
-                    state: s2,
-                    is_repetition: is_repetition2,
-                },
-            ) = (action1, action2)
-            {
-                let group1 = group_ids_by_state_id[*s1];
-                let group2 = group_ids_by_state_id[*s2];
-                if group1 == group2 && is_repetition1 == is_repetition2 {
-                    continue;
-                }
-                info!(
-                    "split states {state_id1} {state_id2} - successors for {} are split: {s1} {s2}",
-                    self.symbol_name(token),
-                );
-                return true;
-            } else if action1 != action2 {
-                info!(
-                    "split states {state_id1} {state_id2} - unequal actions for {}",
-                    self.symbol_name(token),
-                );
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn token_conflicts(
-        &self,
-        left_id: ParseStateId,
-        right_id: ParseStateId,
-        right_state: &ParseState,
-        new_token: Symbol,
-    ) -> bool {
-        if new_token == Symbol::end_of_nonterminal_extra() {
-            info!("split states {left_id} {right_id} - end of non-terminal extra",);
-            return true;
-        }
-
-        // Do not add external tokens; they could conflict lexically with any of the state's
-        // existing lookahead tokens.
-        if new_token.is_external() {
-            info!(
-                "split states {left_id} {right_id} - external token {}",
-                self.symbol_name(&new_token),
-            );
-            return true;
-        }
-
-        if right_state.reserved_words.contains(&new_token) {
-            return false;
-        }
-
-        // Do not add tokens which are both internal and external. Their validity could
-        // influence the behavior of the external scanner.
-        if self
-            .syntax_grammar
-            .external_tokens
-            .iter()
-            .any(|external| external.corresponding_internal_token == Some(new_token))
-        {
-            info!(
-                "split states {left_id} {right_id} - internal/external token {}",
-                self.symbol_name(&new_token),
-            );
-            return true;
-        }
-
-        // Do not add a token if it conflicts with an existing token.
-        for token in right_state.terminal_entries.keys().copied() {
-            if !token.is_terminal() {
-                continue;
-            }
-            if self.syntax_grammar.word_token == Some(token) && self.keywords.contains(&new_token) {
-                continue;
-            }
-            if self.syntax_grammar.word_token == Some(new_token) && self.keywords.contains(&token) {
-                continue;
-            }
-
-            if self
-                .token_conflict_map
-                .does_conflict(new_token.index, token.index)
-                || self
-                    .token_conflict_map
-                    .does_match_same_string(new_token.index, token.index)
-            {
-                info!(
-                    "split states {} {} - token {} conflicts with {}",
-                    left_id,
-                    right_id,
-                    self.symbol_name(&new_token),
-                    self.symbol_name(&token),
-                );
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn symbol_name(&self, symbol: &Symbol) -> &String {
-        if symbol.is_non_terminal() {
-            &self.syntax_grammar.variables[symbol.index].name
-        } else if symbol.is_external() {
-            &self.syntax_grammar.external_tokens[symbol.index].name
-        } else {
-            &self.lexical_grammar.variables[symbol.index].name
-        }
-    }
-
-    fn remove_unused_states(&mut self) {
-        let mut state_usage_map = vec![false; self.parse_table.states.len()];
-
-        state_usage_map[0] = true;
-        state_usage_map[1] = true;
-
-        for state in &self.parse_table.states {
-            for referenced_state in state.referenced_states() {
-                state_usage_map[referenced_state] = true;
-            }
-        }
-        let mut removed_predecessor_count = 0;
-        let mut state_replacement_map = vec![0; self.parse_table.states.len()];
-        for state_id in 0..self.parse_table.states.len() {
-            state_replacement_map[state_id] = state_id - removed_predecessor_count;
-            if !state_usage_map[state_id] {
-                removed_predecessor_count += 1;
-            }
-        }
-        let mut state_id = 0;
-        let mut original_state_id = 0;
-        while state_id < self.parse_table.states.len() {
-            if state_usage_map[original_state_id] {
-                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
-                    state_replacement_map[other_state_id]
-                });
-                state_id += 1;
-            } else {
-                self.parse_table.states.remove(state_id);
-            }
-            original_state_id += 1;
-        }
-    }
-
-    fn reorder_states_by_descending_size(&mut self) {
-        // Get a mapping of old state index -> new_state_index
-        let mut old_ids_by_new_id = (0..self.parse_table.states.len()).collect::<Vec<_>>();
-        old_ids_by_new_id.sort_unstable_by_key(|i| {
-            // Don't changes states 0 (the error state) or 1 (the start state).
-            if *i <= 1 {
-                return *i as i64 - 1_000_000;
-            }
-
-            // Reorder all the other states by descending symbol count.
-            let state = &self.parse_table.states[*i];
-            -((state.terminal_entries.len() + state.nonterminal_entries.len()) as i64)
-        });
-
-        // Get the inverse mapping
-        let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
-        for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
-            new_ids_by_old_id[*old_id] = id;
-        }
-
-        // Reorder the parse states and update their references to reflect
-        // the new ordering.
-        self.parse_table.states = old_ids_by_new_id
-            .iter()
-            .map(|old_id| {
-                let mut state = ParseState::default();
-                mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
-                state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
-                state
-            })
-            .collect();
-    }
-}

cli/generate/src/build_tables/token_conflicts.rs

@@ -1,529 +0,0 @@
-use std::{cmp::Ordering, collections::HashSet, fmt};
-
-use crate::{
-    build_tables::item::TokenSetDisplay,
-    grammars::{LexicalGrammar, SyntaxGrammar},
-    nfa::{CharacterSet, NfaCursor, NfaTransition},
-    rules::TokenSet,
-};
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-struct TokenConflictStatus {
-    matches_prefix: bool,
-    does_match_continuation: bool,
-    does_match_valid_continuation: bool,
-    does_match_separators: bool,
-    matches_same_string: bool,
-    matches_different_string: bool,
-}
-
-pub struct TokenConflictMap<'a> {
-    n: usize,
-    status_matrix: Vec<TokenConflictStatus>,
-    following_tokens: Vec<TokenSet>,
-    starting_chars_by_index: Vec<CharacterSet>,
-    following_chars_by_index: Vec<CharacterSet>,
-    grammar: &'a LexicalGrammar,
-}
-
-impl<'a> TokenConflictMap<'a> {
-    /// Create a token conflict map based on a lexical grammar, which describes the structure
-    /// each token, and a `following_token` map, which indicates which tokens may be appear
-    /// immediately after each other token.
-    ///
-    /// This analyzes the possible kinds of overlap between each pair of tokens and stores
-    /// them in a matrix.
-    pub fn new(grammar: &'a LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
-        let mut cursor = NfaCursor::new(&grammar.nfa, Vec::new());
-        let starting_chars = get_starting_chars(&mut cursor, grammar);
-        let following_chars = get_following_chars(&starting_chars, &following_tokens);
-
-        let n = grammar.variables.len();
-        let mut status_matrix = vec![TokenConflictStatus::default(); n * n];
-        for i in 0..grammar.variables.len() {
-            for j in 0..i {
-                let status = compute_conflict_status(&mut cursor, grammar, &following_chars, i, j);
-                status_matrix[matrix_index(n, i, j)] = status.0;
-                status_matrix[matrix_index(n, j, i)] = status.1;
-            }
-        }
-
-        TokenConflictMap {
-            n,
-            status_matrix,
-            following_tokens,
-            starting_chars_by_index: starting_chars,
-            following_chars_by_index: following_chars,
-            grammar,
-        }
-    }
-
-    /// Does token `i` match any strings that token `j` also matches, such that token `i`
-    /// is preferred over token `j`?
-    pub fn has_same_conflict_status(&self, a: usize, b: usize, other: usize) -> bool {
-        let left = &self.status_matrix[matrix_index(self.n, a, other)];
-        let right = &self.status_matrix[matrix_index(self.n, b, other)];
-        left == right
-    }
-
-    /// Does token `i` match any strings that token `j` does *not* match?
-    pub fn does_match_different_string(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_different_string
-    }
-
-    /// Does token `i` match any strings that token `j` also matches, where
-    /// token `i` is preferred over token `j`?
-    pub fn does_match_same_string(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_same_string
-    }
-
-    pub fn does_conflict(&self, i: usize, j: usize) -> bool {
-        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
-        entry.does_match_valid_continuation
-            || entry.does_match_separators
-            || entry.matches_same_string
-    }
-
-    /// Does token `i` match any strings that are *prefixes* of strings matched by `j`?
-    pub fn does_match_prefix(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_prefix
-    }
-
-    pub fn does_match_shorter_or_longer(&self, i: usize, j: usize) -> bool {
-        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
-        let reverse_entry = &self.status_matrix[matrix_index(self.n, j, i)];
-        (entry.does_match_valid_continuation || entry.does_match_separators)
-            && !reverse_entry.does_match_separators
-    }
-
-    pub fn does_overlap(&self, i: usize, j: usize) -> bool {
-        let status = &self.status_matrix[matrix_index(self.n, i, j)];
-        status.does_match_separators
-            || status.matches_prefix
-            || status.matches_same_string
-            || status.does_match_continuation
-    }
-
-    pub fn prefer_token(grammar: &LexicalGrammar, left: (i32, usize), right: (i32, usize)) -> bool {
-        match left.0.cmp(&right.0) {
-            Ordering::Less => false,
-            Ordering::Greater => true,
-            Ordering::Equal => match grammar.variables[left.1]
-                .implicit_precedence
-                .cmp(&grammar.variables[right.1].implicit_precedence)
-            {
-                Ordering::Less => false,
-                Ordering::Greater => true,
-                Ordering::Equal => left.1 < right.1,
-            },
-        }
-    }
-
-    pub fn prefer_transition(
-        grammar: &LexicalGrammar,
-        t: &NfaTransition,
-        completed_id: usize,
-        completed_precedence: i32,
-        has_separator_transitions: bool,
-    ) -> bool {
-        if t.precedence < completed_precedence {
-            return false;
-        }
-        if t.precedence == completed_precedence {
-            if t.is_separator {
-                return false;
-            }
-            if has_separator_transitions
-                && !grammar
-                    .variable_indices_for_nfa_states(&t.states)
-                    .any(|i| i == completed_id)
-            {
-                return false;
-            }
-        }
-        true
-    }
-}
-
-impl fmt::Debug for TokenConflictMap<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "TokenConflictMap {{")?;
-
-        let syntax_grammar = SyntaxGrammar::default();
-
-        writeln!(f, "  following_tokens: {{")?;
-        for (i, following_tokens) in self.following_tokens.iter().enumerate() {
-            writeln!(
-                f,
-                "    follow({:?}): {},",
-                self.grammar.variables[i].name,
-                TokenSetDisplay(following_tokens, &syntax_grammar, self.grammar)
-            )?;
-        }
-        writeln!(f, "  }},")?;
-
-        writeln!(f, "  starting_characters: {{")?;
-        for i in 0..self.n {
-            writeln!(
-                f,
-                "    {:?}: {:?},",
-                self.grammar.variables[i].name, self.starting_chars_by_index[i]
-            )?;
-        }
-        writeln!(f, "  }},")?;
-
-        writeln!(f, "  following_characters: {{")?;
-        for i in 0..self.n {
-            writeln!(
-                f,
-                "    {:?}: {:?},",
-                self.grammar.variables[i].name, self.following_chars_by_index[i]
-            )?;
-        }
-        writeln!(f, "  }},")?;
-
-        writeln!(f, "  status_matrix: {{")?;
-        for i in 0..self.n {
-            writeln!(f, "    {:?}: {{", self.grammar.variables[i].name)?;
-            for j in 0..self.n {
-                writeln!(
-                    f,
-                    "      {:?}: {:?},",
-                    self.grammar.variables[j].name,
-                    self.status_matrix[matrix_index(self.n, i, j)]
-                )?;
-            }
-            writeln!(f, "    }},")?;
-        }
-        write!(f, "  }},")?;
-        write!(f, "}}")?;
-        Ok(())
-    }
-}
-
-const fn matrix_index(variable_count: usize, i: usize, j: usize) -> usize {
-    variable_count * i + j
-}
-
-fn get_starting_chars(cursor: &mut NfaCursor, grammar: &LexicalGrammar) -> Vec<CharacterSet> {
-    let mut result = Vec::with_capacity(grammar.variables.len());
-    for variable in &grammar.variables {
-        cursor.reset(vec![variable.start_state]);
-        let mut all_chars = CharacterSet::empty();
-        for (chars, _) in cursor.transition_chars() {
-            all_chars = all_chars.add(chars);
-        }
-        result.push(all_chars);
-    }
-    result
-}
-
-fn get_following_chars(
-    starting_chars: &[CharacterSet],
-    following_tokens: &[TokenSet],
-) -> Vec<CharacterSet> {
-    following_tokens
-        .iter()
-        .map(|following_tokens| {
-            let mut chars = CharacterSet::empty();
-            for token in following_tokens.iter() {
-                if token.is_terminal() {
-                    chars = chars.add(&starting_chars[token.index]);
-                }
-            }
-            chars
-        })
-        .collect()
-}
-
-fn compute_conflict_status(
-    cursor: &mut NfaCursor,
-    grammar: &LexicalGrammar,
-    following_chars: &[CharacterSet],
-    i: usize,
-    j: usize,
-) -> (TokenConflictStatus, TokenConflictStatus) {
-    let mut visited_state_sets = HashSet::new();
-    let mut state_set_queue = vec![vec![
-        grammar.variables[i].start_state,
-        grammar.variables[j].start_state,
-    ]];
-    let mut result = (
-        TokenConflictStatus::default(),
-        TokenConflictStatus::default(),
-    );
-
-    while let Some(state_set) = state_set_queue.pop() {
-        let mut live_variable_indices = grammar.variable_indices_for_nfa_states(&state_set);
-
-        // If only one of the two tokens could possibly match from this state, then
-        // there is no reason to analyze any of its successors. Just record the fact
-        // that the token matches a string that the other token does not match.
-        let first_live_variable_index = live_variable_indices.next().unwrap();
-        if live_variable_indices.count() == 0 {
-            if first_live_variable_index == i {
-                result.0.matches_different_string = true;
-            } else {
-                result.1.matches_different_string = true;
-            }
-            continue;
-        }
-
-        // Don't pursue states where there's no potential for conflict.
-        cursor.reset(state_set);
-        let within_separator = cursor.transition_chars().any(|(_, sep)| sep);
-
-        // Examine each possible completed token in this state.
-        let mut completion = None;
-        for (id, precedence) in cursor.completions() {
-            if within_separator {
-                if id == i {
-                    result.0.does_match_separators = true;
-                } else {
-                    result.1.does_match_separators = true;
-                }
-            }
-
-            // If the other token has already completed, then this is
-            // a same-string conflict.
-            if let Some((prev_id, prev_precedence)) = completion {
-                if id == prev_id {
-                    continue;
-                }
-
-                // Determine which of the two tokens is preferred.
-                let preferred_id;
-                if TokenConflictMap::prefer_token(
-                    grammar,
-                    (prev_precedence, prev_id),
-                    (precedence, id),
-                ) {
-                    preferred_id = prev_id;
-                } else {
-                    preferred_id = id;
-                    completion = Some((id, precedence));
-                }
-
-                if preferred_id == i {
-                    result.0.matches_same_string = true;
-                } else {
-                    result.1.matches_same_string = true;
-                }
-            } else {
-                completion = Some((id, precedence));
-            }
-        }
-
-        // Examine each possible transition from this state to detect substring conflicts.
-        for transition in cursor.transitions() {
-            let mut can_advance = true;
-
-            // If there is already a completed token in this state, then determine
-            // if the next state can also match the completed token. If so, then
-            // this is *not* a conflict.
-            if let Some((completed_id, completed_precedence)) = completion {
-                let mut advanced_id = None;
-                let mut successor_contains_completed_id = false;
-                for variable_id in grammar.variable_indices_for_nfa_states(&transition.states) {
-                    if variable_id == completed_id {
-                        successor_contains_completed_id = true;
-                        break;
-                    }
-                    advanced_id = Some(variable_id);
-                }
-
-                // Determine which action is preferred: matching the already complete
-                // token, or continuing on to try and match the other longer token.
-                if let (Some(advanced_id), false) = (advanced_id, successor_contains_completed_id) {
-                    if TokenConflictMap::prefer_transition(
-                        grammar,
-                        &transition,
-                        completed_id,
-                        completed_precedence,
-                        within_separator,
-                    ) {
-                        can_advance = true;
-                        if advanced_id == i {
-                            result.0.does_match_continuation = true;
-                            if transition.characters.does_intersect(&following_chars[j]) {
-                                result.0.does_match_valid_continuation = true;
-                            }
-                        } else {
-                            result.1.does_match_continuation = true;
-                            if transition.characters.does_intersect(&following_chars[i]) {
-                                result.1.does_match_valid_continuation = true;
-                            }
-                        }
-                    } else if completed_id == i {
-                        result.0.matches_prefix = true;
-                    } else {
-                        result.1.matches_prefix = true;
-                    }
-                }
-            }
-
-            if can_advance && visited_state_sets.insert(transition.states.clone()) {
-                state_set_queue.push(transition.states);
-            }
-        }
-    }
-    result
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::{Variable, VariableType},
-        prepare_grammar::{expand_tokens, ExtractedLexicalGrammar},
-        rules::{Precedence, Rule, Symbol},
-    };
-
-    #[test]
-    fn test_starting_characters() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: Vec::new(),
-            variables: vec![
-                Variable {
-                    name: "token_0".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("[a-f]1|0x\\d", ""),
-                },
-                Variable {
-                    name: "token_1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("d*ef", ""),
-                },
-            ],
-        })
-        .unwrap();
-
-        let token_map = TokenConflictMap::new(&grammar, Vec::new());
-
-        assert_eq!(
-            token_map.starting_chars_by_index[0],
-            CharacterSet::empty().add_range('a', 'f').add_char('0')
-        );
-        assert_eq!(
-            token_map.starting_chars_by_index[1],
-            CharacterSet::empty().add_range('d', 'e')
-        );
-    }
-
-    #[test]
-    fn test_token_conflicts() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: Vec::new(),
-            variables: vec![
-                Variable {
-                    name: "in".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("in"),
-                },
-                Variable {
-                    name: "identifier".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("\\w+", ""),
-                },
-                Variable {
-                    name: "instanceof".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("instanceof"),
-                },
-            ],
-        })
-        .unwrap();
-
-        let var = |name| index_of_var(&grammar, name);
-
-        let token_map = TokenConflictMap::new(
-            &grammar,
-            vec![
-                std::iter::once(&Symbol::terminal(var("identifier")))
-                    .copied()
-                    .collect(),
-                std::iter::once(&Symbol::terminal(var("in")))
-                    .copied()
-                    .collect(),
-                std::iter::once(&Symbol::terminal(var("identifier")))
-                    .copied()
-                    .collect(),
-            ],
-        );
-
-        // Given the string "in", the `in` token is preferred over the `identifier` token
-        assert!(token_map.does_match_same_string(var("in"), var("identifier")));
-        assert!(!token_map.does_match_same_string(var("identifier"), var("in")));
-
-        // Depending on what character follows, the string "in" may be treated as part of an
-        // `identifier` token.
-        assert!(token_map.does_conflict(var("identifier"), var("in")));
-
-        // Depending on what character follows, the string "instanceof" may be treated as part of
-        // an `identifier` token.
-        assert!(token_map.does_conflict(var("identifier"), var("instanceof")));
-        assert!(token_map.does_conflict(var("instanceof"), var("in")));
-    }
-
-    #[test]
-    fn test_token_conflicts_with_separators() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: vec![Rule::pattern("\\s", "")],
-            variables: vec![
-                Variable {
-                    name: "x".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("x"),
-                },
-                Variable {
-                    name: "newline".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("\n"),
-                },
-            ],
-        })
-        .unwrap();
-
-        let var = |name| index_of_var(&grammar, name);
-
-        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
-
-        assert!(token_map.does_conflict(var("newline"), var("x")));
-        assert!(!token_map.does_conflict(var("x"), var("newline")));
-    }
-
-    #[test]
-    fn test_token_conflicts_with_open_ended_tokens() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: vec![Rule::pattern("\\s", "")],
-            variables: vec![
-                Variable {
-                    name: "x".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("x"),
-                },
-                Variable {
-                    name: "anything".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*", "")),
-                },
-            ],
-        })
-        .unwrap();
-
-        let var = |name| index_of_var(&grammar, name);
-
-        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
-
-        assert!(token_map.does_match_shorter_or_longer(var("anything"), var("x")));
-        assert!(!token_map.does_match_shorter_or_longer(var("x"), var("anything")));
-    }
-
-    fn index_of_var(grammar: &LexicalGrammar, name: &str) -> usize {
-        grammar
-            .variables
-            .iter()
-            .position(|v| v.name == name)
-            .unwrap()
-    }
-}

cli/generate/src/grammar_files.rs

@@ -1 +0,0 @@
-

cli/generate/src/grammars.rs

@@ -1,273 +0,0 @@
-use std::{collections::HashMap, fmt};
-
-use super::{
-    nfa::Nfa,
-    rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet},
-};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub enum VariableType {
-    Hidden,
-    Auxiliary,
-    Anonymous,
-    Named,
-}
-
-// Input grammar
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct Variable {
-    pub name: String,
-    pub kind: VariableType,
-    pub rule: Rule,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum PrecedenceEntry {
-    Name(String),
-    Symbol(String),
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct InputGrammar {
-    pub name: String,
-    pub variables: Vec<Variable>,
-    pub extra_symbols: Vec<Rule>,
-    pub expected_conflicts: Vec<Vec<String>>,
-    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
-    pub external_tokens: Vec<Rule>,
-    pub variables_to_inline: Vec<String>,
-    pub supertype_symbols: Vec<String>,
-    pub word_token: Option<String>,
-    pub reserved_words: Vec<ReservedWordContext<Rule>>,
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct ReservedWordContext<T> {
-    pub name: String,
-    pub reserved_words: Vec<T>,
-}
-
-// Extracted lexical grammar
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct LexicalVariable {
-    pub name: String,
-    pub kind: VariableType,
-    pub implicit_precedence: i32,
-    pub start_state: u32,
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct LexicalGrammar {
-    pub nfa: Nfa,
-    pub variables: Vec<LexicalVariable>,
-}
-
-// Extracted syntax grammar
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct ProductionStep {
-    pub symbol: Symbol,
-    pub precedence: Precedence,
-    pub associativity: Option<Associativity>,
-    pub alias: Option<Alias>,
-    pub field_name: Option<String>,
-    pub reserved_word_set_id: ReservedWordSetId,
-}
-
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct ReservedWordSetId(pub usize);
-
-impl fmt::Display for ReservedWordSetId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-pub const NO_RESERVED_WORDS: ReservedWordSetId = ReservedWordSetId(usize::MAX);
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct Production {
-    pub steps: Vec<ProductionStep>,
-    pub dynamic_precedence: i32,
-}
-
-#[derive(Default)]
-pub struct InlinedProductionMap {
-    pub productions: Vec<Production>,
-    pub production_map: HashMap<(*const Production, u32), Vec<usize>>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct SyntaxVariable {
-    pub name: String,
-    pub kind: VariableType,
-    pub productions: Vec<Production>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ExternalToken {
-    pub name: String,
-    pub kind: VariableType,
-    pub corresponding_internal_token: Option<Symbol>,
-}
-
-#[derive(Debug, Default)]
-pub struct SyntaxGrammar {
-    pub variables: Vec<SyntaxVariable>,
-    pub extra_symbols: Vec<Symbol>,
-    pub expected_conflicts: Vec<Vec<Symbol>>,
-    pub external_tokens: Vec<ExternalToken>,
-    pub supertype_symbols: Vec<Symbol>,
-    pub variables_to_inline: Vec<Symbol>,
-    pub word_token: Option<Symbol>,
-    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
-    pub reserved_word_sets: Vec<TokenSet>,
-}
-
-#[cfg(test)]
-impl ProductionStep {
-    #[must_use]
-    pub fn new(symbol: Symbol) -> Self {
-        Self {
-            symbol,
-            precedence: Precedence::None,
-            associativity: None,
-            alias: None,
-            field_name: None,
-            reserved_word_set_id: ReservedWordSetId::default(),
-        }
-    }
-
-    pub fn with_prec(
-        mut self,
-        precedence: Precedence,
-        associativity: Option<Associativity>,
-    ) -> Self {
-        self.precedence = precedence;
-        self.associativity = associativity;
-        self
-    }
-
-    pub fn with_alias(mut self, value: &str, is_named: bool) -> Self {
-        self.alias = Some(Alias {
-            value: value.to_string(),
-            is_named,
-        });
-        self
-    }
-
-    pub fn with_field_name(mut self, name: &str) -> Self {
-        self.field_name = Some(name.to_string());
-        self
-    }
-}
-
-impl Production {
-    pub fn first_symbol(&self) -> Option<Symbol> {
-        self.steps.first().map(|s| s.symbol)
-    }
-}
-
-#[cfg(test)]
-impl Variable {
-    pub fn named(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Named,
-            rule,
-        }
-    }
-
-    pub fn auxiliary(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Auxiliary,
-            rule,
-        }
-    }
-
-    pub fn hidden(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Hidden,
-            rule,
-        }
-    }
-
-    pub fn anonymous(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Anonymous,
-            rule,
-        }
-    }
-}
-
-impl VariableType {
-    pub fn is_visible(self) -> bool {
-        self == Self::Named || self == Self::Anonymous
-    }
-}
-
-impl LexicalGrammar {
-    pub fn variable_indices_for_nfa_states<'a>(
-        &'a self,
-        state_ids: &'a [u32],
-    ) -> impl Iterator<Item = usize> + 'a {
-        let mut prev = None;
-        state_ids.iter().filter_map(move |state_id| {
-            let variable_id = self.variable_index_for_nfa_state(*state_id);
-            if prev == Some(variable_id) {
-                None
-            } else {
-                prev = Some(variable_id);
-                prev
-            }
-        })
-    }
-
-    pub fn variable_index_for_nfa_state(&self, state_id: u32) -> usize {
-        self.variables
-            .iter()
-            .position(|v| v.start_state >= state_id)
-            .unwrap()
-    }
-}
-
-impl SyntaxVariable {
-    pub fn is_auxiliary(&self) -> bool {
-        self.kind == VariableType::Auxiliary
-    }
-
-    pub fn is_hidden(&self) -> bool {
-        self.kind == VariableType::Hidden || self.kind == VariableType::Auxiliary
-    }
-}
-
-impl InlinedProductionMap {
-    pub fn inlined_productions<'a>(
-        &'a self,
-        production: &Production,
-        step_index: u32,
-    ) -> Option<impl Iterator<Item = &'a Production> + 'a> {
-        self.production_map
-            .get(&(std::ptr::from_ref::<Production>(production), step_index))
-            .map(|production_indices| {
-                production_indices
-                    .iter()
-                    .copied()
-                    .map(move |index| &self.productions[index])
-            })
-    }
-}
-
-impl fmt::Display for PrecedenceEntry {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Self::Name(n) => write!(f, "'{n}'"),
-            Self::Symbol(s) => write!(f, "$.{s}"),
-        }
-    }
-}

cli/generate/src/lib.rs

@@ -1,439 +0,0 @@
-use std::{
-    env, fs,
-    io::Write,
-    path::{Path, PathBuf},
-    process::{Command, Stdio},
-    sync::LazyLock,
-};
-
-use anyhow::Result;
-use regex::{Regex, RegexBuilder};
-use semver::Version;
-use serde::{Deserialize, Serialize};
-use thiserror::Error;
-
-mod build_tables;
-mod dedup;
-mod grammar_files;
-mod grammars;
-mod nfa;
-mod node_types;
-pub mod parse_grammar;
-mod prepare_grammar;
-mod render;
-mod rules;
-mod tables;
-
-use build_tables::build_tables;
-pub use build_tables::ParseTableBuilderError;
-use grammars::InputGrammar;
-pub use node_types::VariableInfoError;
-use parse_grammar::parse_grammar;
-pub use parse_grammar::ParseGrammarError;
-use prepare_grammar::prepare_grammar;
-pub use prepare_grammar::PrepareGrammarError;
-use render::render_c_code;
-pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
-
-static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
-    RegexBuilder::new("^\\s*//.*")
-        .multi_line(true)
-        .build()
-        .unwrap()
-});
-
-struct GeneratedParser {
-    c_code: String,
-    node_types_json: String,
-}
-
-pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h");
-pub const ARRAY_HEADER: &str = include_str!("templates/array.h");
-
-pub type GenerateResult<T> = Result<T, GenerateError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum GenerateError {
-    #[error("Error with specified path -- {0}")]
-    GrammarPath(String),
-    #[error("{0}")]
-    IO(String),
-    #[error(transparent)]
-    LoadGrammarFile(#[from] LoadGrammarError),
-    #[error(transparent)]
-    ParseGrammar(#[from] ParseGrammarError),
-    #[error(transparent)]
-    Prepare(#[from] PrepareGrammarError),
-    #[error(transparent)]
-    VariableInfo(#[from] VariableInfoError),
-    #[error(transparent)]
-    BuildTables(#[from] ParseTableBuilderError),
-    #[error(transparent)]
-    ParseVersion(#[from] ParseVersionError),
-}
-
-impl From<std::io::Error> for GenerateError {
-    fn from(value: std::io::Error) -> Self {
-        Self::IO(value.to_string())
-    }
-}
-
-pub type LoadGrammarFileResult<T> = Result<T, LoadGrammarError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum LoadGrammarError {
-    #[error("Path to a grammar file with `.js` or `.json` extension is required")]
-    InvalidPath,
-    #[error("Failed to load grammar.js -- {0}")]
-    LoadJSGrammarFile(#[from] JSError),
-    #[error("Failed to load grammar.json -- {0}")]
-    IO(String),
-    #[error("Unknown grammar file extension: {0:?}")]
-    FileExtension(PathBuf),
-}
-
-impl From<std::io::Error> for LoadGrammarError {
-    fn from(value: std::io::Error) -> Self {
-        Self::IO(value.to_string())
-    }
-}
-
-#[derive(Debug, Error, Serialize)]
-pub enum ParseVersionError {
-    #[error("{0}")]
-    Version(String),
-    #[error("{0}")]
-    JSON(String),
-    #[error("{0}")]
-    IO(String),
-}
-
-pub type JSResult<T> = Result<T, JSError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum JSError {
-    #[error("Failed to run `{runtime}` -- {error}")]
-    JSRuntimeSpawn { runtime: String, error: String },
-    #[error("Got invalid UTF8 from `{runtime}` -- {error}")]
-    JSRuntimeUtf8 { runtime: String, error: String },
-    #[error("`{runtime}` process exited with status {code}")]
-    JSRuntimeExit { runtime: String, code: i32 },
-    #[error("{0}")]
-    IO(String),
-    #[error("Could not parse this package's version as semver -- {0}")]
-    Semver(String),
-    #[error("Failed to serialze grammar JSON -- {0}")]
-    Serialzation(String),
-}
-
-impl From<std::io::Error> for JSError {
-    fn from(value: std::io::Error) -> Self {
-        Self::IO(value.to_string())
-    }
-}
-
-impl From<serde_json::Error> for JSError {
-    fn from(value: serde_json::Error) -> Self {
-        Self::Serialzation(value.to_string())
-    }
-}
-
-impl From<semver::Error> for JSError {
-    fn from(value: semver::Error) -> Self {
-        Self::Semver(value.to_string())
-    }
-}
-
-pub fn generate_parser_in_directory(
-    repo_path: &Path,
-    out_path: Option<&str>,
-    grammar_path: Option<&str>,
-    mut abi_version: usize,
-    report_symbol_name: Option<&str>,
-    js_runtime: Option<&str>,
-) -> GenerateResult<()> {
-    let mut repo_path = repo_path.to_owned();
-    let mut grammar_path = grammar_path;
-
-    // Populate a new empty grammar directory.
-    if let Some(path) = grammar_path {
-        let path = PathBuf::from(path);
-        if !path
-            .try_exists()
-            .map_err(|e| GenerateError::GrammarPath(e.to_string()))?
-        {
-            fs::create_dir_all(&path)?;
-            grammar_path = None;
-            repo_path = path;
-        }
-    }
-
-    let grammar_path = grammar_path.map_or_else(|| repo_path.join("grammar.js"), PathBuf::from);
-
-    // Read the grammar file.
-    let grammar_json = load_grammar_file(&grammar_path, js_runtime)?;
-
-    let src_path = out_path.map_or_else(|| repo_path.join("src"), PathBuf::from);
-    let header_path = src_path.join("tree_sitter");
-
-    // Ensure that the output directories exist.
-    fs::create_dir_all(&src_path)?;
-    fs::create_dir_all(&header_path)?;
-
-    if grammar_path.file_name().unwrap() != "grammar.json" {
-        fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| {
-            GenerateError::IO(format!(
-                "Failed to write grammar.json to {src_path:?} -- {e}"
-            ))
-        })?;
-    }
-
-    // Parse and preprocess the grammar.
-    let input_grammar = parse_grammar(&grammar_json)?;
-
-    let semantic_version = read_grammar_version(&repo_path)?;
-
-    if semantic_version.is_none() && abi_version > ABI_VERSION_MIN {
-        println!("Warning: No `tree-sitter.json` file found in your grammar, this file is required to generate with ABI {abi_version}. Using ABI version {ABI_VERSION_MIN} instead.");
-        println!("This file can be set up with `tree-sitter init`. For more information, see https://tree-sitter.github.io/tree-sitter/cli/init.");
-        abi_version = ABI_VERSION_MIN;
-    }
-
-    // Generate the parser and related files.
-    let GeneratedParser {
-        c_code,
-        node_types_json,
-    } = generate_parser_for_grammar_with_opts(
-        &input_grammar,
-        abi_version,
-        semantic_version.map(|v| (v.major as u8, v.minor as u8, v.patch as u8)),
-        report_symbol_name,
-    )?;
-
-    write_file(&src_path.join("parser.c"), c_code)?;
-    write_file(&src_path.join("node-types.json"), node_types_json)?;
-    write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
-    write_file(&header_path.join("array.h"), ARRAY_HEADER)?;
-    write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
-
-    Ok(())
-}
-
-pub fn generate_parser_for_grammar(
-    grammar_json: &str,
-    semantic_version: Option<(u8, u8, u8)>,
-) -> GenerateResult<(String, String)> {
-    let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n");
-    let input_grammar = parse_grammar(&grammar_json)?;
-    let parser = generate_parser_for_grammar_with_opts(
-        &input_grammar,
-        tree_sitter::LANGUAGE_VERSION,
-        semantic_version,
-        None,
-    )?;
-    Ok((input_grammar.name, parser.c_code))
-}
-
-fn generate_parser_for_grammar_with_opts(
-    input_grammar: &InputGrammar,
-    abi_version: usize,
-    semantic_version: Option<(u8, u8, u8)>,
-    report_symbol_name: Option<&str>,
-) -> GenerateResult<GeneratedParser> {
-    let (syntax_grammar, lexical_grammar, inlines, simple_aliases) =
-        prepare_grammar(input_grammar)?;
-    let variable_info =
-        node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?;
-    let node_types_json = node_types::generate_node_types_json(
-        &syntax_grammar,
-        &lexical_grammar,
-        &simple_aliases,
-        &variable_info,
-    );
-    let supertype_symbol_map =
-        node_types::get_supertype_symbol_map(&syntax_grammar, &simple_aliases, &variable_info);
-    let tables = build_tables(
-        &syntax_grammar,
-        &lexical_grammar,
-        &simple_aliases,
-        &variable_info,
-        &inlines,
-        report_symbol_name,
-    )?;
-    let c_code = render_c_code(
-        &input_grammar.name,
-        tables,
-        syntax_grammar,
-        lexical_grammar,
-        simple_aliases,
-        abi_version,
-        semantic_version,
-        supertype_symbol_map,
-    );
-    Ok(GeneratedParser {
-        c_code,
-        node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(),
-    })
-}
-
-/// This will read the `tree-sitter.json` config file and attempt to extract the version.
-///
-/// If the file is not found in the current directory or any of its parent directories, this will
-/// return `None` to maintain backwards compatibility. If the file is found but the version cannot
-/// be parsed as semver, this will return an error.
-fn read_grammar_version(repo_path: &Path) -> Result<Option<Version>, ParseVersionError> {
-    #[derive(Deserialize)]
-    struct TreeSitterJson {
-        metadata: Metadata,
-    }
-
-    #[derive(Deserialize)]
-    struct Metadata {
-        version: String,
-    }
-
-    let filename = "tree-sitter.json";
-    let mut path = repo_path.join(filename);
-
-    loop {
-        let json = path
-            .exists()
-            .then(|| {
-                let contents = fs::read_to_string(path.as_path()).map_err(|e| {
-                    ParseVersionError::IO(format!("Failed to read `{}` -- {e}", path.display()))
-                })?;
-                serde_json::from_str::<TreeSitterJson>(&contents).map_err(|e| {
-                    ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display()))
-                })
-            })
-            .transpose()?;
-        if let Some(json) = json {
-            return Version::parse(&json.metadata.version)
-                .map_err(|e| {
-                    ParseVersionError::Version(format!(
-                        "Failed to parse `{}` version as semver -- {e}",
-                        path.display()
-                    ))
-                })
-                .map(Some);
-        }
-        path.pop(); // filename
-        if !path.pop() {
-            return Ok(None);
-        }
-        path.push(filename);
-    }
-}
-
-pub fn load_grammar_file(
-    grammar_path: &Path,
-    js_runtime: Option<&str>,
-) -> LoadGrammarFileResult<String> {
-    if grammar_path.is_dir() {
-        Err(LoadGrammarError::InvalidPath)?;
-    }
-    match grammar_path.extension().and_then(|e| e.to_str()) {
-        Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?),
-        Some("json") => Ok(fs::read_to_string(grammar_path)?),
-        _ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?,
-    }
-}
-
-fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult<String> {
-    let grammar_path = fs::canonicalize(grammar_path)?;
-
-    #[cfg(windows)]
-    let grammar_path = url::Url::from_file_path(grammar_path)
-        .expect("Failed to convert path to URL")
-        .to_string();
-
-    let js_runtime = js_runtime.unwrap_or("node");
-
-    let mut js_command = Command::new(js_runtime);
-    match js_runtime {
-        "node" => {
-            js_command.args(["--input-type=module", "-"]);
-        }
-        "bun" => {
-            js_command.arg("-");
-        }
-        "deno" => {
-            js_command.args(["run", "--allow-all", "-"]);
-        }
-        _ => {}
-    }
-
-    let mut js_process = js_command
-        .env("TREE_SITTER_GRAMMAR_PATH", grammar_path)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()
-        .map_err(|e| JSError::JSRuntimeSpawn {
-            runtime: js_runtime.to_string(),
-            error: e.to_string(),
-        })?;
-
-    let mut js_stdin = js_process
-        .stdin
-        .take()
-        .ok_or_else(|| JSError::IO(format!("Failed to open stdin for `{js_runtime}`")))?;
-
-    let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?;
-    write!(
-        js_stdin,
-        "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {};
-         globalThis.TREE_SITTER_CLI_VERSION_MINOR = {};
-         globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
-        cli_version.major, cli_version.minor, cli_version.patch,
-    )
-    .map_err(|e| {
-        JSError::IO(format!(
-            "Failed to write tree-sitter version to `{js_runtime}`'s stdin -- {e}"
-        ))
-    })?;
-    js_stdin.write(include_bytes!("./dsl.js")).map_err(|e| {
-        JSError::IO(format!(
-            "Failed to write grammar dsl to `{js_runtime}`'s stdin -- {e}"
-        ))
-    })?;
-    drop(js_stdin);
-
-    let output = js_process
-        .wait_with_output()
-        .map_err(|e| JSError::IO(format!("Failed to read output from `{js_runtime}` -- {e}")))?;
-    match output.status.code() {
-        None => panic!("`{js_runtime}` process was killed"),
-        Some(0) => {
-            let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 {
-                runtime: js_runtime.to_string(),
-                error: e.to_string(),
-            })?;
-
-            let mut grammar_json = &stdout[..];
-
-            if let Some(pos) = stdout.rfind('\n') {
-                // If there's a newline, split the last line from the rest of the output
-                let node_output = &stdout[..pos];
-                grammar_json = &stdout[pos + 1..];
-
-                let mut stdout = std::io::stdout().lock();
-                stdout.write_all(node_output.as_bytes())?;
-                stdout.write_all(b"\n")?;
-                stdout.flush()?;
-            }
-
-            Ok(serde_json::to_string_pretty(&serde_json::from_str::<
-                serde_json::Value,
-            >(grammar_json)?)?)
-        }
-        Some(code) => Err(JSError::JSRuntimeExit {
-            runtime: js_runtime.to_string(),
-            code,
-        }),
-    }
-}
-
-pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> {
-    fs::write(path, body)
-        .map_err(|e| GenerateError::IO(format!("Failed to write {:?} -- {e}", path.file_name())))
-}

cli/generate/src/parse_grammar.rs

@@ -1,433 +0,0 @@
-use std::collections::HashSet;
-
-use anyhow::Result;
-use serde::{Deserialize, Serialize};
-use serde_json::{Map, Value};
-use thiserror::Error;
-
-use super::{
-    grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType},
-    rules::{Precedence, Rule},
-};
-use crate::grammars::ReservedWordContext;
-
-#[derive(Deserialize)]
-#[serde(tag = "type")]
-#[allow(non_camel_case_types)]
-#[allow(clippy::upper_case_acronyms)]
-enum RuleJSON {
-    ALIAS {
-        content: Box<RuleJSON>,
-        named: bool,
-        value: String,
-    },
-    BLANK,
-    STRING {
-        value: String,
-    },
-    PATTERN {
-        value: String,
-        flags: Option<String>,
-    },
-    SYMBOL {
-        name: String,
-    },
-    CHOICE {
-        members: Vec<RuleJSON>,
-    },
-    FIELD {
-        name: String,
-        content: Box<RuleJSON>,
-    },
-    SEQ {
-        members: Vec<RuleJSON>,
-    },
-    REPEAT {
-        content: Box<RuleJSON>,
-    },
-    REPEAT1 {
-        content: Box<RuleJSON>,
-    },
-    PREC_DYNAMIC {
-        value: i32,
-        content: Box<RuleJSON>,
-    },
-    PREC_LEFT {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    PREC_RIGHT {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    PREC {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    TOKEN {
-        content: Box<RuleJSON>,
-    },
-    IMMEDIATE_TOKEN {
-        content: Box<RuleJSON>,
-    },
-    RESERVED {
-        context_name: String,
-        content: Box<RuleJSON>,
-    },
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-enum PrecedenceValueJSON {
-    Integer(i32),
-    Name(String),
-}
-
-#[derive(Deserialize)]
-pub struct GrammarJSON {
-    pub name: String,
-    rules: Map<String, Value>,
-    #[serde(default)]
-    precedences: Vec<Vec<RuleJSON>>,
-    #[serde(default)]
-    conflicts: Vec<Vec<String>>,
-    #[serde(default)]
-    externals: Vec<RuleJSON>,
-    #[serde(default)]
-    extras: Vec<RuleJSON>,
-    #[serde(default)]
-    inline: Vec<String>,
-    #[serde(default)]
-    supertypes: Vec<String>,
-    #[serde(default)]
-    word: Option<String>,
-    #[serde(default)]
-    reserved: Map<String, Value>,
-}
-
-pub type ParseGrammarResult<T> = Result<T, ParseGrammarError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum ParseGrammarError {
-    #[error("{0}")]
-    Serialization(String),
-    #[error("Rules in the `extras` array must not contain empty strings")]
-    InvalidExtra,
-    #[error("Invalid rule in precedences array. Only strings and symbols are allowed")]
-    Unexpected,
-    #[error("Reserved word sets must be arrays")]
-    InvalidReservedWordSet,
-    #[error("Grammar Error: Unexpected rule `{0}` in `token()` call")]
-    UnexpectedRule(String),
-}
-
-impl From<serde_json::Error> for ParseGrammarError {
-    fn from(value: serde_json::Error) -> Self {
-        Self::Serialization(value.to_string())
-    }
-}
-
-/// Check if a rule is referenced by another rule.
-///
-/// This function is used to determine if a variable is used in a given rule,
-/// and `is_other` indicates if the rule is an external, and if it is,
-/// to not assume that a named symbol that is equal to itself means it's being referenced.
-///
-/// For example, if we have an external rule **and** a normal rule both called `foo`,
-/// `foo` should not be thought of as directly used unless it's used within another rule.
-fn rule_is_referenced(rule: &Rule, target: &str, is_external: bool) -> bool {
-    match rule {
-        Rule::NamedSymbol(name) => name == target && !is_external,
-        Rule::Choice(rules) | Rule::Seq(rules) => {
-            rules.iter().any(|r| rule_is_referenced(r, target, false))
-        }
-        Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => {
-            rule_is_referenced(rule, target, is_external)
-        }
-        Rule::Repeat(inner) => rule_is_referenced(inner, target, false),
-        Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false,
-    }
-}
-
-fn variable_is_used(
-    grammar_rules: &[(String, Rule)],
-    extras: &[Rule],
-    externals: &[Rule],
-    target_name: &str,
-    in_progress: &mut HashSet<String>,
-) -> bool {
-    let root = &grammar_rules.first().unwrap().0;
-    if target_name == root {
-        return true;
-    }
-
-    if extras
-        .iter()
-        .any(|rule| rule_is_referenced(rule, target_name, false))
-    {
-        return true;
-    }
-
-    if externals
-        .iter()
-        .any(|rule| rule_is_referenced(rule, target_name, true))
-    {
-        return true;
-    }
-
-    in_progress.insert(target_name.to_string());
-    let result = grammar_rules
-        .iter()
-        .filter(|(key, _)| *key != target_name)
-        .any(|(name, rule)| {
-            if !rule_is_referenced(rule, target_name, false) || in_progress.contains(name) {
-                return false;
-            }
-            variable_is_used(grammar_rules, extras, externals, name, in_progress)
-        });
-    in_progress.remove(target_name);
-
-    result
-}
-
-pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
-    let mut grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
-
-    let mut extra_symbols =
-        grammar_json
-            .extras
-            .into_iter()
-            .try_fold(Vec::<Rule>::new(), |mut acc, item| {
-                let rule = parse_rule(item, false)?;
-                if let Rule::String(ref value) = rule {
-                    if value.is_empty() {
-                        Err(ParseGrammarError::InvalidExtra)?;
-                    }
-                }
-                acc.push(rule);
-                ParseGrammarResult::Ok(acc)
-            })?;
-
-    let mut external_tokens = grammar_json
-        .externals
-        .into_iter()
-        .map(|e| parse_rule(e, false))
-        .collect::<ParseGrammarResult<Vec<_>>>()?;
-
-    let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len());
-    for list in grammar_json.precedences {
-        let mut ordering = Vec::with_capacity(list.len());
-        for entry in list {
-            ordering.push(match entry {
-                RuleJSON::STRING { value } => PrecedenceEntry::Name(value),
-                RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name),
-                _ => Err(ParseGrammarError::Unexpected)?,
-            });
-        }
-        precedence_orderings.push(ordering);
-    }
-
-    let mut variables = Vec::with_capacity(grammar_json.rules.len());
-
-    let rules = grammar_json
-        .rules
-        .into_iter()
-        .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?, false)?)))
-        .collect::<ParseGrammarResult<Vec<_>>>()?;
-
-    let mut in_progress = HashSet::new();
-
-    for (name, rule) in &rules {
-        if !variable_is_used(
-            &rules,
-            &extra_symbols,
-            &external_tokens,
-            name,
-            &mut in_progress,
-        ) && grammar_json.word.as_ref().is_none_or(|w| w != name)
-        {
-            grammar_json.conflicts.retain(|r| !r.contains(name));
-            grammar_json.supertypes.retain(|r| r != name);
-            grammar_json.inline.retain(|r| r != name);
-            extra_symbols.retain(|r| !rule_is_referenced(r, name, true));
-            external_tokens.retain(|r| !rule_is_referenced(r, name, true));
-            precedence_orderings.retain(|r| {
-                !r.iter().any(|e| {
-                    let PrecedenceEntry::Symbol(s) = e else {
-                        return false;
-                    };
-                    s == name
-                })
-            });
-            continue;
-        }
-        variables.push(Variable {
-            name: name.clone(),
-            kind: VariableType::Named,
-            rule: rule.clone(),
-        });
-    }
-
-    let reserved_words = grammar_json
-        .reserved
-        .into_iter()
-        .map(|(name, rule_values)| {
-            let mut reserved_words = Vec::new();
-
-            let Value::Array(rule_values) = rule_values else {
-                Err(ParseGrammarError::InvalidReservedWordSet)?
-            };
-
-            for value in rule_values {
-                reserved_words.push(parse_rule(serde_json::from_value(value)?, false)?);
-            }
-            Ok(ReservedWordContext {
-                name,
-                reserved_words,
-            })
-        })
-        .collect::<ParseGrammarResult<Vec<_>>>()?;
-
-    Ok(InputGrammar {
-        name: grammar_json.name,
-        word_token: grammar_json.word,
-        expected_conflicts: grammar_json.conflicts,
-        supertype_symbols: grammar_json.supertypes,
-        variables_to_inline: grammar_json.inline,
-        precedence_orderings,
-        variables,
-        extra_symbols,
-        external_tokens,
-        reserved_words,
-    })
-}
-
-fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult<Rule> {
-    match json {
-        RuleJSON::ALIAS {
-            content,
-            value,
-            named,
-        } => parse_rule(*content, is_token).map(|r| Rule::alias(r, value, named)),
-        RuleJSON::BLANK => Ok(Rule::Blank),
-        RuleJSON::STRING { value } => Ok(Rule::String(value)),
-        RuleJSON::PATTERN { value, flags } => Ok(Rule::Pattern(
-            value,
-            flags.map_or(String::new(), |f| {
-                f.matches(|c| {
-                    if c == 'i' {
-                        true
-                    } else {
-                        // silently ignore unicode flags
-                        if c != 'u' && c != 'v' {
-                            eprintln!("Warning: unsupported flag {c}");
-                        }
-                        false
-                    }
-                })
-                .collect()
-            }),
-        )),
-        RuleJSON::SYMBOL { name } => {
-            if is_token {
-                Err(ParseGrammarError::UnexpectedRule(name))?
-            } else {
-                Ok(Rule::NamedSymbol(name))
-            }
-        }
-        RuleJSON::CHOICE { members } => members
-            .into_iter()
-            .map(|m| parse_rule(m, is_token))
-            .collect::<ParseGrammarResult<Vec<_>>>()
-            .map(Rule::choice),
-        RuleJSON::FIELD { content, name } => {
-            parse_rule(*content, is_token).map(|r| Rule::field(name, r))
-        }
-        RuleJSON::SEQ { members } => members
-            .into_iter()
-            .map(|m| parse_rule(m, is_token))
-            .collect::<ParseGrammarResult<Vec<_>>>()
-            .map(Rule::seq),
-        RuleJSON::REPEAT1 { content } => parse_rule(*content, is_token).map(Rule::repeat),
-        RuleJSON::REPEAT { content } => {
-            parse_rule(*content, is_token).map(|m| Rule::choice(vec![Rule::repeat(m), Rule::Blank]))
-        }
-        RuleJSON::PREC { value, content } => {
-            parse_rule(*content, is_token).map(|r| Rule::prec(value.into(), r))
-        }
-        RuleJSON::PREC_LEFT { value, content } => {
-            parse_rule(*content, is_token).map(|r| Rule::prec_left(value.into(), r))
-        }
-        RuleJSON::PREC_RIGHT { value, content } => {
-            parse_rule(*content, is_token).map(|r| Rule::prec_right(value.into(), r))
-        }
-        RuleJSON::PREC_DYNAMIC { value, content } => {
-            parse_rule(*content, is_token).map(|r| Rule::prec_dynamic(value, r))
-        }
-        RuleJSON::RESERVED {
-            content,
-            context_name,
-        } => parse_rule(*content, is_token).map(|r| Rule::Reserved {
-            rule: Box::new(r),
-            context_name,
-        }),
-        RuleJSON::TOKEN { content } => parse_rule(*content, true).map(Rule::token),
-        RuleJSON::IMMEDIATE_TOKEN { content } => {
-            parse_rule(*content, is_token).map(Rule::immediate_token)
-        }
-    }
-}
-
-impl From<PrecedenceValueJSON> for Precedence {
-    fn from(val: PrecedenceValueJSON) -> Self {
-        match val {
-            PrecedenceValueJSON::Integer(i) => Self::Integer(i),
-            PrecedenceValueJSON::Name(i) => Self::Name(i),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_grammar() {
-        let grammar = parse_grammar(
-            r#"{
-            "name": "my_lang",
-            "rules": {
-                "file": {
-                    "type": "REPEAT1",
-                    "content": {
-                        "type": "SYMBOL",
-                        "name": "statement"
-                    }
-                },
-                "statement": {
-                    "type": "STRING",
-                    "value": "foo"
-                }
-            }
-        }"#,
-        )
-        .unwrap();
-
-        assert_eq!(grammar.name, "my_lang");
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable {
-                    name: "file".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::NamedSymbol("statement".to_string()))
-                },
-                Variable {
-                    name: "statement".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::String("foo".to_string())
-                },
-            ]
-        );
-    }
-}

cli/generate/src/prepare_grammar/expand_repeats.rs

@@ -1,289 +0,0 @@
-use std::{collections::HashMap, mem};
-
-use super::ExtractedSyntaxGrammar;
-use crate::{
-    grammars::{Variable, VariableType},
-    rules::{Rule, Symbol},
-};
-
-struct Expander {
-    variable_name: String,
-    repeat_count_in_variable: usize,
-    preceding_symbol_count: usize,
-    auxiliary_variables: Vec<Variable>,
-    existing_repeats: HashMap<Rule, Symbol>,
-}
-
-impl Expander {
-    fn expand_variable(&mut self, index: usize, variable: &mut Variable) -> bool {
-        self.variable_name.clear();
-        self.variable_name.push_str(&variable.name);
-        self.repeat_count_in_variable = 0;
-        let mut rule = Rule::Blank;
-        mem::swap(&mut rule, &mut variable.rule);
-
-        // In the special case of a hidden variable with a repetition at its top level,
-        // convert that rule itself into a binary tree structure instead of introducing
-        // another auxiliary rule.
-        if let (VariableType::Hidden, Rule::Repeat(repeated_content)) = (variable.kind, &rule) {
-            let inner_rule = self.expand_rule(repeated_content);
-            variable.rule = self.wrap_rule_in_binary_tree(Symbol::non_terminal(index), inner_rule);
-            variable.kind = VariableType::Auxiliary;
-            return true;
-        }
-
-        variable.rule = self.expand_rule(&rule);
-        false
-    }
-
-    fn expand_rule(&mut self, rule: &Rule) -> Rule {
-        match rule {
-            // For choices, sequences, and metadata, descend into the child rules,
-            // replacing any nested repetitions.
-            Rule::Choice(elements) => Rule::Choice(
-                elements
-                    .iter()
-                    .map(|element| self.expand_rule(element))
-                    .collect(),
-            ),
-
-            Rule::Seq(elements) => Rule::Seq(
-                elements
-                    .iter()
-                    .map(|element| self.expand_rule(element))
-                    .collect(),
-            ),
-
-            Rule::Metadata { rule, params } => Rule::Metadata {
-                rule: Box::new(self.expand_rule(rule)),
-                params: params.clone(),
-            },
-
-            // For repetitions, introduce an auxiliary rule that contains the
-            // repeated content, but can also contain a recursive binary tree structure.
-            Rule::Repeat(content) => {
-                let inner_rule = self.expand_rule(content);
-
-                if let Some(existing_symbol) = self.existing_repeats.get(&inner_rule) {
-                    return Rule::Symbol(*existing_symbol);
-                }
-
-                self.repeat_count_in_variable += 1;
-                let rule_name = format!(
-                    "{}_repeat{}",
-                    self.variable_name, self.repeat_count_in_variable
-                );
-                let repeat_symbol = Symbol::non_terminal(
-                    self.preceding_symbol_count + self.auxiliary_variables.len(),
-                );
-                self.existing_repeats
-                    .insert(inner_rule.clone(), repeat_symbol);
-                self.auxiliary_variables.push(Variable {
-                    name: rule_name,
-                    kind: VariableType::Auxiliary,
-                    rule: self.wrap_rule_in_binary_tree(repeat_symbol, inner_rule),
-                });
-
-                Rule::Symbol(repeat_symbol)
-            }
-
-            // For primitive rules, don't change anything.
-            _ => rule.clone(),
-        }
-    }
-
-    fn wrap_rule_in_binary_tree(&self, symbol: Symbol, rule: Rule) -> Rule {
-        Rule::choice(vec![
-            Rule::Seq(vec![Rule::Symbol(symbol), Rule::Symbol(symbol)]),
-            rule,
-        ])
-    }
-}
-
-pub(super) fn expand_repeats(mut grammar: ExtractedSyntaxGrammar) -> ExtractedSyntaxGrammar {
-    let mut expander = Expander {
-        variable_name: String::new(),
-        repeat_count_in_variable: 0,
-        preceding_symbol_count: grammar.variables.len(),
-        auxiliary_variables: Vec::new(),
-        existing_repeats: HashMap::new(),
-    };
-
-    for (i, variable) in grammar.variables.iter_mut().enumerate() {
-        let expanded_top_level_repetition = expander.expand_variable(i, variable);
-
-        // If a hidden variable had a top-level repetition and it was converted to
-        // a recursive rule, then it can't be inlined.
-        if expanded_top_level_repetition {
-            grammar
-                .variables_to_inline
-                .retain(|symbol| *symbol != Symbol::non_terminal(i));
-        }
-    }
-
-    grammar.variables.extend(expander.auxiliary_variables);
-    grammar
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_basic_repeat_expansion() {
-        // Repeats nested inside of sequences and choices are expanded.
-        let grammar = expand_repeats(build_grammar(vec![Variable::named(
-            "rule0",
-            Rule::seq(vec![
-                Rule::terminal(10),
-                Rule::choice(vec![
-                    Rule::repeat(Rule::terminal(11)),
-                    Rule::repeat(Rule::terminal(12)),
-                ]),
-                Rule::terminal(13),
-            ]),
-        )]));
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "rule0",
-                    Rule::seq(vec![
-                        Rule::terminal(10),
-                        Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),]),
-                        Rule::terminal(13),
-                    ])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
-                        Rule::terminal(11),
-                    ])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat2",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
-                        Rule::terminal(12),
-                    ])
-                ),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_repeat_deduplication() {
-        // Terminal 4 appears inside of a repeat in three different places.
-        let grammar = expand_repeats(build_grammar(vec![
-            Variable::named(
-                "rule0",
-                Rule::choice(vec![
-                    Rule::seq(vec![Rule::terminal(1), Rule::repeat(Rule::terminal(4))]),
-                    Rule::seq(vec![Rule::terminal(2), Rule::repeat(Rule::terminal(4))]),
-                ]),
-            ),
-            Variable::named(
-                "rule1",
-                Rule::seq(vec![Rule::terminal(3), Rule::repeat(Rule::terminal(4))]),
-            ),
-        ]));
-
-        // Only one auxiliary rule is created for repeating terminal 4.
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "rule0",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::terminal(1), Rule::non_terminal(2)]),
-                        Rule::seq(vec![Rule::terminal(2), Rule::non_terminal(2)]),
-                    ])
-                ),
-                Variable::named(
-                    "rule1",
-                    Rule::seq(vec![Rule::terminal(3), Rule::non_terminal(2),])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
-                        Rule::terminal(4),
-                    ])
-                )
-            ]
-        );
-    }
-
-    #[test]
-    fn test_expansion_of_nested_repeats() {
-        let grammar = expand_repeats(build_grammar(vec![Variable::named(
-            "rule0",
-            Rule::seq(vec![
-                Rule::terminal(10),
-                Rule::repeat(Rule::seq(vec![
-                    Rule::terminal(11),
-                    Rule::repeat(Rule::terminal(12)),
-                ])),
-            ]),
-        )]));
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "rule0",
-                    Rule::seq(vec![Rule::terminal(10), Rule::non_terminal(2),])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
-                        Rule::terminal(12),
-                    ])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat2",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
-                        Rule::seq(vec![Rule::terminal(11), Rule::non_terminal(1),]),
-                    ])
-                ),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_expansion_of_repeats_at_top_of_hidden_rules() {
-        let grammar = expand_repeats(build_grammar(vec![
-            Variable::named("rule0", Rule::non_terminal(1)),
-            Variable::hidden(
-                "_rule1",
-                Rule::repeat(Rule::choice(vec![Rule::terminal(11), Rule::terminal(12)])),
-            ),
-        ]));
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named("rule0", Rule::non_terminal(1),),
-                Variable::auxiliary(
-                    "_rule1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1)]),
-                        Rule::terminal(11),
-                        Rule::terminal(12),
-                    ]),
-                ),
-            ]
-        );
-    }
-
-    fn build_grammar(variables: Vec<Variable>) -> ExtractedSyntaxGrammar {
-        ExtractedSyntaxGrammar {
-            variables,
-            ..Default::default()
-        }
-    }
-}

cli/generate/src/prepare_grammar/expand_tokens.rs

@@ -1,826 +0,0 @@
-use anyhow::Result;
-use regex_syntax::{
-    hir::{Class, Hir, HirKind},
-    ParserBuilder,
-};
-use serde::Serialize;
-use thiserror::Error;
-
-use super::ExtractedLexicalGrammar;
-use crate::{
-    grammars::{LexicalGrammar, LexicalVariable},
-    nfa::{CharacterSet, Nfa, NfaState},
-    rules::{Precedence, Rule},
-};
-
-struct NfaBuilder {
-    nfa: Nfa,
-    is_sep: bool,
-    precedence_stack: Vec<i32>,
-}
-
-pub type ExpandTokensResult<T> = Result<T, ExpandTokensError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum ExpandTokensError {
-    #[error(
-        "The rule `{0}` matches the empty string.
-Tree-sitter does not support syntactic rules that match the empty string
-unless they are used only as the grammar's start rule.
-        "
-    )]
-    EmptyString(String),
-    #[error(transparent)]
-    Processing(ExpandTokensProcessingError),
-    #[error(transparent)]
-    ExpandRule(ExpandRuleError),
-}
-
-#[derive(Debug, Error, Serialize)]
-pub struct ExpandTokensProcessingError {
-    rule: String,
-    error: ExpandRuleError,
-}
-
-impl std::fmt::Display for ExpandTokensProcessingError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(
-            f,
-            "Error processing rule {}: Grammar error: Unexpected rule {:?}",
-            self.rule, self.error
-        )?;
-        Ok(())
-    }
-}
-
-fn get_implicit_precedence(rule: &Rule) -> i32 {
-    match rule {
-        Rule::String(_) => 2,
-        Rule::Metadata { rule, params } => {
-            if params.is_main_token {
-                get_implicit_precedence(rule) + 1
-            } else {
-                get_implicit_precedence(rule)
-            }
-        }
-        _ => 0,
-    }
-}
-
-const fn get_completion_precedence(rule: &Rule) -> i32 {
-    if let Rule::Metadata { params, .. } = rule {
-        if let Precedence::Integer(p) = params.precedence {
-            return p;
-        }
-    }
-    0
-}
-
-pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> ExpandTokensResult<LexicalGrammar> {
-    let mut builder = NfaBuilder {
-        nfa: Nfa::new(),
-        is_sep: true,
-        precedence_stack: vec![0],
-    };
-
-    let separator_rule = if grammar.separators.is_empty() {
-        Rule::Blank
-    } else {
-        grammar.separators.push(Rule::Blank);
-        Rule::repeat(Rule::choice(grammar.separators))
-    };
-
-    let mut variables = Vec::new();
-    for (i, variable) in grammar.variables.into_iter().enumerate() {
-        if variable.rule.is_empty() {
-            Err(ExpandTokensError::EmptyString(variable.name.clone()))?;
-        }
-
-        let is_immediate_token = match &variable.rule {
-            Rule::Metadata { params, .. } => params.is_main_token,
-            _ => false,
-        };
-
-        builder.is_sep = false;
-        builder.nfa.states.push(NfaState::Accept {
-            variable_index: i,
-            precedence: get_completion_precedence(&variable.rule),
-        });
-        let last_state_id = builder.nfa.last_state_id();
-        builder
-            .expand_rule(&variable.rule, last_state_id)
-            .map_err(|e| {
-                ExpandTokensError::Processing(ExpandTokensProcessingError {
-                    rule: variable.name.clone(),
-                    error: e,
-                })
-            })?;
-
-        if !is_immediate_token {
-            builder.is_sep = true;
-            let last_state_id = builder.nfa.last_state_id();
-            builder
-                .expand_rule(&separator_rule, last_state_id)
-                .map_err(ExpandTokensError::ExpandRule)?;
-        }
-
-        variables.push(LexicalVariable {
-            name: variable.name,
-            kind: variable.kind,
-            implicit_precedence: get_implicit_precedence(&variable.rule),
-            start_state: builder.nfa.last_state_id(),
-        });
-    }
-
-    Ok(LexicalGrammar {
-        nfa: builder.nfa,
-        variables,
-    })
-}
-
-pub type ExpandRuleResult<T> = Result<T, ExpandRuleError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum ExpandRuleError {
-    #[error("Grammar error: Unexpected rule {0:?}")]
-    UnexpectedRule(Rule),
-    #[error("{0}")]
-    Parse(String),
-    #[error(transparent)]
-    ExpandRegex(ExpandRegexError),
-}
-
-pub type ExpandRegexResult<T> = Result<T, ExpandRegexError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum ExpandRegexError {
-    #[error("{0}")]
-    Utf8(String),
-    #[error("Regex error: Assertions are not supported")]
-    Assertion,
-}
-
-impl NfaBuilder {
-    fn expand_rule(&mut self, rule: &Rule, mut next_state_id: u32) -> ExpandRuleResult<bool> {
-        match rule {
-            Rule::Pattern(s, f) => {
-                // With unicode enabled, `\w`, `\s` and `\d` expand to character sets that are much
-                // larger than intended, so we replace them with the actual
-                // character sets they should represent. If the full unicode range
-                // of `\w`, `\s` or `\d` are needed then `\p{L}`, `\p{Z}` and `\p{N}` should be
-                // used.
-                let s = s
-                    .replace(r"\w", r"[0-9A-Za-z_]")
-                    .replace(r"\s", r"[\t-\r ]")
-                    .replace(r"\d", r"[0-9]")
-                    .replace(r"\W", r"[^0-9A-Za-z_]")
-                    .replace(r"\S", r"[^\t-\r ]")
-                    .replace(r"\D", r"[^0-9]");
-                let mut parser = ParserBuilder::new()
-                    .case_insensitive(f.contains('i'))
-                    .unicode(true)
-                    .utf8(false)
-                    .build();
-                let hir = parser
-                    .parse(&s)
-                    .map_err(|e| ExpandRuleError::Parse(e.to_string()))?;
-                self.expand_regex(&hir, next_state_id)
-                    .map_err(ExpandRuleError::ExpandRegex)
-            }
-            Rule::String(s) => {
-                for c in s.chars().rev() {
-                    self.push_advance(CharacterSet::empty().add_char(c), next_state_id);
-                    next_state_id = self.nfa.last_state_id();
-                }
-                Ok(!s.is_empty())
-            }
-            Rule::Choice(elements) => {
-                let mut alternative_state_ids = Vec::new();
-                for element in elements {
-                    if self.expand_rule(element, next_state_id)? {
-                        alternative_state_ids.push(self.nfa.last_state_id());
-                    } else {
-                        alternative_state_ids.push(next_state_id);
-                    }
-                }
-                alternative_state_ids.sort_unstable();
-                alternative_state_ids.dedup();
-                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
-                for alternative_state_id in alternative_state_ids {
-                    self.push_split(alternative_state_id);
-                }
-                Ok(true)
-            }
-            Rule::Seq(elements) => {
-                let mut result = false;
-                for element in elements.iter().rev() {
-                    if self.expand_rule(element, next_state_id)? {
-                        result = true;
-                    }
-                    next_state_id = self.nfa.last_state_id();
-                }
-                Ok(result)
-            }
-            Rule::Repeat(rule) => {
-                self.nfa.states.push(NfaState::Accept {
-                    variable_index: 0,
-                    precedence: 0,
-                }); // Placeholder for split
-                let split_state_id = self.nfa.last_state_id();
-                if self.expand_rule(rule, split_state_id)? {
-                    self.nfa.states[split_state_id as usize] =
-                        NfaState::Split(self.nfa.last_state_id(), next_state_id);
-                    Ok(true)
-                } else {
-                    Ok(false)
-                }
-            }
-            Rule::Metadata { rule, params } => {
-                let has_precedence = if let Precedence::Integer(precedence) = &params.precedence {
-                    self.precedence_stack.push(*precedence);
-                    true
-                } else {
-                    false
-                };
-                let result = self.expand_rule(rule, next_state_id);
-                if has_precedence {
-                    self.precedence_stack.pop();
-                }
-                result
-            }
-            Rule::Blank => Ok(false),
-            _ => Err(ExpandRuleError::UnexpectedRule(rule.clone()))?,
-        }
-    }
-
-    fn expand_regex(&mut self, hir: &Hir, mut next_state_id: u32) -> ExpandRegexResult<bool> {
-        match hir.kind() {
-            HirKind::Empty => Ok(false),
-            HirKind::Literal(literal) => {
-                for character in std::str::from_utf8(&literal.0)
-                    .map_err(|e| ExpandRegexError::Utf8(e.to_string()))?
-                    .chars()
-                    .rev()
-                {
-                    let char_set = CharacterSet::from_char(character);
-                    self.push_advance(char_set, next_state_id);
-                    next_state_id = self.nfa.last_state_id();
-                }
-
-                Ok(true)
-            }
-            HirKind::Class(class) => match class {
-                Class::Unicode(class) => {
-                    let mut chars = CharacterSet::default();
-                    for c in class.ranges() {
-                        chars = chars.add_range(c.start(), c.end());
-                    }
-
-                    // For some reason, the long s `ſ` is included if the letter `s` is in a
-                    // pattern, so we remove it.
-                    if chars.range_count() == 3
-                        && chars
-                            .ranges()
-                            // exact check to ensure that `ſ` wasn't intentionally added.
-                            .all(|r| ['s'..='s', 'S'..='S', 'ſ'..='ſ'].contains(&r))
-                    {
-                        chars = chars.difference(CharacterSet::from_char('ſ'));
-                    }
-                    self.push_advance(chars, next_state_id);
-                    Ok(true)
-                }
-                Class::Bytes(bytes_class) => {
-                    let mut chars = CharacterSet::default();
-                    for c in bytes_class.ranges() {
-                        chars = chars.add_range(c.start().into(), c.end().into());
-                    }
-                    self.push_advance(chars, next_state_id);
-                    Ok(true)
-                }
-            },
-            HirKind::Look(_) => Err(ExpandRegexError::Assertion)?,
-            HirKind::Repetition(repetition) => match (repetition.min, repetition.max) {
-                (0, Some(1)) => self.expand_zero_or_one(&repetition.sub, next_state_id),
-                (1, None) => self.expand_one_or_more(&repetition.sub, next_state_id),
-                (0, None) => self.expand_zero_or_more(&repetition.sub, next_state_id),
-                (min, Some(max)) if min == max => {
-                    self.expand_count(&repetition.sub, min, next_state_id)
-                }
-                (min, None) => {
-                    if self.expand_zero_or_more(&repetition.sub, next_state_id)? {
-                        self.expand_count(&repetition.sub, min, next_state_id)
-                    } else {
-                        Ok(false)
-                    }
-                }
-                (min, Some(max)) => {
-                    let mut result = self.expand_count(&repetition.sub, min, next_state_id)?;
-                    for _ in min..max {
-                        if result {
-                            next_state_id = self.nfa.last_state_id();
-                        }
-                        if self.expand_zero_or_one(&repetition.sub, next_state_id)? {
-                            result = true;
-                        }
-                    }
-                    Ok(result)
-                }
-            },
-            HirKind::Capture(capture) => self.expand_regex(&capture.sub, next_state_id),
-            HirKind::Concat(concat) => {
-                let mut result = false;
-                for hir in concat.iter().rev() {
-                    if self.expand_regex(hir, next_state_id)? {
-                        result = true;
-                        next_state_id = self.nfa.last_state_id();
-                    }
-                }
-                Ok(result)
-            }
-            HirKind::Alternation(alternations) => {
-                let mut alternative_state_ids = Vec::new();
-                for hir in alternations {
-                    if self.expand_regex(hir, next_state_id)? {
-                        alternative_state_ids.push(self.nfa.last_state_id());
-                    } else {
-                        alternative_state_ids.push(next_state_id);
-                    }
-                }
-                alternative_state_ids.sort_unstable();
-                alternative_state_ids.dedup();
-                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
-                for alternative_state_id in alternative_state_ids {
-                    self.push_split(alternative_state_id);
-                }
-                Ok(true)
-            }
-        }
-    }
-
-    fn expand_one_or_more(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult<bool> {
-        self.nfa.states.push(NfaState::Accept {
-            variable_index: 0,
-            precedence: 0,
-        }); // Placeholder for split
-        let split_state_id = self.nfa.last_state_id();
-        if self.expand_regex(hir, split_state_id)? {
-            self.nfa.states[split_state_id as usize] =
-                NfaState::Split(self.nfa.last_state_id(), next_state_id);
-            Ok(true)
-        } else {
-            self.nfa.states.pop();
-            Ok(false)
-        }
-    }
-
-    fn expand_zero_or_one(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult<bool> {
-        if self.expand_regex(hir, next_state_id)? {
-            self.push_split(next_state_id);
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    fn expand_zero_or_more(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult<bool> {
-        if self.expand_one_or_more(hir, next_state_id)? {
-            self.push_split(next_state_id);
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    fn expand_count(
-        &mut self,
-        hir: &Hir,
-        count: u32,
-        mut next_state_id: u32,
-    ) -> ExpandRegexResult<bool> {
-        let mut result = false;
-        for _ in 0..count {
-            if self.expand_regex(hir, next_state_id)? {
-                result = true;
-                next_state_id = self.nfa.last_state_id();
-            }
-        }
-        Ok(result)
-    }
-
-    fn push_advance(&mut self, chars: CharacterSet, state_id: u32) {
-        let precedence = *self.precedence_stack.last().unwrap();
-        self.nfa.states.push(NfaState::Advance {
-            chars,
-            state_id,
-            precedence,
-            is_sep: self.is_sep,
-        });
-    }
-
-    fn push_split(&mut self, state_id: u32) {
-        let last_state_id = self.nfa.last_state_id();
-        self.nfa
-            .states
-            .push(NfaState::Split(state_id, last_state_id));
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::Variable,
-        nfa::{NfaCursor, NfaTransition},
-    };
-
-    fn simulate_nfa<'a>(grammar: &'a LexicalGrammar, s: &'a str) -> Option<(usize, &'a str)> {
-        let start_states = grammar.variables.iter().map(|v| v.start_state).collect();
-        let mut cursor = NfaCursor::new(&grammar.nfa, start_states);
-
-        let mut result = None;
-        let mut result_precedence = i32::MIN;
-        let mut start_char = 0;
-        let mut end_char = 0;
-        for c in s.chars() {
-            for (id, precedence) in cursor.completions() {
-                if result.is_none() || result_precedence <= precedence {
-                    result = Some((id, &s[start_char..end_char]));
-                    result_precedence = precedence;
-                }
-            }
-            if let Some(NfaTransition {
-                states,
-                is_separator,
-                ..
-            }) = cursor
-                .transitions()
-                .into_iter()
-                .find(|t| t.characters.contains(c) && t.precedence >= result_precedence)
-            {
-                cursor.reset(states);
-                end_char += c.len_utf8();
-                if is_separator {
-                    start_char = end_char;
-                }
-            } else {
-                break;
-            }
-        }
-
-        for (id, precedence) in cursor.completions() {
-            if result.is_none() || result_precedence <= precedence {
-                result = Some((id, &s[start_char..end_char]));
-                result_precedence = precedence;
-            }
-        }
-
-        result
-    }
-
-    #[test]
-    fn test_rule_expansion() {
-        struct Row {
-            rules: Vec<Rule>,
-            separators: Vec<Rule>,
-            examples: Vec<(&'static str, Option<(usize, &'static str)>)>,
-        }
-
-        let table = [
-            // regex with sequences and alternatives
-            Row {
-                rules: vec![Rule::pattern("(a|b|c)d(e|f|g)h?", "")],
-                separators: vec![],
-                examples: vec![
-                    ("ade1", Some((0, "ade"))),
-                    ("bdf1", Some((0, "bdf"))),
-                    ("bdfh1", Some((0, "bdfh"))),
-                    ("ad1", None),
-                ],
-            },
-            // regex with repeats
-            Row {
-                rules: vec![Rule::pattern("a*", "")],
-                separators: vec![],
-                examples: vec![("aaa1", Some((0, "aaa"))), ("b", Some((0, "")))],
-            },
-            // regex with repeats in sequences
-            Row {
-                rules: vec![Rule::pattern("a((bc)+|(de)*)f", "")],
-                separators: vec![],
-                examples: vec![
-                    ("af1", Some((0, "af"))),
-                    ("adedef1", Some((0, "adedef"))),
-                    ("abcbcbcf1", Some((0, "abcbcbcf"))),
-                    ("a", None),
-                ],
-            },
-            // regex with character ranges
-            Row {
-                rules: vec![Rule::pattern("[a-fA-F0-9]+", "")],
-                separators: vec![],
-                examples: vec![("A1ff0.", Some((0, "A1ff0")))],
-            },
-            // regex with perl character classes
-            Row {
-                rules: vec![Rule::pattern("\\w\\d\\s", "")],
-                separators: vec![],
-                examples: vec![("_0  ", Some((0, "_0 ")))],
-            },
-            // string
-            Row {
-                rules: vec![Rule::string("abc")],
-                separators: vec![],
-                examples: vec![("abcd", Some((0, "abc"))), ("ab", None)],
-            },
-            // complex rule containing strings and regexes
-            Row {
-                rules: vec![Rule::repeat(Rule::seq(vec![
-                    Rule::string("{"),
-                    Rule::pattern("[a-f]+", ""),
-                    Rule::string("}"),
-                ]))],
-                separators: vec![],
-                examples: vec![
-                    ("{a}{", Some((0, "{a}"))),
-                    ("{a}{d", Some((0, "{a}"))),
-                    ("ab", None),
-                ],
-            },
-            // longest match rule
-            Row {
-                rules: vec![
-                    Rule::pattern("a|bc", ""),
-                    Rule::pattern("aa", ""),
-                    Rule::pattern("bcd", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("a.", Some((0, "a"))),
-                    ("bc.", Some((0, "bc"))),
-                    ("aa.", Some((1, "aa"))),
-                    ("bcd?", Some((2, "bcd"))),
-                    ("b.", None),
-                    ("c.", None),
-                ],
-            },
-            // regex with an alternative including the empty string
-            Row {
-                rules: vec![Rule::pattern("a(b|)+c", "")],
-                separators: vec![],
-                examples: vec![
-                    ("ac.", Some((0, "ac"))),
-                    ("abc.", Some((0, "abc"))),
-                    ("abbc.", Some((0, "abbc"))),
-                ],
-            },
-            // separators
-            Row {
-                rules: vec![Rule::pattern("[a-f]+", "")],
-                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
-                examples: vec![
-                    ("  a", Some((0, "a"))),
-                    ("  \nb", Some((0, "b"))),
-                    ("  \\a", None),
-                    ("  \\\na", Some((0, "a"))),
-                ],
-            },
-            // shorter tokens with higher precedence
-            Row {
-                rules: vec![
-                    Rule::prec(Precedence::Integer(2), Rule::pattern("abc", "")),
-                    Rule::prec(Precedence::Integer(1), Rule::pattern("ab[cd]e", "")),
-                    Rule::pattern("[a-e]+", ""),
-                ],
-                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
-                examples: vec![
-                    ("abceef", Some((0, "abc"))),
-                    ("abdeef", Some((1, "abde"))),
-                    ("aeeeef", Some((2, "aeeee"))),
-                ],
-            },
-            // immediate tokens with higher precedence
-            Row {
-                rules: vec![
-                    Rule::prec(Precedence::Integer(1), Rule::pattern("[^a]+", "")),
-                    Rule::immediate_token(Rule::prec(
-                        Precedence::Integer(2),
-                        Rule::pattern("[^ab]+", ""),
-                    )),
-                ],
-                separators: vec![Rule::pattern("\\s", "")],
-                examples: vec![("cccb", Some((1, "ccc")))],
-            },
-            Row {
-                rules: vec![Rule::seq(vec![
-                    Rule::string("a"),
-                    Rule::choice(vec![Rule::string("b"), Rule::string("c")]),
-                    Rule::string("d"),
-                ])],
-                separators: vec![],
-                examples: vec![
-                    ("abd", Some((0, "abd"))),
-                    ("acd", Some((0, "acd"))),
-                    ("abc", None),
-                    ("ad", None),
-                    ("d", None),
-                    ("a", None),
-                ],
-            },
-            // nested choices within sequences
-            Row {
-                rules: vec![Rule::seq(vec![
-                    Rule::pattern("[0-9]+", ""),
-                    Rule::choice(vec![
-                        Rule::Blank,
-                        Rule::choice(vec![Rule::seq(vec![
-                            Rule::choice(vec![Rule::string("e"), Rule::string("E")]),
-                            Rule::choice(vec![
-                                Rule::Blank,
-                                Rule::choice(vec![Rule::string("+"), Rule::string("-")]),
-                            ]),
-                            Rule::pattern("[0-9]+", ""),
-                        ])]),
-                    ]),
-                ])],
-                separators: vec![],
-                examples: vec![
-                    ("12", Some((0, "12"))),
-                    ("12e", Some((0, "12"))),
-                    ("12g", Some((0, "12"))),
-                    ("12e3", Some((0, "12e3"))),
-                    ("12e+", Some((0, "12"))),
-                    ("12E+34 +", Some((0, "12E+34"))),
-                    ("12e34", Some((0, "12e34"))),
-                ],
-            },
-            // nested groups
-            Row {
-                rules: vec![Rule::seq(vec![Rule::pattern(r"([^x\\]|\\(.|\n))+", "")])],
-                separators: vec![],
-                examples: vec![("abcx", Some((0, "abc"))), ("abc\\0x", Some((0, "abc\\0")))],
-            },
-            // allowing unrecognized escape sequences
-            Row {
-                rules: vec![
-                    // Escaped forward slash (used in JS because '/' is the regex delimiter)
-                    Rule::pattern(r"\/", ""),
-                    // Escaped quotes
-                    Rule::pattern(r#"\"\'"#, ""),
-                    // Quote preceded by a literal backslash
-                    Rule::pattern(r"[\\']+", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("/", Some((0, "/"))),
-                    ("\"\'", Some((1, "\"\'"))),
-                    (r"'\'a", Some((2, r"'\'"))),
-                ],
-            },
-            // unicode property escapes
-            Row {
-                rules: vec![
-                    Rule::pattern(r"\p{L}+\P{L}+", ""),
-                    Rule::pattern(r"\p{White_Space}+\P{White_Space}+[\p{White_Space}]*", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("  123   abc", Some((1, "  123   "))),
-                    ("ბΨƁ___ƀƔ", Some((0, "ბΨƁ___"))),
-                ],
-            },
-            // unicode property escapes in bracketed sets
-            Row {
-                rules: vec![Rule::pattern(r"[\p{L}\p{Nd}]+", "")],
-                separators: vec![],
-                examples: vec![("abΨ12٣٣, ok", Some((0, "abΨ12٣٣")))],
-            },
-            // unicode character escapes
-            Row {
-                rules: vec![
-                    Rule::pattern(r"\u{00dc}", ""),
-                    Rule::pattern(r"\U{000000dd}", ""),
-                    Rule::pattern(r"\u00de", ""),
-                    Rule::pattern(r"\U000000df", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("\u{00dc}", Some((0, "\u{00dc}"))),
-                    ("\u{00dd}", Some((1, "\u{00dd}"))),
-                    ("\u{00de}", Some((2, "\u{00de}"))),
-                    ("\u{00df}", Some((3, "\u{00df}"))),
-                ],
-            },
-            Row {
-                rules: vec![
-                    Rule::pattern(r"u\{[0-9a-fA-F]+\}", ""),
-                    // Already-escaped curly braces
-                    Rule::pattern(r"\{[ab]{3}\}", ""),
-                    // Unicode codepoints
-                    Rule::pattern(r"\u{1000A}", ""),
-                    // Unicode codepoints (lowercase)
-                    Rule::pattern(r"\u{1000b}", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("u{1234} ok", Some((0, "u{1234}"))),
-                    ("{aba}}", Some((1, "{aba}"))),
-                    ("\u{1000A}", Some((2, "\u{1000A}"))),
-                    ("\u{1000b}", Some((3, "\u{1000b}"))),
-                ],
-            },
-            // Emojis
-            Row {
-                rules: vec![Rule::pattern(r"\p{Emoji}+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("🐎", Some((0, "🐎"))),
-                    ("🐴🐴", Some((0, "🐴🐴"))),
-                    ("#0", Some((0, "#0"))), // These chars are technically emojis!
-                    ("⻢", None),
-                    ("♞", None),
-                    ("horse", None),
-                ],
-            },
-            // Intersection
-            Row {
-                rules: vec![Rule::pattern(r"[[0-7]&&[4-9]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("456", Some((0, "456"))),
-                    ("64", Some((0, "64"))),
-                    ("452", Some((0, "45"))),
-                    ("91", None),
-                    ("8", None),
-                    ("3", None),
-                ],
-            },
-            // Difference
-            Row {
-                rules: vec![Rule::pattern(r"[[0-9]--[4-7]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("123", Some((0, "123"))),
-                    ("83", Some((0, "83"))),
-                    ("9", Some((0, "9"))),
-                    ("124", Some((0, "12"))),
-                    ("67", None),
-                    ("4", None),
-                ],
-            },
-            // Symmetric difference
-            Row {
-                rules: vec![Rule::pattern(r"[[0-7]~~[4-9]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("123", Some((0, "123"))),
-                    ("83", Some((0, "83"))),
-                    ("9", Some((0, "9"))),
-                    ("124", Some((0, "12"))),
-                    ("67", None),
-                    ("4", None),
-                ],
-            },
-            // Nested set operations
-            Row {
-                //               0 1 2 3 4 5 6 7 8 9
-                // [0-5]:        y y y y y y
-                // [2-4]:            y y y
-                // [0-5]--[2-4]: y y       y
-                // [3-9]:              y y y y y y y
-                // [6-7]:                    y y
-                // [3-9]--[5-7]:       y y y     y y
-                // final regex:  y y   y y       y y
-                rules: vec![Rule::pattern(r"[[[0-5]--[2-4]]~~[[3-9]--[6-7]]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("01", Some((0, "01"))),
-                    ("432", Some((0, "43"))),
-                    ("8", Some((0, "8"))),
-                    ("9", Some((0, "9"))),
-                    ("2", None),
-                    ("567", None),
-                ],
-            },
-        ];
-
-        for Row {
-            rules,
-            separators,
-            examples,
-        } in &table
-        {
-            let grammar = expand_tokens(ExtractedLexicalGrammar {
-                separators: separators.clone(),
-                variables: rules
-                    .iter()
-                    .map(|rule| Variable::named("", rule.clone()))
-                    .collect(),
-            })
-            .unwrap();
-
-            for (haystack, needle) in examples {
-                assert_eq!(simulate_nfa(&grammar, haystack), *needle);
-            }
-        }
-    }
-}

cli/generate/src/prepare_grammar/extract_default_aliases.rs

@@ -1,304 +0,0 @@
-use crate::{
-    grammars::{LexicalGrammar, SyntaxGrammar},
-    rules::{Alias, AliasMap, Symbol, SymbolType},
-};
-
-#[derive(Clone, Default)]
-struct SymbolStatus {
-    aliases: Vec<(Alias, usize)>,
-    appears_unaliased: bool,
-}
-
-// Update the grammar by finding symbols that always are aliased, and for each such symbol,
-// promoting one of its aliases to a "default alias", which is applied globally instead
-// of in a context-specific way.
-//
-// This has two benefits:
-// * It reduces the overhead of storing production-specific alias info in the parse table.
-// * Within an `ERROR` node, no context-specific aliases will be applied. This transformation
-//   ensures that the children of an `ERROR` node have symbols that are consistent with the way that
-//   they would appear in a valid syntax tree.
-pub(super) fn extract_default_aliases(
-    syntax_grammar: &mut SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-) -> AliasMap {
-    let mut terminal_status_list = vec![SymbolStatus::default(); lexical_grammar.variables.len()];
-    let mut non_terminal_status_list =
-        vec![SymbolStatus::default(); syntax_grammar.variables.len()];
-    let mut external_status_list =
-        vec![SymbolStatus::default(); syntax_grammar.external_tokens.len()];
-
-    // For each grammar symbol, find all of the aliases under which the symbol appears,
-    // and determine whether or not the symbol ever appears *unaliased*.
-    for variable in &syntax_grammar.variables {
-        for production in &variable.productions {
-            for step in &production.steps {
-                let status = match step.symbol.kind {
-                    SymbolType::External => &mut external_status_list[step.symbol.index],
-                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
-                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
-                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                        panic!("Unexpected end token")
-                    }
-                };
-
-                // Default aliases don't work for inlined variables.
-                if syntax_grammar.variables_to_inline.contains(&step.symbol) {
-                    continue;
-                }
-
-                if let Some(alias) = &step.alias {
-                    if let Some(count_for_alias) = status
-                        .aliases
-                        .iter_mut()
-                        .find_map(|(a, count)| if a == alias { Some(count) } else { None })
-                    {
-                        *count_for_alias += 1;
-                    } else {
-                        status.aliases.push((alias.clone(), 1));
-                    }
-                } else {
-                    status.appears_unaliased = true;
-                }
-            }
-        }
-    }
-
-    for symbol in &syntax_grammar.extra_symbols {
-        let status = match symbol.kind {
-            SymbolType::External => &mut external_status_list[symbol.index],
-            SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
-            SymbolType::Terminal => &mut terminal_status_list[symbol.index],
-            SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                panic!("Unexpected end token")
-            }
-        };
-        status.appears_unaliased = true;
-    }
-
-    let symbols_with_statuses = (terminal_status_list
-        .iter_mut()
-        .enumerate()
-        .map(|(i, status)| (Symbol::terminal(i), status)))
-    .chain(
-        non_terminal_status_list
-            .iter_mut()
-            .enumerate()
-            .map(|(i, status)| (Symbol::non_terminal(i), status)),
-    )
-    .chain(
-        external_status_list
-            .iter_mut()
-            .enumerate()
-            .map(|(i, status)| (Symbol::external(i), status)),
-    );
-
-    // For each symbol that always appears aliased, find the alias the occurs most often,
-    // and designate that alias as the symbol's "default alias". Store all of these
-    // default aliases in a map that will be returned.
-    let mut result = AliasMap::new();
-    for (symbol, status) in symbols_with_statuses {
-        if status.appears_unaliased {
-            status.aliases.clear();
-        } else if let Some(default_entry) = status
-            .aliases
-            .iter()
-            .enumerate()
-            .max_by_key(|(i, (_, count))| (count, -(*i as i64)))
-            .map(|(_, entry)| entry.clone())
-        {
-            status.aliases.clear();
-            status.aliases.push(default_entry.clone());
-            result.insert(symbol, default_entry.0);
-        }
-    }
-
-    // Wherever a symbol is aliased as its default alias, remove the usage of the alias,
-    // because it will now be redundant.
-    let mut alias_positions_to_clear = Vec::new();
-    for variable in &mut syntax_grammar.variables {
-        alias_positions_to_clear.clear();
-
-        for (i, production) in variable.productions.iter().enumerate() {
-            for (j, step) in production.steps.iter().enumerate() {
-                let status = match step.symbol.kind {
-                    SymbolType::External => &mut external_status_list[step.symbol.index],
-                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
-                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
-                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                        panic!("Unexpected end token")
-                    }
-                };
-
-                // If this step is aliased as the symbol's default alias, then remove that alias.
-                if step.alias.is_some()
-                    && step.alias.as_ref() == status.aliases.first().map(|t| &t.0)
-                {
-                    let mut other_productions_must_use_this_alias_at_this_index = false;
-                    for (other_i, other_production) in variable.productions.iter().enumerate() {
-                        if other_i != i
-                            && other_production.steps.len() > j
-                            && other_production.steps[j].alias == step.alias
-                            && result.get(&other_production.steps[j].symbol) != step.alias.as_ref()
-                        {
-                            other_productions_must_use_this_alias_at_this_index = true;
-                            break;
-                        }
-                    }
-
-                    if !other_productions_must_use_this_alias_at_this_index {
-                        alias_positions_to_clear.push((i, j));
-                    }
-                }
-            }
-        }
-
-        for (production_index, step_index) in &alias_positions_to_clear {
-            variable.productions[*production_index].steps[*step_index].alias = None;
-        }
-    }
-
-    result
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType},
-        nfa::Nfa,
-    };
-
-    #[test]
-    fn test_extract_simple_aliases() {
-        let mut syntax_grammar = SyntaxGrammar {
-            variables: vec![
-                SyntaxVariable {
-                    name: "v1".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
-                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
-                            ProductionStep::new(Symbol::terminal(2)).with_alias("a3", true),
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "v2".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            // Token 0 is always aliased as "a1".
-                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
-                            // Token 1 is aliased within rule `v1` above, but not here.
-                            ProductionStep::new(Symbol::terminal(1)),
-                            // Token 2 is aliased differently here than in `v1`. The alias from
-                            // `v1` should be promoted to the default alias, because `v1` appears
-                            // first in the grammar.
-                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
-                            // Token 3 is also aliased differently here than in `v1`. In this case,
-                            // this alias should be promoted to the default alias, because it is
-                            // used a greater number of times (twice).
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
-                        ],
-                    }],
-                },
-            ],
-            ..Default::default()
-        };
-
-        let lexical_grammar = LexicalGrammar {
-            nfa: Nfa::new(),
-            variables: vec![
-                LexicalVariable {
-                    name: "t0".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-                LexicalVariable {
-                    name: "t1".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-                LexicalVariable {
-                    name: "t2".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-                LexicalVariable {
-                    name: "t3".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-            ],
-        };
-
-        let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
-        assert_eq!(default_aliases.len(), 3);
-
-        assert_eq!(
-            default_aliases.get(&Symbol::terminal(0)),
-            Some(&Alias {
-                value: "a1".to_string(),
-                is_named: true,
-            })
-        );
-        assert_eq!(
-            default_aliases.get(&Symbol::terminal(2)),
-            Some(&Alias {
-                value: "a3".to_string(),
-                is_named: true,
-            })
-        );
-        assert_eq!(
-            default_aliases.get(&Symbol::terminal(3)),
-            Some(&Alias {
-                value: "a6".to_string(),
-                is_named: true,
-            })
-        );
-        assert_eq!(default_aliases.get(&Symbol::terminal(1)), None);
-
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![
-                SyntaxVariable {
-                    name: "v1".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(0)),
-                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
-                            ProductionStep::new(Symbol::terminal(2)),
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
-                        ],
-                    },],
-                },
-                SyntaxVariable {
-                    name: "v2".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(0)),
-                            ProductionStep::new(Symbol::terminal(1)),
-                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
-                            ProductionStep::new(Symbol::terminal(3)),
-                            ProductionStep::new(Symbol::terminal(3)),
-                        ],
-                    },],
-                },
-            ]
-        );
-    }
-}

cli/generate/src/prepare_grammar/extract_tokens.rs

@@ -1,609 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::Result;
-use serde::Serialize;
-use thiserror::Error;
-
-use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar};
-use crate::{
-    grammars::{ExternalToken, ReservedWordContext, Variable, VariableType},
-    rules::{MetadataParams, Rule, Symbol, SymbolType},
-};
-
-pub type ExtractTokensResult<T> = Result<T, ExtractTokensError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum ExtractTokensError {
-    #[error(
-        "The rule `{0}` contains an empty string.
-
-Tree-sitter does not support syntactic rules that contain an empty string
-unless they are used only as the grammar's start rule.
-"
-    )]
-    EmptyString(String),
-    #[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
-    ExternalTokenNonTerminal(String),
-    #[error("Non-symbol rules cannot be used as external tokens")]
-    NonSymbolExternalToken,
-    #[error("Non-terminal symbol '{0}' cannot be used as the word token, because its rule is duplicated in '{1}'")]
-    NonTerminalWordToken(String, String),
-    #[error("Reserved words must be tokens")]
-    NonTokenReservedWord,
-}
-
-pub(super) fn extract_tokens(
-    mut grammar: InternedGrammar,
-) -> ExtractTokensResult<(ExtractedSyntaxGrammar, ExtractedLexicalGrammar)> {
-    let mut extractor = TokenExtractor {
-        current_variable_name: String::new(),
-        current_variable_token_count: 0,
-        is_first_rule: false,
-        extracted_variables: Vec::new(),
-        extracted_usage_counts: Vec::new(),
-    };
-
-    for (i, variable) in &mut grammar.variables.iter_mut().enumerate() {
-        extractor.extract_tokens_in_variable(i == 0, variable)?;
-    }
-
-    for variable in &mut grammar.external_tokens {
-        extractor.extract_tokens_in_variable(false, variable)?;
-    }
-
-    let mut lexical_variables = Vec::with_capacity(extractor.extracted_variables.len());
-    for variable in extractor.extracted_variables {
-        lexical_variables.push(variable);
-    }
-
-    // If a variable's entire rule was extracted as a token and that token didn't
-    // appear within any other rule, then remove that variable from the syntax
-    // grammar, giving its name to the token in the lexical grammar. Any symbols
-    // that pointed to that variable will need to be updated to point to the
-    // variable in the lexical grammar. Symbols that pointed to later variables
-    // will need to have their indices decremented.
-    let mut variables = Vec::new();
-    let mut symbol_replacer = SymbolReplacer {
-        replacements: HashMap::new(),
-    };
-    for (i, variable) in grammar.variables.into_iter().enumerate() {
-        if let Rule::Symbol(Symbol {
-            kind: SymbolType::Terminal,
-            index,
-        }) = variable.rule
-        {
-            if i > 0 && extractor.extracted_usage_counts[index] == 1 {
-                let lexical_variable = &mut lexical_variables[index];
-                if lexical_variable.kind == VariableType::Auxiliary
-                    || variable.kind != VariableType::Hidden
-                {
-                    lexical_variable.kind = variable.kind;
-                    lexical_variable.name = variable.name;
-                    symbol_replacer.replacements.insert(i, index);
-                    continue;
-                }
-            }
-        }
-        variables.push(variable);
-    }
-
-    for variable in &mut variables {
-        variable.rule = symbol_replacer.replace_symbols_in_rule(&variable.rule);
-    }
-
-    let expected_conflicts = grammar
-        .expected_conflicts
-        .into_iter()
-        .map(|conflict| {
-            let mut result = conflict
-                .iter()
-                .map(|symbol| symbol_replacer.replace_symbol(*symbol))
-                .collect::<Vec<_>>();
-            result.sort_unstable();
-            result.dedup();
-            result
-        })
-        .collect();
-
-    let supertype_symbols = grammar
-        .supertype_symbols
-        .into_iter()
-        .map(|symbol| symbol_replacer.replace_symbol(symbol))
-        .collect();
-
-    let variables_to_inline = grammar
-        .variables_to_inline
-        .into_iter()
-        .map(|symbol| symbol_replacer.replace_symbol(symbol))
-        .collect();
-
-    let mut separators = Vec::new();
-    let mut extra_symbols = Vec::new();
-    for rule in grammar.extra_symbols {
-        if let Rule::Symbol(symbol) = rule {
-            extra_symbols.push(symbol_replacer.replace_symbol(symbol));
-        } else if let Some(index) = lexical_variables.iter().position(|v| v.rule == rule) {
-            extra_symbols.push(Symbol::terminal(index));
-        } else {
-            separators.push(rule);
-        }
-    }
-
-    let mut external_tokens = Vec::new();
-    for external_token in grammar.external_tokens {
-        let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule);
-        if let Rule::Symbol(symbol) = rule {
-            if symbol.is_non_terminal() {
-                Err(ExtractTokensError::ExternalTokenNonTerminal(
-                    variables[symbol.index].name.clone(),
-                ))?;
-            }
-
-            if symbol.is_external() {
-                external_tokens.push(ExternalToken {
-                    name: external_token.name,
-                    kind: external_token.kind,
-                    corresponding_internal_token: None,
-                });
-            } else {
-                external_tokens.push(ExternalToken {
-                    name: lexical_variables[symbol.index].name.clone(),
-                    kind: external_token.kind,
-                    corresponding_internal_token: Some(symbol),
-                });
-            }
-        } else {
-            Err(ExtractTokensError::NonSymbolExternalToken)?;
-        }
-    }
-
-    let mut word_token = None;
-    if let Some(token) = grammar.word_token {
-        let token = symbol_replacer.replace_symbol(token);
-        if token.is_non_terminal() {
-            let word_token_variable = &variables[token.index];
-            let conflicting_variable = variables
-                .iter()
-                .enumerate()
-                .find(|(i, v)| *i != token.index && v.rule == word_token_variable.rule)
-                .expect("Failed to find a variable with the same rule as the word token");
-
-            Err(ExtractTokensError::NonTerminalWordToken(
-                word_token_variable.name.clone(),
-                conflicting_variable.1.name.clone(),
-            ))?;
-        }
-        word_token = Some(token);
-    }
-
-    let mut reserved_word_contexts = Vec::new();
-    for reserved_word_context in grammar.reserved_word_sets {
-        let mut reserved_words = Vec::new();
-        for reserved_rule in reserved_word_context.reserved_words {
-            if let Rule::Symbol(symbol) = reserved_rule {
-                reserved_words.push(symbol_replacer.replace_symbol(symbol));
-            } else if let Some(index) = lexical_variables
-                .iter()
-                .position(|v| v.rule == reserved_rule)
-            {
-                reserved_words.push(Symbol::terminal(index));
-            } else {
-                Err(ExtractTokensError::NonTokenReservedWord)?;
-            }
-        }
-        reserved_word_contexts.push(ReservedWordContext {
-            name: reserved_word_context.name,
-            reserved_words,
-        });
-    }
-
-    Ok((
-        ExtractedSyntaxGrammar {
-            variables,
-            expected_conflicts,
-            extra_symbols,
-            variables_to_inline,
-            supertype_symbols,
-            external_tokens,
-            word_token,
-            precedence_orderings: grammar.precedence_orderings,
-            reserved_word_sets: reserved_word_contexts,
-        },
-        ExtractedLexicalGrammar {
-            variables: lexical_variables,
-            separators,
-        },
-    ))
-}
-
-struct TokenExtractor {
-    current_variable_name: String,
-    current_variable_token_count: usize,
-    is_first_rule: bool,
-    extracted_variables: Vec<Variable>,
-    extracted_usage_counts: Vec<usize>,
-}
-
-struct SymbolReplacer {
-    replacements: HashMap<usize, usize>,
-}
-
-impl TokenExtractor {
-    fn extract_tokens_in_variable(
-        &mut self,
-        is_first: bool,
-        variable: &mut Variable,
-    ) -> ExtractTokensResult<()> {
-        self.current_variable_name.clear();
-        self.current_variable_name.push_str(&variable.name);
-        self.current_variable_token_count = 0;
-        self.is_first_rule = is_first;
-        variable.rule = self.extract_tokens_in_rule(&variable.rule)?;
-        Ok(())
-    }
-
-    fn extract_tokens_in_rule(&mut self, input: &Rule) -> ExtractTokensResult<Rule> {
-        match input {
-            Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()),
-            Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()),
-            Rule::Metadata { params, rule } => {
-                if params.is_token {
-                    let mut params = params.clone();
-                    params.is_token = false;
-
-                    let mut string_value = None;
-                    if let Rule::String(value) = rule.as_ref() {
-                        string_value = Some(value);
-                    }
-
-                    let rule_to_extract = if params == MetadataParams::default() {
-                        rule.as_ref()
-                    } else {
-                        input
-                    };
-
-                    Ok(self.extract_token(rule_to_extract, string_value)?.into())
-                } else {
-                    Ok(Rule::Metadata {
-                        params: params.clone(),
-                        rule: Box::new(self.extract_tokens_in_rule(rule)?),
-                    })
-                }
-            }
-            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(
-                self.extract_tokens_in_rule(content)?,
-            ))),
-            Rule::Seq(elements) => Ok(Rule::Seq(
-                elements
-                    .iter()
-                    .map(|e| self.extract_tokens_in_rule(e))
-                    .collect::<ExtractTokensResult<Vec<_>>>()?,
-            )),
-            Rule::Choice(elements) => Ok(Rule::Choice(
-                elements
-                    .iter()
-                    .map(|e| self.extract_tokens_in_rule(e))
-                    .collect::<ExtractTokensResult<Vec<_>>>()?,
-            )),
-            Rule::Reserved { rule, context_name } => Ok(Rule::Reserved {
-                rule: Box::new(self.extract_tokens_in_rule(rule)?),
-                context_name: context_name.clone(),
-            }),
-            _ => Ok(input.clone()),
-        }
-    }
-
-    fn extract_token(
-        &mut self,
-        rule: &Rule,
-        string_value: Option<&String>,
-    ) -> ExtractTokensResult<Symbol> {
-        for (i, variable) in self.extracted_variables.iter_mut().enumerate() {
-            if variable.rule == *rule {
-                self.extracted_usage_counts[i] += 1;
-                return Ok(Symbol::terminal(i));
-            }
-        }
-
-        let index = self.extracted_variables.len();
-        let variable = if let Some(string_value) = string_value {
-            if string_value.is_empty() && !self.is_first_rule {
-                Err(ExtractTokensError::EmptyString(
-                    self.current_variable_name.clone(),
-                ))?;
-            }
-            Variable {
-                name: string_value.clone(),
-                kind: VariableType::Anonymous,
-                rule: rule.clone(),
-            }
-        } else {
-            self.current_variable_token_count += 1;
-            Variable {
-                name: format!(
-                    "{}_token{}",
-                    self.current_variable_name, self.current_variable_token_count
-                ),
-                kind: VariableType::Auxiliary,
-                rule: rule.clone(),
-            }
-        };
-
-        self.extracted_variables.push(variable);
-        self.extracted_usage_counts.push(1);
-        Ok(Symbol::terminal(index))
-    }
-}
-
-impl SymbolReplacer {
-    fn replace_symbols_in_rule(&mut self, rule: &Rule) -> Rule {
-        match rule {
-            Rule::Symbol(symbol) => self.replace_symbol(*symbol).into(),
-            Rule::Choice(elements) => Rule::Choice(
-                elements
-                    .iter()
-                    .map(|e| self.replace_symbols_in_rule(e))
-                    .collect(),
-            ),
-            Rule::Seq(elements) => Rule::Seq(
-                elements
-                    .iter()
-                    .map(|e| self.replace_symbols_in_rule(e))
-                    .collect(),
-            ),
-            Rule::Repeat(content) => Rule::Repeat(Box::new(self.replace_symbols_in_rule(content))),
-            Rule::Metadata { rule, params } => Rule::Metadata {
-                params: params.clone(),
-                rule: Box::new(self.replace_symbols_in_rule(rule)),
-            },
-            Rule::Reserved { rule, context_name } => Rule::Reserved {
-                rule: Box::new(self.replace_symbols_in_rule(rule)),
-                context_name: context_name.clone(),
-            },
-            _ => rule.clone(),
-        }
-    }
-
-    fn replace_symbol(&self, symbol: Symbol) -> Symbol {
-        if !symbol.is_non_terminal() {
-            return symbol;
-        }
-
-        if let Some(replacement) = self.replacements.get(&symbol.index) {
-            return Symbol::terminal(*replacement);
-        }
-
-        let mut adjusted_index = symbol.index;
-        for replaced_index in self.replacements.keys() {
-            if *replaced_index < symbol.index {
-                adjusted_index -= 1;
-            }
-        }
-
-        Symbol::non_terminal(adjusted_index)
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_extraction() {
-        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
-            Variable::named(
-                "rule_0",
-                Rule::repeat(Rule::seq(vec![
-                    Rule::string("a"),
-                    Rule::pattern("b", ""),
-                    Rule::choice(vec![
-                        Rule::non_terminal(1),
-                        Rule::non_terminal(2),
-                        Rule::token(Rule::repeat(Rule::choice(vec![
-                            Rule::string("c"),
-                            Rule::string("d"),
-                        ]))),
-                    ]),
-                ])),
-            ),
-            Variable::named("rule_1", Rule::pattern("e", "")),
-            Variable::named("rule_2", Rule::pattern("b", "")),
-            Variable::named(
-                "rule_3",
-                Rule::seq(vec![Rule::non_terminal(2), Rule::Blank]),
-            ),
-        ]))
-        .unwrap();
-
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![
-                Variable::named(
-                    "rule_0",
-                    Rule::repeat(Rule::seq(vec![
-                        // The string "a" was replaced by a symbol referencing the lexical grammar
-                        Rule::terminal(0),
-                        // The pattern "b" was replaced by a symbol referencing the lexical grammar
-                        Rule::terminal(1),
-                        Rule::choice(vec![
-                            // The symbol referencing `rule_1` was replaced by a symbol referencing
-                            // the lexical grammar.
-                            Rule::terminal(3),
-                            // The symbol referencing `rule_2` had its index decremented because
-                            // `rule_1` was moved to the lexical grammar.
-                            Rule::non_terminal(1),
-                            // The rule wrapped in `token` was replaced by a symbol referencing
-                            // the lexical grammar.
-                            Rule::terminal(2),
-                        ])
-                    ]))
-                ),
-                // The pattern "e" was only used in once place: as the definition of `rule_1`,
-                // so that rule was moved to the lexical grammar. The pattern "b" appeared in
-                // two places, so it was not moved into the lexical grammar.
-                Variable::named("rule_2", Rule::terminal(1)),
-                Variable::named(
-                    "rule_3",
-                    Rule::seq(vec![Rule::non_terminal(1), Rule::Blank,])
-                ),
-            ]
-        );
-
-        assert_eq!(
-            lexical_grammar.variables,
-            vec![
-                Variable::anonymous("a", Rule::string("a")),
-                Variable::auxiliary("rule_0_token1", Rule::pattern("b", "")),
-                Variable::auxiliary(
-                    "rule_0_token2",
-                    Rule::repeat(Rule::choice(vec![Rule::string("c"), Rule::string("d"),]))
-                ),
-                Variable::named("rule_1", Rule::pattern("e", "")),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_start_rule_is_token() {
-        let (syntax_grammar, lexical_grammar) =
-            extract_tokens(build_grammar(vec![Variable::named(
-                "rule_0",
-                Rule::string("hello"),
-            )]))
-            .unwrap();
-
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![Variable::named("rule_0", Rule::terminal(0)),]
-        );
-        assert_eq!(
-            lexical_grammar.variables,
-            vec![Variable::anonymous("hello", Rule::string("hello")),]
-        );
-    }
-
-    #[test]
-    fn test_extracting_extra_symbols() {
-        let mut grammar = build_grammar(vec![
-            Variable::named("rule_0", Rule::string("x")),
-            Variable::named("comment", Rule::pattern("//.*", "")),
-        ]);
-        grammar.extra_symbols = vec![Rule::string(" "), Rule::non_terminal(1)];
-
-        let (syntax_grammar, lexical_grammar) = extract_tokens(grammar).unwrap();
-        assert_eq!(syntax_grammar.extra_symbols, vec![Symbol::terminal(1),]);
-        assert_eq!(lexical_grammar.separators, vec![Rule::string(" "),]);
-    }
-
-    #[test]
-    fn test_extract_externals() {
-        let mut grammar = build_grammar(vec![
-            Variable::named(
-                "rule_0",
-                Rule::seq(vec![
-                    Rule::external(0),
-                    Rule::string("a"),
-                    Rule::non_terminal(1),
-                    Rule::non_terminal(2),
-                ]),
-            ),
-            Variable::named("rule_1", Rule::string("b")),
-            Variable::named("rule_2", Rule::string("c")),
-        ]);
-        grammar.external_tokens = vec![
-            Variable::named("external_0", Rule::external(0)),
-            Variable::anonymous("a", Rule::string("a")),
-            Variable::named("rule_2", Rule::non_terminal(2)),
-        ];
-
-        let (syntax_grammar, _) = extract_tokens(grammar).unwrap();
-
-        assert_eq!(
-            syntax_grammar.external_tokens,
-            vec![
-                ExternalToken {
-                    name: "external_0".to_string(),
-                    kind: VariableType::Named,
-                    corresponding_internal_token: None,
-                },
-                ExternalToken {
-                    name: "a".to_string(),
-                    kind: VariableType::Anonymous,
-                    corresponding_internal_token: Some(Symbol::terminal(0)),
-                },
-                ExternalToken {
-                    name: "rule_2".to_string(),
-                    kind: VariableType::Named,
-                    corresponding_internal_token: Some(Symbol::terminal(2)),
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_error_on_external_with_same_name_as_non_terminal() {
-        let mut grammar = build_grammar(vec![
-            Variable::named(
-                "rule_0",
-                Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
-            ),
-            Variable::named(
-                "rule_1",
-                Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2)]),
-            ),
-            Variable::named("rule_2", Rule::string("a")),
-        ]);
-        grammar.external_tokens = vec![Variable::named("rule_1", Rule::non_terminal(1))];
-
-        match extract_tokens(grammar) {
-            Err(e) => {
-                assert_eq!(e.to_string(), "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule");
-            }
-            _ => {
-                panic!("Expected an error but got no error");
-            }
-        }
-    }
-
-    #[test]
-    fn test_extraction_on_hidden_terminal() {
-        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
-            Variable::named("rule_0", Rule::non_terminal(1)),
-            Variable::hidden("_rule_1", Rule::string("a")),
-        ]))
-        .unwrap();
-
-        // The rule `_rule_1` should not "absorb" the
-        // terminal "a", since it is hidden,
-        // so we expect two variables still
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![
-                Variable::named("rule_0", Rule::non_terminal(1)),
-                Variable::hidden("_rule_1", Rule::terminal(0)),
-            ]
-        );
-
-        // We should not have a hidden rule in our lexical grammar, only the terminal "a"
-        assert_eq!(
-            lexical_grammar.variables,
-            vec![Variable::anonymous("a", Rule::string("a"))]
-        );
-    }
-
-    #[test]
-    fn test_extraction_with_empty_string() {
-        assert!(extract_tokens(build_grammar(vec![
-            Variable::named("rule_0", Rule::non_terminal(1)),
-            Variable::hidden("_rule_1", Rule::string("")),
-        ]))
-        .is_err());
-    }
-
-    fn build_grammar(variables: Vec<Variable>) -> InternedGrammar {
-        InternedGrammar {
-            variables,
-            ..Default::default()
-        }
-    }
-}

cli/generate/src/prepare_grammar/flatten_grammar.rs

@@ -1,539 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::Result;
-use serde::Serialize;
-use thiserror::Error;
-
-use super::ExtractedSyntaxGrammar;
-use crate::{
-    grammars::{
-        Production, ProductionStep, ReservedWordSetId, SyntaxGrammar, SyntaxVariable, Variable,
-    },
-    rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet},
-};
-
-pub type FlattenGrammarResult<T> = Result<T, FlattenGrammarError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum FlattenGrammarError {
-    #[error("No such reserved word set: {0}")]
-    NoReservedWordSet(String),
-    #[error(
-        "The rule `{0}` matches the empty string.
-
-Tree-sitter does not support syntactic rules that match the empty string
-unless they are used only as the grammar's start rule.
-"
-    )]
-    EmptyString(String),
-    #[error("Rule `{0}` cannot be inlined because it contains a reference to itself")]
-    RecursiveInline(String),
-}
-
-struct RuleFlattener {
-    production: Production,
-    reserved_word_set_ids: HashMap<String, ReservedWordSetId>,
-    precedence_stack: Vec<Precedence>,
-    associativity_stack: Vec<Associativity>,
-    reserved_word_stack: Vec<ReservedWordSetId>,
-    alias_stack: Vec<Alias>,
-    field_name_stack: Vec<String>,
-}
-
-impl RuleFlattener {
-    const fn new(reserved_word_set_ids: HashMap<String, ReservedWordSetId>) -> Self {
-        Self {
-            production: Production {
-                steps: Vec::new(),
-                dynamic_precedence: 0,
-            },
-            reserved_word_set_ids,
-            precedence_stack: Vec::new(),
-            associativity_stack: Vec::new(),
-            reserved_word_stack: Vec::new(),
-            alias_stack: Vec::new(),
-            field_name_stack: Vec::new(),
-        }
-    }
-
-    fn flatten_variable(&mut self, variable: Variable) -> FlattenGrammarResult<SyntaxVariable> {
-        let mut productions = Vec::new();
-        for rule in extract_choices(variable.rule) {
-            let production = self.flatten_rule(rule)?;
-            if !productions.contains(&production) {
-                productions.push(production);
-            }
-        }
-        Ok(SyntaxVariable {
-            name: variable.name,
-            kind: variable.kind,
-            productions,
-        })
-    }
-
-    fn flatten_rule(&mut self, rule: Rule) -> FlattenGrammarResult<Production> {
-        self.production = Production::default();
-        self.alias_stack.clear();
-        self.reserved_word_stack.clear();
-        self.precedence_stack.clear();
-        self.associativity_stack.clear();
-        self.field_name_stack.clear();
-        self.apply(rule, true)?;
-        Ok(self.production.clone())
-    }
-
-    fn apply(&mut self, rule: Rule, at_end: bool) -> FlattenGrammarResult<bool> {
-        match rule {
-            Rule::Seq(members) => {
-                let mut result = false;
-                let last_index = members.len() - 1;
-                for (i, member) in members.into_iter().enumerate() {
-                    result |= self.apply(member, i == last_index && at_end)?;
-                }
-                Ok(result)
-            }
-            Rule::Metadata { rule, params } => {
-                let mut has_precedence = false;
-                if !params.precedence.is_none() {
-                    has_precedence = true;
-                    self.precedence_stack.push(params.precedence);
-                }
-
-                let mut has_associativity = false;
-                if let Some(associativity) = params.associativity {
-                    has_associativity = true;
-                    self.associativity_stack.push(associativity);
-                }
-
-                let mut has_alias = false;
-                if let Some(alias) = params.alias {
-                    has_alias = true;
-                    self.alias_stack.push(alias);
-                }
-
-                let mut has_field_name = false;
-                if let Some(field_name) = params.field_name {
-                    has_field_name = true;
-                    self.field_name_stack.push(field_name);
-                }
-
-                if params.dynamic_precedence.abs() > self.production.dynamic_precedence.abs() {
-                    self.production.dynamic_precedence = params.dynamic_precedence;
-                }
-
-                let did_push = self.apply(*rule, at_end)?;
-
-                if has_precedence {
-                    self.precedence_stack.pop();
-                    if did_push && !at_end {
-                        self.production.steps.last_mut().unwrap().precedence = self
-                            .precedence_stack
-                            .last()
-                            .cloned()
-                            .unwrap_or(Precedence::None);
-                    }
-                }
-
-                if has_associativity {
-                    self.associativity_stack.pop();
-                    if did_push && !at_end {
-                        self.production.steps.last_mut().unwrap().associativity =
-                            self.associativity_stack.last().copied();
-                    }
-                }
-
-                if has_alias {
-                    self.alias_stack.pop();
-                }
-
-                if has_field_name {
-                    self.field_name_stack.pop();
-                }
-
-                Ok(did_push)
-            }
-            Rule::Reserved { rule, context_name } => {
-                self.reserved_word_stack.push(
-                    self.reserved_word_set_ids
-                        .get(&context_name)
-                        .copied()
-                        .ok_or_else(|| {
-                            FlattenGrammarError::NoReservedWordSet(context_name.clone())
-                        })?,
-                );
-                let did_push = self.apply(*rule, at_end)?;
-                self.reserved_word_stack.pop();
-                Ok(did_push)
-            }
-            Rule::Symbol(symbol) => {
-                self.production.steps.push(ProductionStep {
-                    symbol,
-                    precedence: self
-                        .precedence_stack
-                        .last()
-                        .cloned()
-                        .unwrap_or(Precedence::None),
-                    associativity: self.associativity_stack.last().copied(),
-                    reserved_word_set_id: self
-                        .reserved_word_stack
-                        .last()
-                        .copied()
-                        .unwrap_or(ReservedWordSetId::default()),
-                    alias: self.alias_stack.last().cloned(),
-                    field_name: self.field_name_stack.last().cloned(),
-                });
-                Ok(true)
-            }
-            _ => Ok(false),
-        }
-    }
-}
-
-fn extract_choices(rule: Rule) -> Vec<Rule> {
-    match rule {
-        Rule::Seq(elements) => {
-            let mut result = vec![Rule::Blank];
-            for element in elements {
-                let extraction = extract_choices(element);
-                let mut next_result = Vec::new();
-                for entry in result {
-                    for extraction_entry in &extraction {
-                        next_result.push(Rule::Seq(vec![entry.clone(), extraction_entry.clone()]));
-                    }
-                }
-                result = next_result;
-            }
-            result
-        }
-        Rule::Choice(elements) => {
-            let mut result = Vec::new();
-            for element in elements {
-                for rule in extract_choices(element) {
-                    result.push(rule);
-                }
-            }
-            result
-        }
-        Rule::Metadata { rule, params } => extract_choices(*rule)
-            .into_iter()
-            .map(|rule| Rule::Metadata {
-                rule: Box::new(rule),
-                params: params.clone(),
-            })
-            .collect(),
-        Rule::Reserved { rule, context_name } => extract_choices(*rule)
-            .into_iter()
-            .map(|rule| Rule::Reserved {
-                rule: Box::new(rule),
-                context_name: context_name.clone(),
-            })
-            .collect(),
-        _ => vec![rule],
-    }
-}
-
-fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool {
-    for variable in variables {
-        for production in &variable.productions {
-            for step in &production.steps {
-                if step.symbol == symbol {
-                    return true;
-                }
-            }
-        }
-    }
-    false
-}
-
-pub(super) fn flatten_grammar(
-    grammar: ExtractedSyntaxGrammar,
-) -> FlattenGrammarResult<SyntaxGrammar> {
-    let mut reserved_word_set_ids_by_name = HashMap::new();
-    for (ix, set) in grammar.reserved_word_sets.iter().enumerate() {
-        reserved_word_set_ids_by_name.insert(set.name.clone(), ReservedWordSetId(ix));
-    }
-
-    let mut flattener = RuleFlattener::new(reserved_word_set_ids_by_name);
-    let variables = grammar
-        .variables
-        .into_iter()
-        .map(|variable| flattener.flatten_variable(variable))
-        .collect::<FlattenGrammarResult<Vec<_>>>()?;
-
-    for (i, variable) in variables.iter().enumerate() {
-        let symbol = Symbol::non_terminal(i);
-
-        for production in &variable.productions {
-            if production.steps.is_empty() && symbol_is_used(&variables, symbol) {
-                Err(FlattenGrammarError::EmptyString(variable.name.clone()))?;
-            }
-
-            if grammar.variables_to_inline.contains(&symbol)
-                && production.steps.iter().any(|step| step.symbol == symbol)
-            {
-                Err(FlattenGrammarError::RecursiveInline(variable.name.clone()))?;
-            }
-        }
-    }
-    let mut reserved_word_sets = grammar
-        .reserved_word_sets
-        .into_iter()
-        .map(|set| set.reserved_words.into_iter().collect())
-        .collect::<Vec<_>>();
-
-    // If no default reserved word set is specified, there are no reserved words.
-    if reserved_word_sets.is_empty() {
-        reserved_word_sets.push(TokenSet::default());
-    }
-
-    Ok(SyntaxGrammar {
-        extra_symbols: grammar.extra_symbols,
-        expected_conflicts: grammar.expected_conflicts,
-        variables_to_inline: grammar.variables_to_inline,
-        precedence_orderings: grammar.precedence_orderings,
-        external_tokens: grammar.external_tokens,
-        supertype_symbols: grammar.supertype_symbols,
-        word_token: grammar.word_token,
-        reserved_word_sets,
-        variables,
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::grammars::VariableType;
-
-    #[test]
-    fn test_flatten_grammar() {
-        let mut flattener = RuleFlattener::new(HashMap::default());
-        let result = flattener
-            .flatten_variable(Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::seq(vec![
-                    Rule::non_terminal(1),
-                    Rule::prec_left(
-                        Precedence::Integer(101),
-                        Rule::seq(vec![
-                            Rule::non_terminal(2),
-                            Rule::choice(vec![
-                                Rule::prec_right(
-                                    Precedence::Integer(102),
-                                    Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
-                                ),
-                                Rule::non_terminal(5),
-                            ]),
-                            Rule::non_terminal(6),
-                        ]),
-                    ),
-                    Rule::non_terminal(7),
-                ]),
-            })
-            .unwrap();
-
-        assert_eq!(
-            result.productions,
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(3))
-                            .with_prec(Precedence::Integer(102), Some(Associativity::Right)),
-                        ProductionStep::new(Symbol::non_terminal(4))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ]
-                },
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(5))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ]
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_maximum_dynamic_precedence() {
-        let mut flattener = RuleFlattener::new(HashMap::default());
-        let result = flattener
-            .flatten_variable(Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::seq(vec![
-                    Rule::non_terminal(1),
-                    Rule::prec_dynamic(
-                        101,
-                        Rule::seq(vec![
-                            Rule::non_terminal(2),
-                            Rule::choice(vec![
-                                Rule::prec_dynamic(
-                                    102,
-                                    Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
-                                ),
-                                Rule::non_terminal(5),
-                            ]),
-                            Rule::non_terminal(6),
-                        ]),
-                    ),
-                    Rule::non_terminal(7),
-                ]),
-            })
-            .unwrap();
-
-        assert_eq!(
-            result.productions,
-            vec![
-                Production {
-                    dynamic_precedence: 102,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::non_terminal(3)),
-                        ProductionStep::new(Symbol::non_terminal(4)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ],
-                },
-                Production {
-                    dynamic_precedence: 101,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::non_terminal(5)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ],
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_final_precedence() {
-        let mut flattener = RuleFlattener::new(HashMap::default());
-        let result = flattener
-            .flatten_variable(Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::prec_left(
-                    Precedence::Integer(101),
-                    Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
-                ),
-            })
-            .unwrap();
-
-        assert_eq!(
-            result.productions,
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    ProductionStep::new(Symbol::non_terminal(1))
-                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                    ProductionStep::new(Symbol::non_terminal(2))
-                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                ]
-            }]
-        );
-
-        let result = flattener
-            .flatten_variable(Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::prec_left(
-                    Precedence::Integer(101),
-                    Rule::seq(vec![Rule::non_terminal(1)]),
-                ),
-            })
-            .unwrap();
-
-        assert_eq!(
-            result.productions,
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![ProductionStep::new(Symbol::non_terminal(1))
-                    .with_prec(Precedence::Integer(101), Some(Associativity::Left)),]
-            }]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_field_names() {
-        let mut flattener = RuleFlattener::new(HashMap::default());
-        let result = flattener
-            .flatten_variable(Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::seq(vec![
-                    Rule::field("first-thing".to_string(), Rule::terminal(1)),
-                    Rule::terminal(2),
-                    Rule::choice(vec![
-                        Rule::Blank,
-                        Rule::field("second-thing".to_string(), Rule::terminal(3)),
-                    ]),
-                ]),
-            })
-            .unwrap();
-
-        assert_eq!(
-            result.productions,
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
-                        ProductionStep::new(Symbol::terminal(2))
-                    ]
-                },
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
-                        ProductionStep::new(Symbol::terminal(2)),
-                        ProductionStep::new(Symbol::terminal(3)).with_field_name("second-thing"),
-                    ]
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_recursive_inline_variable() {
-        let result = flatten_grammar(ExtractedSyntaxGrammar {
-            extra_symbols: Vec::new(),
-            expected_conflicts: Vec::new(),
-            variables_to_inline: vec![Symbol::non_terminal(0)],
-            precedence_orderings: Vec::new(),
-            external_tokens: Vec::new(),
-            supertype_symbols: Vec::new(),
-            word_token: None,
-            reserved_word_sets: Vec::new(),
-            variables: vec![Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::seq(vec![
-                    Rule::non_terminal(0),
-                    Rule::non_terminal(1),
-                    Rule::non_terminal(2),
-                ]),
-            }],
-        });
-
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Rule `test` cannot be inlined because it contains a reference to itself.",
-        );
-    }
-}

cli/generate/src/prepare_grammar/intern_symbols.rs

@@ -1,293 +0,0 @@
-use anyhow::Result;
-use serde::Serialize;
-use thiserror::Error;
-
-use super::InternedGrammar;
-use crate::{
-    grammars::{InputGrammar, ReservedWordContext, Variable, VariableType},
-    rules::{Rule, Symbol},
-};
-
-pub type InternSymbolsResult<T> = Result<T, InternSymbolsError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum InternSymbolsError {
-    #[error("A grammar's start rule must be visible.")]
-    HiddenStartRule,
-    #[error("Undefined symbol `{0}`")]
-    Undefined(String),
-    #[error("Undefined symbol `{0}` in grammar's supertypes array")]
-    UndefinedSupertype(String),
-    #[error("Undefined symbol `{0}` in grammar's conflicts array")]
-    UndefinedConflict(String),
-    #[error("Undefined symbol `{0}` as grammar's word token")]
-    UndefinedWordToken(String),
-}
-
-pub(super) fn intern_symbols(grammar: &InputGrammar) -> InternSymbolsResult<InternedGrammar> {
-    let interner = Interner { grammar };
-
-    if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden {
-        Err(InternSymbolsError::HiddenStartRule)?;
-    }
-
-    let mut variables = Vec::with_capacity(grammar.variables.len());
-    for variable in &grammar.variables {
-        variables.push(Variable {
-            name: variable.name.clone(),
-            kind: variable_type_for_name(&variable.name),
-            rule: interner.intern_rule(&variable.rule, Some(&variable.name))?,
-        });
-    }
-
-    let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len());
-    for external_token in &grammar.external_tokens {
-        let rule = interner.intern_rule(external_token, None)?;
-        let (name, kind) = if let Rule::NamedSymbol(name) = external_token {
-            (name.clone(), variable_type_for_name(name))
-        } else {
-            (String::new(), VariableType::Anonymous)
-        };
-        external_tokens.push(Variable { name, kind, rule });
-    }
-
-    let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len());
-    for extra_token in &grammar.extra_symbols {
-        extra_symbols.push(interner.intern_rule(extra_token, None)?);
-    }
-
-    let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len());
-    for supertype_symbol_name in &grammar.supertype_symbols {
-        supertype_symbols.push(interner.intern_name(supertype_symbol_name).ok_or_else(|| {
-            InternSymbolsError::UndefinedSupertype(supertype_symbol_name.clone())
-        })?);
-    }
-
-    let mut reserved_words = Vec::with_capacity(grammar.reserved_words.len());
-    for reserved_word_set in &grammar.reserved_words {
-        let mut interned_set = Vec::new();
-        for rule in &reserved_word_set.reserved_words {
-            interned_set.push(interner.intern_rule(rule, None)?);
-        }
-        reserved_words.push(ReservedWordContext {
-            name: reserved_word_set.name.clone(),
-            reserved_words: interned_set,
-        });
-    }
-
-    let mut expected_conflicts = Vec::new();
-    for conflict in &grammar.expected_conflicts {
-        let mut interned_conflict = Vec::with_capacity(conflict.len());
-        for name in conflict {
-            interned_conflict.push(
-                interner
-                    .intern_name(name)
-                    .ok_or_else(|| InternSymbolsError::UndefinedConflict(name.clone()))?,
-            );
-        }
-        expected_conflicts.push(interned_conflict);
-    }
-
-    let mut variables_to_inline = Vec::new();
-    for name in &grammar.variables_to_inline {
-        if let Some(symbol) = interner.intern_name(name) {
-            variables_to_inline.push(symbol);
-        }
-    }
-
-    let mut word_token = None;
-    if let Some(name) = grammar.word_token.as_ref() {
-        word_token = Some(
-            interner
-                .intern_name(name)
-                .ok_or_else(|| InternSymbolsError::UndefinedWordToken(name.clone()))?,
-        );
-    }
-
-    for (i, variable) in variables.iter_mut().enumerate() {
-        if supertype_symbols.contains(&Symbol::non_terminal(i)) {
-            variable.kind = VariableType::Hidden;
-        }
-    }
-
-    Ok(InternedGrammar {
-        variables,
-        external_tokens,
-        extra_symbols,
-        expected_conflicts,
-        variables_to_inline,
-        supertype_symbols,
-        word_token,
-        precedence_orderings: grammar.precedence_orderings.clone(),
-        reserved_word_sets: reserved_words,
-    })
-}
-
-struct Interner<'a> {
-    grammar: &'a InputGrammar,
-}
-
-impl Interner<'_> {
-    fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> InternSymbolsResult<Rule> {
-        match rule {
-            Rule::Choice(elements) => {
-                self.check_single(elements, name);
-                let mut result = Vec::with_capacity(elements.len());
-                for element in elements {
-                    result.push(self.intern_rule(element, name)?);
-                }
-                Ok(Rule::Choice(result))
-            }
-            Rule::Seq(elements) => {
-                self.check_single(elements, name);
-                let mut result = Vec::with_capacity(elements.len());
-                for element in elements {
-                    result.push(self.intern_rule(element, name)?);
-                }
-                Ok(Rule::Seq(result))
-            }
-            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(self.intern_rule(content, name)?))),
-            Rule::Metadata { rule, params } => Ok(Rule::Metadata {
-                rule: Box::new(self.intern_rule(rule, name)?),
-                params: params.clone(),
-            }),
-            Rule::Reserved { rule, context_name } => Ok(Rule::Reserved {
-                rule: Box::new(self.intern_rule(rule, name)?),
-                context_name: context_name.clone(),
-            }),
-            Rule::NamedSymbol(name) => self.intern_name(name).map_or_else(
-                || Err(InternSymbolsError::Undefined(name.clone())),
-                |symbol| Ok(Rule::Symbol(symbol)),
-            ),
-            _ => Ok(rule.clone()),
-        }
-    }
-
-    fn intern_name(&self, symbol: &str) -> Option<Symbol> {
-        for (i, variable) in self.grammar.variables.iter().enumerate() {
-            if variable.name == symbol {
-                return Some(Symbol::non_terminal(i));
-            }
-        }
-
-        for (i, external_token) in self.grammar.external_tokens.iter().enumerate() {
-            if let Rule::NamedSymbol(name) = external_token {
-                if name == symbol {
-                    return Some(Symbol::external(i));
-                }
-            }
-        }
-
-        None
-    }
-
-    // In the case of a seq or choice rule of 1 element in a hidden rule, weird
-    // inconsistent behavior with queries can occur. So we should warn the user about it.
-    fn check_single(&self, elements: &[Rule], name: Option<&str>) {
-        if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) {
-            eprintln!(
-                "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.",
-                name.unwrap_or_default()
-            );
-        }
-    }
-}
-
-fn variable_type_for_name(name: &str) -> VariableType {
-    if name.starts_with('_') {
-        VariableType::Hidden
-    } else {
-        VariableType::Named
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_basic_repeat_expansion() {
-        let grammar = intern_symbols(&build_grammar(vec![
-            Variable::named("x", Rule::choice(vec![Rule::named("y"), Rule::named("_z")])),
-            Variable::named("y", Rule::named("_z")),
-            Variable::named("_z", Rule::string("a")),
-        ]))
-        .unwrap();
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "x",
-                    Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),])
-                ),
-                Variable::named("y", Rule::non_terminal(2)),
-                Variable::hidden("_z", Rule::string("a")),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_interning_external_token_names() {
-        // Variable `y` is both an internal and an external token.
-        // Variable `z` is just an external token.
-        let mut input_grammar = build_grammar(vec![
-            Variable::named(
-                "w",
-                Rule::choice(vec![Rule::named("x"), Rule::named("y"), Rule::named("z")]),
-            ),
-            Variable::named("x", Rule::string("a")),
-            Variable::named("y", Rule::string("b")),
-        ]);
-        input_grammar
-            .external_tokens
-            .extend(vec![Rule::named("y"), Rule::named("z")]);
-
-        let grammar = intern_symbols(&input_grammar).unwrap();
-
-        // Variable `y` is referred to by its internal index.
-        // Variable `z` is referred to by its external index.
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "w",
-                    Rule::choice(vec![
-                        Rule::non_terminal(1),
-                        Rule::non_terminal(2),
-                        Rule::external(1),
-                    ])
-                ),
-                Variable::named("x", Rule::string("a")),
-                Variable::named("y", Rule::string("b")),
-            ]
-        );
-
-        // The external token for `y` refers back to its internal index.
-        assert_eq!(
-            grammar.external_tokens,
-            vec![
-                Variable::named("y", Rule::non_terminal(2)),
-                Variable::named("z", Rule::external(1)),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_grammar_with_undefined_symbols() {
-        let result = intern_symbols(&build_grammar(vec![Variable::named("x", Rule::named("y"))]));
-
-        match result {
-            Err(e) => assert_eq!(e.to_string(), "Undefined symbol `y`"),
-            _ => panic!("Expected an error but got none"),
-        }
-    }
-
-    fn build_grammar(variables: Vec<Variable>) -> InputGrammar {
-        InputGrammar {
-            variables,
-            name: "the_language".to_string(),
-            ..Default::default()
-        }
-    }
-}

cli/generate/src/prepare_grammar/mod.rs

@@ -1,325 +0,0 @@
-mod expand_repeats;
-mod expand_tokens;
-mod extract_default_aliases;
-mod extract_tokens;
-mod flatten_grammar;
-mod intern_symbols;
-mod process_inlines;
-
-use std::{
-    cmp::Ordering,
-    collections::{hash_map, HashMap, HashSet},
-    mem,
-};
-
-use anyhow::Result;
-pub use expand_tokens::ExpandTokensError;
-pub use extract_tokens::ExtractTokensError;
-pub use flatten_grammar::FlattenGrammarError;
-pub use intern_symbols::InternSymbolsError;
-pub use process_inlines::ProcessInlinesError;
-use serde::Serialize;
-use thiserror::Error;
-
-pub use self::expand_tokens::expand_tokens;
-use self::{
-    expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
-    extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
-    intern_symbols::intern_symbols, process_inlines::process_inlines,
-};
-use super::{
-    grammars::{
-        ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
-        SyntaxGrammar, Variable,
-    },
-    rules::{AliasMap, Precedence, Rule, Symbol},
-};
-use crate::grammars::ReservedWordContext;
-
-pub struct IntermediateGrammar<T, U> {
-    variables: Vec<Variable>,
-    extra_symbols: Vec<T>,
-    expected_conflicts: Vec<Vec<Symbol>>,
-    precedence_orderings: Vec<Vec<PrecedenceEntry>>,
-    external_tokens: Vec<U>,
-    variables_to_inline: Vec<Symbol>,
-    supertype_symbols: Vec<Symbol>,
-    word_token: Option<Symbol>,
-    reserved_word_sets: Vec<ReservedWordContext<T>>,
-}
-
-pub type InternedGrammar = IntermediateGrammar<Rule, Variable>;
-
-pub type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct ExtractedLexicalGrammar {
-    pub variables: Vec<Variable>,
-    pub separators: Vec<Rule>,
-}
-
-impl<T, U> Default for IntermediateGrammar<T, U> {
-    fn default() -> Self {
-        Self {
-            variables: Vec::default(),
-            extra_symbols: Vec::default(),
-            expected_conflicts: Vec::default(),
-            precedence_orderings: Vec::default(),
-            external_tokens: Vec::default(),
-            variables_to_inline: Vec::default(),
-            supertype_symbols: Vec::default(),
-            word_token: Option::default(),
-            reserved_word_sets: Vec::default(),
-        }
-    }
-}
-
-pub type PrepareGrammarResult<T> = Result<T, PrepareGrammarError>;
-
-#[derive(Debug, Error, Serialize)]
-#[error(transparent)]
-pub enum PrepareGrammarError {
-    ValidatePrecedences(#[from] ValidatePrecedenceError),
-    InternSymbols(#[from] InternSymbolsError),
-    ExtractTokens(#[from] ExtractTokensError),
-    FlattenGrammar(#[from] FlattenGrammarError),
-    ExpandTokens(#[from] ExpandTokensError),
-    ProcessInlines(#[from] ProcessInlinesError),
-}
-
-pub type ValidatePrecedenceResult<T> = Result<T, ValidatePrecedenceError>;
-
-#[derive(Debug, Error, Serialize)]
-#[error(transparent)]
-pub enum ValidatePrecedenceError {
-    Undeclared(#[from] UndeclaredPrecedenceError),
-    Ordering(#[from] ConflictingPrecedenceOrderingError),
-}
-
-#[derive(Debug, Error, Serialize)]
-pub struct UndeclaredPrecedenceError {
-    pub precedence: String,
-    pub rule: String,
-}
-
-impl std::fmt::Display for UndeclaredPrecedenceError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "Undeclared precedence '{}' in rule '{}'",
-            self.precedence, self.rule
-        )?;
-        Ok(())
-    }
-}
-
-#[derive(Debug, Error, Serialize)]
-pub struct ConflictingPrecedenceOrderingError {
-    pub precedence_1: String,
-    pub precedence_2: String,
-}
-
-impl std::fmt::Display for ConflictingPrecedenceOrderingError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "Conflicting orderings for precedences {} and {}",
-            self.precedence_1, self.precedence_2
-        )?;
-        Ok(())
-    }
-}
-
-/// Transform an input grammar into separate components that are ready
-/// for parse table construction.
-pub fn prepare_grammar(
-    input_grammar: &InputGrammar,
-) -> PrepareGrammarResult<(
-    SyntaxGrammar,
-    LexicalGrammar,
-    InlinedProductionMap,
-    AliasMap,
-)> {
-    validate_precedences(input_grammar)?;
-
-    let interned_grammar = intern_symbols(input_grammar)?;
-    let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
-    let syntax_grammar = expand_repeats(syntax_grammar);
-    let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
-    let lexical_grammar = expand_tokens(lexical_grammar)?;
-    let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
-    let inlines = process_inlines(&syntax_grammar, &lexical_grammar)?;
-    Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
-}
-
-/// Check that all of the named precedences used in the grammar are declared
-/// within the `precedences` lists, and also that there are no conflicting
-/// precedence orderings declared in those lists.
-fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()> {
-    // Check that no rule contains a named precedence that is not present in
-    // any of the `precedences` lists.
-    fn validate(
-        rule_name: &str,
-        rule: &Rule,
-        names: &HashSet<&String>,
-    ) -> ValidatePrecedenceResult<()> {
-        match rule {
-            Rule::Repeat(rule) => validate(rule_name, rule, names),
-            Rule::Seq(elements) | Rule::Choice(elements) => elements
-                .iter()
-                .try_for_each(|e| validate(rule_name, e, names)),
-            Rule::Metadata { rule, params } => {
-                if let Precedence::Name(n) = &params.precedence {
-                    if !names.contains(n) {
-                        Err(UndeclaredPrecedenceError {
-                            precedence: n.to_string(),
-                            rule: rule_name.to_string(),
-                        })?;
-                    }
-                }
-                validate(rule_name, rule, names)?;
-                Ok(())
-            }
-            _ => Ok(()),
-        }
-    }
-
-    // For any two precedence names `a` and `b`, if `a` comes before `b`
-    // in some list, then it cannot come *after* `b` in any list.
-    let mut pairs = HashMap::new();
-    for list in &grammar.precedence_orderings {
-        for (i, mut entry1) in list.iter().enumerate() {
-            for mut entry2 in list.iter().skip(i + 1) {
-                if entry2 == entry1 {
-                    continue;
-                }
-                let mut ordering = Ordering::Greater;
-                if entry1 > entry2 {
-                    ordering = Ordering::Less;
-                    mem::swap(&mut entry1, &mut entry2);
-                }
-                match pairs.entry((entry1, entry2)) {
-                    hash_map::Entry::Vacant(e) => {
-                        e.insert(ordering);
-                    }
-                    hash_map::Entry::Occupied(e) => {
-                        if e.get() != &ordering {
-                            Err(ConflictingPrecedenceOrderingError {
-                                precedence_1: entry1.to_string(),
-                                precedence_2: entry2.to_string(),
-                            })?;
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    let precedence_names = grammar
-        .precedence_orderings
-        .iter()
-        .flat_map(|l| l.iter())
-        .filter_map(|p| {
-            if let PrecedenceEntry::Name(n) = p {
-                Some(n)
-            } else {
-                None
-            }
-        })
-        .collect::<HashSet<&String>>();
-    for variable in &grammar.variables {
-        validate(&variable.name, &variable.rule, &precedence_names)?;
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::grammars::VariableType;
-
-    #[test]
-    fn test_validate_precedences_with_undeclared_precedence() {
-        let grammar = InputGrammar {
-            precedence_orderings: vec![
-                vec![
-                    PrecedenceEntry::Name("a".to_string()),
-                    PrecedenceEntry::Name("b".to_string()),
-                ],
-                vec![
-                    PrecedenceEntry::Name("b".to_string()),
-                    PrecedenceEntry::Name("c".to_string()),
-                    PrecedenceEntry::Name("d".to_string()),
-                ],
-            ],
-            variables: vec![
-                Variable {
-                    name: "v1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::Seq(vec![
-                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
-                    ]),
-                },
-                Variable {
-                    name: "v2".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::Choice(vec![
-                        Rule::prec_left(Precedence::Name("omg".to_string()), Rule::string("y")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
-                    ])),
-                },
-            ],
-            ..Default::default()
-        };
-
-        let result = validate_precedences(&grammar);
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Undeclared precedence 'omg' in rule 'v2'",
-        );
-    }
-
-    #[test]
-    fn test_validate_precedences_with_conflicting_order() {
-        let grammar = InputGrammar {
-            precedence_orderings: vec![
-                vec![
-                    PrecedenceEntry::Name("a".to_string()),
-                    PrecedenceEntry::Name("b".to_string()),
-                ],
-                vec![
-                    PrecedenceEntry::Name("b".to_string()),
-                    PrecedenceEntry::Name("c".to_string()),
-                    PrecedenceEntry::Name("a".to_string()),
-                ],
-            ],
-            variables: vec![
-                Variable {
-                    name: "v1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::Seq(vec![
-                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
-                    ]),
-                },
-                Variable {
-                    name: "v2".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::Choice(vec![
-                        Rule::prec_left(Precedence::Name("a".to_string()), Rule::string("y")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
-                    ])),
-                },
-            ],
-            ..Default::default()
-        };
-
-        let result = validate_precedences(&grammar);
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Conflicting orderings for precedences 'a' and 'b'",
-        );
-    }
-}

cli/generate/src/prepare_grammar/process_inlines.rs

@@ -1,558 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::Result;
-use serde::Serialize;
-use thiserror::Error;
-
-use crate::{
-    grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
-    rules::SymbolType,
-};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-struct ProductionStepId {
-    // A `None` value here means that the production itself was produced via inlining,
-    // and is stored in the builder's `productions` vector, as opposed to being
-    // stored in one of the grammar's variables.
-    variable_index: Option<usize>,
-    production_index: usize,
-    step_index: usize,
-}
-
-struct InlinedProductionMapBuilder {
-    production_indices_by_step_id: HashMap<ProductionStepId, Vec<usize>>,
-    productions: Vec<Production>,
-}
-
-impl InlinedProductionMapBuilder {
-    fn build(mut self, grammar: &SyntaxGrammar) -> InlinedProductionMap {
-        let mut step_ids_to_process = Vec::new();
-        for (variable_index, variable) in grammar.variables.iter().enumerate() {
-            for production_index in 0..variable.productions.len() {
-                step_ids_to_process.push(ProductionStepId {
-                    variable_index: Some(variable_index),
-                    production_index,
-                    step_index: 0,
-                });
-                while !step_ids_to_process.is_empty() {
-                    let mut i = 0;
-                    while i < step_ids_to_process.len() {
-                        let step_id = step_ids_to_process[i];
-                        if let Some(step) = self.production_step_for_id(step_id, grammar) {
-                            if grammar.variables_to_inline.contains(&step.symbol) {
-                                let inlined_step_ids = self
-                                    .inline_production_at_step(step_id, grammar)
-                                    .iter()
-                                    .copied()
-                                    .map(|production_index| ProductionStepId {
-                                        variable_index: None,
-                                        production_index,
-                                        step_index: step_id.step_index,
-                                    });
-                                step_ids_to_process.splice(i..=i, inlined_step_ids);
-                            } else {
-                                step_ids_to_process[i] = ProductionStepId {
-                                    variable_index: step_id.variable_index,
-                                    production_index: step_id.production_index,
-                                    step_index: step_id.step_index + 1,
-                                };
-                                i += 1;
-                            }
-                        } else {
-                            step_ids_to_process.remove(i);
-                        }
-                    }
-                }
-            }
-        }
-
-        let productions = self.productions;
-        let production_indices_by_step_id = self.production_indices_by_step_id;
-        let production_map = production_indices_by_step_id
-            .into_iter()
-            .map(|(step_id, production_indices)| {
-                let production = step_id.variable_index.map_or_else(
-                    || &productions[step_id.production_index],
-                    |variable_index| {
-                        &grammar.variables[variable_index].productions[step_id.production_index]
-                    },
-                ) as *const Production;
-                ((production, step_id.step_index as u32), production_indices)
-            })
-            .collect();
-
-        InlinedProductionMap {
-            productions,
-            production_map,
-        }
-    }
-
-    fn inline_production_at_step<'a>(
-        &'a mut self,
-        step_id: ProductionStepId,
-        grammar: &'a SyntaxGrammar,
-    ) -> &'a [usize] {
-        // Build a list of productions produced by inlining rules.
-        let mut i = 0;
-        let step_index = step_id.step_index;
-        let mut productions_to_add = vec![self.production_for_id(step_id, grammar).clone()];
-        while i < productions_to_add.len() {
-            if let Some(step) = productions_to_add[i].steps.get(step_index) {
-                let symbol = step.symbol;
-                if grammar.variables_to_inline.contains(&symbol) {
-                    // Remove the production from the vector, replacing it with a placeholder.
-                    let production = productions_to_add
-                        .splice(i..=i, std::iter::once(&Production::default()).cloned())
-                        .next()
-                        .unwrap();
-
-                    // Replace the placeholder with the inlined productions.
-                    productions_to_add.splice(
-                        i..=i,
-                        grammar.variables[symbol.index].productions.iter().map(|p| {
-                            let mut production = production.clone();
-                            let removed_step = production
-                                .steps
-                                .splice(step_index..=step_index, p.steps.iter().cloned())
-                                .next()
-                                .unwrap();
-                            let inserted_steps =
-                                &mut production.steps[step_index..(step_index + p.steps.len())];
-                            if let Some(alias) = removed_step.alias {
-                                for inserted_step in inserted_steps.iter_mut() {
-                                    inserted_step.alias = Some(alias.clone());
-                                }
-                            }
-                            if let Some(field_name) = removed_step.field_name {
-                                for inserted_step in inserted_steps.iter_mut() {
-                                    inserted_step.field_name = Some(field_name.clone());
-                                }
-                            }
-                            if let Some(last_inserted_step) = inserted_steps.last_mut() {
-                                if last_inserted_step.precedence.is_none() {
-                                    last_inserted_step.precedence = removed_step.precedence;
-                                }
-                                if last_inserted_step.associativity.is_none() {
-                                    last_inserted_step.associativity = removed_step.associativity;
-                                }
-                            }
-                            if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() {
-                                production.dynamic_precedence = p.dynamic_precedence;
-                            }
-                            production
-                        }),
-                    );
-
-                    continue;
-                }
-            }
-            i += 1;
-        }
-
-        // Store all the computed productions.
-        let result = productions_to_add
-            .into_iter()
-            .map(|production| {
-                self.productions
-                    .iter()
-                    .position(|p| *p == production)
-                    .unwrap_or_else(|| {
-                        self.productions.push(production);
-                        self.productions.len() - 1
-                    })
-            })
-            .collect();
-
-        // Cache these productions based on the original production step.
-        self.production_indices_by_step_id
-            .entry(step_id)
-            .or_insert(result)
-    }
-
-    fn production_for_id<'a>(
-        &'a self,
-        id: ProductionStepId,
-        grammar: &'a SyntaxGrammar,
-    ) -> &'a Production {
-        id.variable_index.map_or_else(
-            || &self.productions[id.production_index],
-            |variable_index| &grammar.variables[variable_index].productions[id.production_index],
-        )
-    }
-
-    fn production_step_for_id<'a>(
-        &'a self,
-        id: ProductionStepId,
-        grammar: &'a SyntaxGrammar,
-    ) -> Option<&'a ProductionStep> {
-        self.production_for_id(id, grammar).steps.get(id.step_index)
-    }
-}
-
-pub type ProcessInlinesResult<T> = Result<T, ProcessInlinesError>;
-
-#[derive(Debug, Error, Serialize)]
-pub enum ProcessInlinesError {
-    #[error("External token `{0}` cannot be inlined")]
-    ExternalToken(String),
-    #[error("Token `{0}` cannot be inlined")]
-    Token(String),
-    #[error("Rule `{0}` cannot be inlined because it is the first rule")]
-    FirstRule(String),
-}
-
-pub(super) fn process_inlines(
-    grammar: &SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-) -> ProcessInlinesResult<InlinedProductionMap> {
-    for symbol in &grammar.variables_to_inline {
-        match symbol.kind {
-            SymbolType::External => {
-                Err(ProcessInlinesError::ExternalToken(
-                    grammar.external_tokens[symbol.index].name.clone(),
-                ))?;
-            }
-            SymbolType::Terminal => {
-                Err(ProcessInlinesError::Token(
-                    lexical_grammar.variables[symbol.index].name.clone(),
-                ))?;
-            }
-            SymbolType::NonTerminal if symbol.index == 0 => {
-                Err(ProcessInlinesError::FirstRule(
-                    grammar.variables[symbol.index].name.clone(),
-                ))?;
-            }
-            _ => {}
-        }
-    }
-
-    Ok(InlinedProductionMapBuilder {
-        productions: Vec::new(),
-        production_indices_by_step_id: HashMap::new(),
-    }
-    .build(grammar))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::{LexicalVariable, SyntaxVariable, VariableType},
-        rules::{Associativity, Precedence, Symbol},
-    };
-
-    #[test]
-    fn test_basic_inlining() {
-        let grammar = SyntaxGrammar {
-            variables_to_inline: vec![Symbol::non_terminal(1)],
-            variables: vec![
-                SyntaxVariable {
-                    name: "non-terminal-0".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(10)),
-                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
-                            ProductionStep::new(Symbol::terminal(11)),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-1".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![
-                        Production {
-                            dynamic_precedence: 0,
-                            steps: vec![
-                                ProductionStep::new(Symbol::terminal(12)),
-                                ProductionStep::new(Symbol::terminal(13)),
-                            ],
-                        },
-                        Production {
-                            dynamic_precedence: -2,
-                            steps: vec![ProductionStep::new(Symbol::terminal(14))],
-                        },
-                    ],
-                },
-            ],
-            ..Default::default()
-        };
-
-        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
-
-        // Nothing to inline at step 0.
-        assert!(inline_map
-            .inlined_productions(&grammar.variables[0].productions[0], 0)
-            .is_none());
-
-        // Inlining variable 1 yields two productions.
-        assert_eq!(
-            inline_map
-                .inlined_productions(&grammar.variables[0].productions[0], 1)
-                .unwrap()
-                .cloned()
-                .collect::<Vec<_>>(),
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(12)),
-                        ProductionStep::new(Symbol::terminal(13)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                    ],
-                },
-                Production {
-                    dynamic_precedence: -2,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(14)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                    ],
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_nested_inlining() {
-        let grammar = SyntaxGrammar {
-            variables: vec![
-                SyntaxVariable {
-                    name: "non-terminal-0".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(10)),
-                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
-                            ProductionStep::new(Symbol::terminal(11)),
-                            ProductionStep::new(Symbol::non_terminal(2)), // inlined
-                            ProductionStep::new(Symbol::terminal(12)),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-1".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![
-                        Production {
-                            dynamic_precedence: 0,
-                            steps: vec![ProductionStep::new(Symbol::terminal(13))],
-                        },
-                        Production {
-                            dynamic_precedence: 0,
-                            steps: vec![
-                                ProductionStep::new(Symbol::non_terminal(3)), // inlined
-                                ProductionStep::new(Symbol::terminal(14)),
-                            ],
-                        },
-                    ],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-2".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![ProductionStep::new(Symbol::terminal(15))],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-3".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![ProductionStep::new(Symbol::terminal(16))],
-                    }],
-                },
-            ],
-            variables_to_inline: vec![
-                Symbol::non_terminal(1),
-                Symbol::non_terminal(2),
-                Symbol::non_terminal(3),
-            ],
-            ..Default::default()
-        };
-
-        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
-
-        let productions = inline_map
-            .inlined_productions(&grammar.variables[0].productions[0], 1)
-            .unwrap()
-            .collect::<Vec<_>>();
-
-        assert_eq!(
-            productions.iter().copied().cloned().collect::<Vec<_>>(),
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(13)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::terminal(12)),
-                    ],
-                },
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(16)),
-                        ProductionStep::new(Symbol::terminal(14)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::terminal(12)),
-                    ],
-                },
-            ]
-        );
-
-        assert_eq!(
-            inline_map
-                .inlined_productions(productions[0], 3)
-                .unwrap()
-                .cloned()
-                .collect::<Vec<_>>(),
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    ProductionStep::new(Symbol::terminal(10)),
-                    ProductionStep::new(Symbol::terminal(13)),
-                    ProductionStep::new(Symbol::terminal(11)),
-                    ProductionStep::new(Symbol::terminal(15)),
-                    ProductionStep::new(Symbol::terminal(12)),
-                ],
-            },]
-        );
-    }
-
-    #[test]
-    fn test_inlining_with_precedence_and_alias() {
-        let grammar = SyntaxGrammar {
-            variables_to_inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
-            variables: vec![
-                SyntaxVariable {
-                    name: "non-terminal-0".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            // inlined
-                            ProductionStep::new(Symbol::non_terminal(1))
-                                .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
-                            ProductionStep::new(Symbol::terminal(10)),
-                            // inlined
-                            ProductionStep::new(Symbol::non_terminal(2))
-                                .with_alias("outer_alias", true),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-1".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(11))
-                                .with_prec(Precedence::Integer(2), None)
-                                .with_alias("inner_alias", true),
-                            ProductionStep::new(Symbol::terminal(12)),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-2".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![ProductionStep::new(Symbol::terminal(13))],
-                    }],
-                },
-            ],
-            ..Default::default()
-        };
-
-        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
-
-        let productions = inline_map
-            .inlined_productions(&grammar.variables[0].productions[0], 0)
-            .unwrap()
-            .collect::<Vec<_>>();
-
-        assert_eq!(
-            productions.iter().copied().cloned().collect::<Vec<_>>(),
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    // The first step in the inlined production retains its precedence
-                    // and alias.
-                    ProductionStep::new(Symbol::terminal(11))
-                        .with_prec(Precedence::Integer(2), None)
-                        .with_alias("inner_alias", true),
-                    // The final step of the inlined production inherits the precedence of
-                    // the inlined step.
-                    ProductionStep::new(Symbol::terminal(12))
-                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
-                    ProductionStep::new(Symbol::terminal(10)),
-                    ProductionStep::new(Symbol::non_terminal(2)).with_alias("outer_alias", true),
-                ]
-            }],
-        );
-
-        assert_eq!(
-            inline_map
-                .inlined_productions(productions[0], 3)
-                .unwrap()
-                .cloned()
-                .collect::<Vec<_>>(),
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    ProductionStep::new(Symbol::terminal(11))
-                        .with_prec(Precedence::Integer(2), None)
-                        .with_alias("inner_alias", true),
-                    ProductionStep::new(Symbol::terminal(12))
-                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
-                    ProductionStep::new(Symbol::terminal(10)),
-                    // All steps of the inlined production inherit their alias from the
-                    // inlined step.
-                    ProductionStep::new(Symbol::terminal(13)).with_alias("outer_alias", true),
-                ]
-            }],
-        );
-    }
-
-    #[test]
-    fn test_error_when_inlining_tokens() {
-        let lexical_grammar = LexicalGrammar {
-            variables: vec![LexicalVariable {
-                name: "something".to_string(),
-                kind: VariableType::Named,
-                implicit_precedence: 0,
-                start_state: 0,
-            }],
-            ..Default::default()
-        };
-
-        let grammar = SyntaxGrammar {
-            variables_to_inline: vec![Symbol::terminal(0)],
-            variables: vec![SyntaxVariable {
-                name: "non-terminal-0".to_string(),
-                kind: VariableType::Named,
-                productions: vec![Production {
-                    dynamic_precedence: 0,
-                    steps: vec![ProductionStep::new(Symbol::terminal(0))],
-                }],
-            }],
-            ..Default::default()
-        };
-
-        if let Err(error) = process_inlines(&grammar, &lexical_grammar) {
-            assert_eq!(error.to_string(), "Token `something` cannot be inlined");
-        } else {
-            panic!("expected an error, but got none");
-        }
-    }
-}

cli/generate/src/rules.rs

@@ -1,544 +0,0 @@
-use std::{collections::HashMap, fmt};
-
-use serde::Serialize;
-use smallbitvec::SmallBitVec;
-
-use super::grammars::VariableType;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
-pub enum SymbolType {
-    External,
-    End,
-    EndOfNonTerminalExtra,
-    Terminal,
-    NonTerminal,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
-pub enum Associativity {
-    Left,
-    Right,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
-pub struct Alias {
-    pub value: String,
-    pub is_named: bool,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default, Serialize)]
-pub enum Precedence {
-    #[default]
-    None,
-    Integer(i32),
-    Name(String),
-}
-
-pub type AliasMap = HashMap<Symbol, Alias>;
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize)]
-pub struct MetadataParams {
-    pub precedence: Precedence,
-    pub dynamic_precedence: i32,
-    pub associativity: Option<Associativity>,
-    pub is_token: bool,
-    pub is_main_token: bool,
-    pub alias: Option<Alias>,
-    pub field_name: Option<String>,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
-pub struct Symbol {
-    pub kind: SymbolType,
-    pub index: usize,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
-pub enum Rule {
-    Blank,
-    String(String),
-    Pattern(String, String),
-    NamedSymbol(String),
-    Symbol(Symbol),
-    Choice(Vec<Rule>),
-    Metadata {
-        params: MetadataParams,
-        rule: Box<Rule>,
-    },
-    Repeat(Box<Rule>),
-    Seq(Vec<Rule>),
-    Reserved {
-        rule: Box<Rule>,
-        context_name: String,
-    },
-}
-
-// Because tokens are represented as small (~400 max) unsigned integers,
-// sets of tokens can be efficiently represented as bit vectors with each
-// index corresponding to a token, and each value representing whether or not
-// the token is present in the set.
-#[derive(Default, Clone, PartialEq, Eq, Hash)]
-pub struct TokenSet {
-    terminal_bits: SmallBitVec,
-    external_bits: SmallBitVec,
-    eof: bool,
-    end_of_nonterminal_extra: bool,
-}
-
-impl fmt::Debug for TokenSet {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.iter()).finish()
-    }
-}
-
-impl PartialOrd for TokenSet {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for TokenSet {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.terminal_bits
-            .iter()
-            .cmp(other.terminal_bits.iter())
-            .then_with(|| self.external_bits.iter().cmp(other.external_bits.iter()))
-            .then_with(|| self.eof.cmp(&other.eof))
-            .then_with(|| {
-                self.end_of_nonterminal_extra
-                    .cmp(&other.end_of_nonterminal_extra)
-            })
-    }
-}
-
-impl Rule {
-    pub fn field(name: String, content: Self) -> Self {
-        add_metadata(content, move |params| {
-            params.field_name = Some(name);
-        })
-    }
-
-    pub fn alias(content: Self, value: String, is_named: bool) -> Self {
-        add_metadata(content, move |params| {
-            params.alias = Some(Alias { value, is_named });
-        })
-    }
-
-    pub fn token(content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.is_token = true;
-        })
-    }
-
-    pub fn immediate_token(content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.is_token = true;
-            params.is_main_token = true;
-        })
-    }
-
-    pub fn prec(value: Precedence, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.precedence = value;
-        })
-    }
-
-    pub fn prec_left(value: Precedence, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.associativity = Some(Associativity::Left);
-            params.precedence = value;
-        })
-    }
-
-    pub fn prec_right(value: Precedence, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.associativity = Some(Associativity::Right);
-            params.precedence = value;
-        })
-    }
-
-    pub fn prec_dynamic(value: i32, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.dynamic_precedence = value;
-        })
-    }
-
-    pub fn repeat(rule: Self) -> Self {
-        Self::Repeat(Box::new(rule))
-    }
-
-    pub fn choice(rules: Vec<Self>) -> Self {
-        let mut elements = Vec::with_capacity(rules.len());
-        for rule in rules {
-            choice_helper(&mut elements, rule);
-        }
-        Self::Choice(elements)
-    }
-
-    pub const fn seq(rules: Vec<Self>) -> Self {
-        Self::Seq(rules)
-    }
-
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Blank | Self::Pattern(..) | Self::NamedSymbol(_) | Self::Symbol(_) => false,
-            Self::String(string) => string.is_empty(),
-            Self::Metadata { rule, .. } | Self::Repeat(rule) | Self::Reserved { rule, .. } => {
-                rule.is_empty()
-            }
-            Self::Choice(rules) => rules.iter().any(Self::is_empty),
-            Self::Seq(rules) => rules.iter().all(Self::is_empty),
-        }
-    }
-}
-
-impl Alias {
-    #[must_use]
-    pub const fn kind(&self) -> VariableType {
-        if self.is_named {
-            VariableType::Named
-        } else {
-            VariableType::Anonymous
-        }
-    }
-}
-
-impl Precedence {
-    #[must_use]
-    pub const fn is_none(&self) -> bool {
-        matches!(self, Self::None)
-    }
-}
-
-#[cfg(test)]
-impl Rule {
-    #[must_use]
-    pub const fn terminal(index: usize) -> Self {
-        Self::Symbol(Symbol::terminal(index))
-    }
-
-    #[must_use]
-    pub const fn non_terminal(index: usize) -> Self {
-        Self::Symbol(Symbol::non_terminal(index))
-    }
-
-    #[must_use]
-    pub const fn external(index: usize) -> Self {
-        Self::Symbol(Symbol::external(index))
-    }
-
-    #[must_use]
-    pub fn named(name: &'static str) -> Self {
-        Self::NamedSymbol(name.to_string())
-    }
-
-    #[must_use]
-    pub fn string(value: &'static str) -> Self {
-        Self::String(value.to_string())
-    }
-
-    #[must_use]
-    pub fn pattern(value: &'static str, flags: &'static str) -> Self {
-        Self::Pattern(value.to_string(), flags.to_string())
-    }
-}
-
-impl Symbol {
-    #[must_use]
-    pub fn is_terminal(&self) -> bool {
-        self.kind == SymbolType::Terminal
-    }
-
-    #[must_use]
-    pub fn is_non_terminal(&self) -> bool {
-        self.kind == SymbolType::NonTerminal
-    }
-
-    #[must_use]
-    pub fn is_external(&self) -> bool {
-        self.kind == SymbolType::External
-    }
-
-    #[must_use]
-    pub fn is_eof(&self) -> bool {
-        self.kind == SymbolType::End
-    }
-
-    #[must_use]
-    pub const fn non_terminal(index: usize) -> Self {
-        Self {
-            kind: SymbolType::NonTerminal,
-            index,
-        }
-    }
-
-    #[must_use]
-    pub const fn terminal(index: usize) -> Self {
-        Self {
-            kind: SymbolType::Terminal,
-            index,
-        }
-    }
-
-    #[must_use]
-    pub const fn external(index: usize) -> Self {
-        Self {
-            kind: SymbolType::External,
-            index,
-        }
-    }
-
-    #[must_use]
-    pub const fn end() -> Self {
-        Self {
-            kind: SymbolType::End,
-            index: 0,
-        }
-    }
-
-    #[must_use]
-    pub const fn end_of_nonterminal_extra() -> Self {
-        Self {
-            kind: SymbolType::EndOfNonTerminalExtra,
-            index: 0,
-        }
-    }
-}
-
-impl From<Symbol> for Rule {
-    #[must_use]
-    fn from(symbol: Symbol) -> Self {
-        Self::Symbol(symbol)
-    }
-}
-
-impl TokenSet {
-    #[must_use]
-    pub const fn new() -> Self {
-        Self {
-            terminal_bits: SmallBitVec::new(),
-            external_bits: SmallBitVec::new(),
-            eof: false,
-            end_of_nonterminal_extra: false,
-        }
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = Symbol> + '_ {
-        self.terminal_bits
-            .iter()
-            .enumerate()
-            .filter_map(|(i, value)| {
-                if value {
-                    Some(Symbol::terminal(i))
-                } else {
-                    None
-                }
-            })
-            .chain(
-                self.external_bits
-                    .iter()
-                    .enumerate()
-                    .filter_map(|(i, value)| {
-                        if value {
-                            Some(Symbol::external(i))
-                        } else {
-                            None
-                        }
-                    }),
-            )
-            .chain(if self.eof { Some(Symbol::end()) } else { None })
-            .chain(if self.end_of_nonterminal_extra {
-                Some(Symbol::end_of_nonterminal_extra())
-            } else {
-                None
-            })
-    }
-
-    pub fn terminals(&self) -> impl Iterator<Item = Symbol> + '_ {
-        self.terminal_bits
-            .iter()
-            .enumerate()
-            .filter_map(|(i, value)| {
-                if value {
-                    Some(Symbol::terminal(i))
-                } else {
-                    None
-                }
-            })
-    }
-
-    pub fn contains(&self, symbol: &Symbol) -> bool {
-        match symbol.kind {
-            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
-            SymbolType::Terminal => self.terminal_bits.get(symbol.index).unwrap_or(false),
-            SymbolType::External => self.external_bits.get(symbol.index).unwrap_or(false),
-            SymbolType::End => self.eof,
-            SymbolType::EndOfNonTerminalExtra => self.end_of_nonterminal_extra,
-        }
-    }
-
-    pub fn contains_terminal(&self, index: usize) -> bool {
-        self.terminal_bits.get(index).unwrap_or(false)
-    }
-
-    pub fn insert(&mut self, other: Symbol) {
-        let vec = match other.kind {
-            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
-            SymbolType::Terminal => &mut self.terminal_bits,
-            SymbolType::External => &mut self.external_bits,
-            SymbolType::End => {
-                self.eof = true;
-                return;
-            }
-            SymbolType::EndOfNonTerminalExtra => {
-                self.end_of_nonterminal_extra = true;
-                return;
-            }
-        };
-        if other.index >= vec.len() {
-            vec.resize(other.index + 1, false);
-        }
-        vec.set(other.index, true);
-    }
-
-    pub fn remove(&mut self, other: &Symbol) -> bool {
-        let vec = match other.kind {
-            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
-            SymbolType::Terminal => &mut self.terminal_bits,
-            SymbolType::External => &mut self.external_bits,
-            SymbolType::End => {
-                return if self.eof {
-                    self.eof = false;
-                    true
-                } else {
-                    false
-                }
-            }
-            SymbolType::EndOfNonTerminalExtra => {
-                return if self.end_of_nonterminal_extra {
-                    self.end_of_nonterminal_extra = false;
-                    true
-                } else {
-                    false
-                };
-            }
-        };
-        if other.index < vec.len() && vec[other.index] {
-            vec.set(other.index, false);
-            while vec.last() == Some(false) {
-                vec.pop();
-            }
-            return true;
-        }
-        false
-    }
-
-    pub fn is_empty(&self) -> bool {
-        !self.eof
-            && !self.end_of_nonterminal_extra
-            && !self.terminal_bits.iter().any(|a| a)
-            && !self.external_bits.iter().any(|a| a)
-    }
-
-    pub fn len(&self) -> usize {
-        self.eof as usize
-            + self.end_of_nonterminal_extra as usize
-            + self.terminal_bits.iter().filter(|b| *b).count()
-            + self.external_bits.iter().filter(|b| *b).count()
-    }
-
-    pub fn insert_all_terminals(&mut self, other: &Self) -> bool {
-        let mut result = false;
-        if other.terminal_bits.len() > self.terminal_bits.len() {
-            self.terminal_bits.resize(other.terminal_bits.len(), false);
-        }
-        for (i, element) in other.terminal_bits.iter().enumerate() {
-            if element {
-                result |= !self.terminal_bits[i];
-                self.terminal_bits.set(i, element);
-            }
-        }
-        result
-    }
-
-    fn insert_all_externals(&mut self, other: &Self) -> bool {
-        let mut result = false;
-        if other.external_bits.len() > self.external_bits.len() {
-            self.external_bits.resize(other.external_bits.len(), false);
-        }
-        for (i, element) in other.external_bits.iter().enumerate() {
-            if element {
-                result |= !self.external_bits[i];
-                self.external_bits.set(i, element);
-            }
-        }
-        result
-    }
-
-    pub fn insert_all(&mut self, other: &Self) -> bool {
-        let mut result = false;
-        if other.eof {
-            result |= !self.eof;
-            self.eof = true;
-        }
-        if other.end_of_nonterminal_extra {
-            result |= !self.end_of_nonterminal_extra;
-            self.end_of_nonterminal_extra = true;
-        }
-        result |= self.insert_all_terminals(other);
-        result |= self.insert_all_externals(other);
-        result
-    }
-}
-
-impl FromIterator<Symbol> for TokenSet {
-    fn from_iter<T: IntoIterator<Item = Symbol>>(iter: T) -> Self {
-        let mut result = Self::new();
-        for symbol in iter {
-            result.insert(symbol);
-        }
-        result
-    }
-}
-
-fn add_metadata<T: FnOnce(&mut MetadataParams)>(input: Rule, f: T) -> Rule {
-    match input {
-        Rule::Metadata { rule, mut params } if !params.is_token => {
-            f(&mut params);
-            Rule::Metadata { rule, params }
-        }
-        _ => {
-            let mut params = MetadataParams::default();
-            f(&mut params);
-            Rule::Metadata {
-                rule: Box::new(input),
-                params,
-            }
-        }
-    }
-}
-
-fn choice_helper(result: &mut Vec<Rule>, rule: Rule) {
-    match rule {
-        Rule::Choice(elements) => {
-            for element in elements {
-                choice_helper(result, element);
-            }
-        }
-        _ => {
-            if !result.contains(&rule) {
-                result.push(rule);
-            }
-        }
-    }
-}
-
-impl fmt::Display for Precedence {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Integer(i) => write!(f, "{i}"),
-            Self::Name(s) => write!(f, "'{s}'"),
-            Self::None => write!(f, "none"),
-        }
-    }
-}

cli/generate/src/tables.rs

@@ -1,167 +0,0 @@
-use std::collections::BTreeMap;
-
-use super::{
-    nfa::CharacterSet,
-    rules::{Alias, Symbol, TokenSet},
-};
-pub type ProductionInfoId = usize;
-pub type ParseStateId = usize;
-pub type LexStateId = usize;
-
-use std::hash::BuildHasherDefault;
-
-use indexmap::IndexMap;
-use rustc_hash::FxHasher;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum ParseAction {
-    Accept,
-    Shift {
-        state: ParseStateId,
-        is_repetition: bool,
-    },
-    ShiftExtra,
-    Recover,
-    Reduce {
-        symbol: Symbol,
-        child_count: usize,
-        dynamic_precedence: i32,
-        production_id: ProductionInfoId,
-    },
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum GotoAction {
-    Goto(ParseStateId),
-    ShiftExtra,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct ParseTableEntry {
-    pub actions: Vec<ParseAction>,
-    pub reusable: bool,
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct ParseState {
-    pub id: ParseStateId,
-    pub terminal_entries: IndexMap<Symbol, ParseTableEntry, BuildHasherDefault<FxHasher>>,
-    pub nonterminal_entries: IndexMap<Symbol, GotoAction, BuildHasherDefault<FxHasher>>,
-    pub reserved_words: TokenSet,
-    pub lex_state_id: usize,
-    pub external_lex_state_id: usize,
-    pub core_id: usize,
-}
-
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct FieldLocation {
-    pub index: usize,
-    pub inherited: bool,
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct ProductionInfo {
-    pub alias_sequence: Vec<Option<Alias>>,
-    pub field_map: BTreeMap<String, Vec<FieldLocation>>,
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct ParseTable {
-    pub states: Vec<ParseState>,
-    pub symbols: Vec<Symbol>,
-    pub production_infos: Vec<ProductionInfo>,
-    pub max_aliased_production_length: usize,
-    pub external_lex_states: Vec<TokenSet>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub struct AdvanceAction {
-    pub state: LexStateId,
-    pub in_main_token: bool,
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
-pub struct LexState {
-    pub accept_action: Option<Symbol>,
-    pub eof_action: Option<AdvanceAction>,
-    pub advance_actions: Vec<(CharacterSet, AdvanceAction)>,
-}
-
-#[derive(Debug, PartialEq, Eq, Default)]
-pub struct LexTable {
-    pub states: Vec<LexState>,
-}
-
-impl ParseTableEntry {
-    #[must_use]
-    pub const fn new() -> Self {
-        Self {
-            reusable: true,
-            actions: Vec::new(),
-        }
-    }
-}
-
-impl ParseState {
-    pub fn is_end_of_non_terminal_extra(&self) -> bool {
-        self.terminal_entries
-            .contains_key(&Symbol::end_of_nonterminal_extra())
-    }
-
-    pub fn referenced_states(&self) -> impl Iterator<Item = ParseStateId> + '_ {
-        self.terminal_entries
-            .iter()
-            .flat_map(|(_, entry)| {
-                entry.actions.iter().filter_map(|action| match action {
-                    ParseAction::Shift { state, .. } => Some(*state),
-                    _ => None,
-                })
-            })
-            .chain(self.nonterminal_entries.iter().filter_map(|(_, action)| {
-                if let GotoAction::Goto(state) = action {
-                    Some(*state)
-                } else {
-                    None
-                }
-            }))
-    }
-
-    pub fn update_referenced_states<F>(&mut self, mut f: F)
-    where
-        F: FnMut(usize, &Self) -> usize,
-    {
-        let mut updates = Vec::new();
-        for (symbol, entry) in &self.terminal_entries {
-            for (i, action) in entry.actions.iter().enumerate() {
-                if let ParseAction::Shift { state, .. } = action {
-                    let result = f(*state, self);
-                    if result != *state {
-                        updates.push((*symbol, i, result));
-                    }
-                }
-            }
-        }
-        for (symbol, action) in &self.nonterminal_entries {
-            if let GotoAction::Goto(other_state) = action {
-                let result = f(*other_state, self);
-                if result != *other_state {
-                    updates.push((*symbol, 0, result));
-                }
-            }
-        }
-        for (symbol, action_index, new_state) in updates {
-            if symbol.is_non_terminal() {
-                self.nonterminal_entries
-                    .insert(symbol, GotoAction::Goto(new_state));
-            } else {
-                let entry = self.terminal_entries.get_mut(&symbol).unwrap();
-                if let ParseAction::Shift { is_repetition, .. } = entry.actions[action_index] {
-                    entry.actions[action_index] = ParseAction::Shift {
-                        state: new_state,
-                        is_repetition,
-                    };
-                }
-            }
-        }
-    }
-}

cli/loader/emscripten-version

@@ -1 +0,0 @@
-4.0.1

cli/src/logger.rs

@@ -1,30 +0,0 @@
-use log::{LevelFilter, Log, Metadata, Record};
-
-#[allow(dead_code)]
-struct Logger {
-    pub filter: Option<String>,
-}
-
-impl Log for Logger {
-    fn enabled(&self, _: &Metadata) -> bool {
-        true
-    }
-
-    fn log(&self, record: &Record) {
-        eprintln!(
-            "[{}] {}",
-            record
-                .module_path()
-                .unwrap_or_default()
-                .trim_start_matches("rust_tree_sitter_cli::"),
-            record.args()
-        );
-    }
-
-    fn flush(&self) {}
-}
-
-pub fn init() {
-    log::set_boxed_logger(Box::new(Logger { filter: None })).unwrap();
-    log::set_max_level(LevelFilter::Info);
-}

cli/src/playground.html

@@ -1,410 +0,0 @@
-<head>
-  <meta charset="utf-8">
-  <title>tree-sitter THE_LANGUAGE_NAME</title>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.css">
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.19.0/clusterize.min.css">
-  <link rel="icon" type="image/png" href="https://tree-sitter.github.io/tree-sitter/assets/images/favicon-32x32.png"
-    sizes="32x32" />
-  <link rel="icon" type="image/png" href="https://tree-sitter.github.io/tree-sitter/assets/images/favicon-16x16.png"
-    sizes="16x16" />
-</head>
-
-<body>
-  <div id="playground-container" style="visibility: hidden;">
-    <header>
-      <div class="header-item">
-        <span class="language-name">Language: THE_LANGUAGE_NAME</span>
-      </div>
-
-      <div class="header-item">
-        <input id="logging-checkbox" type="checkbox">
-        <label for="logging-checkbox">log</label>
-      </div>
-
-      <div class="header-item">
-        <input id="anonymous-nodes-checkbox" type="checkbox">
-        <label for="anonymous-nodes-checkbox">show anonymous nodes</label>
-      </div>
-
-      <div class="header-item">
-        <input id="query-checkbox" type="checkbox">
-        <label for="query-checkbox">query</label>
-      </div>
-
-      <div class="header-item">
-        <input id="accessibility-checkbox" type="checkbox">
-        <label for="accessibility-checkbox">accessibility</label>
-      </div>
-
-      <div class="header-item">
-        <label for="update-time">parse time: </label>
-        <span id="update-time"></span>
-      </div>
-
-      <div class="header-item">
-        <a href="https://tree-sitter.github.io/tree-sitter/7-playground.html#about">(?)</a>
-      </div>
-
-      <select id="language-select" style="display: none;">
-        <option value="parser">Parser</option>
-      </select>
-
-      <div class="header-item">
-        <button id="theme-toggle" class="theme-toggle" aria-label="Toggle theme">
-          <svg class="sun-icon" viewBox="0 0 24 24" width="16" height="16">
-            <path fill="currentColor"
-              d="M12 17.5a5.5 5.5 0 1 0 0-11 5.5 5.5 0 0 0 0 11zm0 1.5a7 7 0 1 1 0-14 7 7 0 0 1 0 14zm0-16a1 1 0 0 1 1 1v2a1 1 0 1 1-2 0V4a1 1 0 0 1 1-1zm0 15a1 1 0 0 1 1 1v2a1 1 0 1 1-2 0v-2a1 1 0 0 1 1-1zm9-9a1 1 0 0 1-1 1h-2a1 1 0 1 1 0-2h2a1 1 0 0 1 1 1zM4 12a1 1 0 0 1-1 1H1a1 1 0 1 1 0-2h2a1 1 0 0 1 1 1z" />
-          </svg>
-          <svg class="moon-icon" viewBox="0 0 24 24" width="16" height="16">
-            <path fill="currentColor"
-              d="M12.1 22c-5.5 0-10-4.5-10-10s4.5-10 10-10c.2 0 .3 0 .5.1-1.3 1.4-2 3.2-2 5.2 0 4.1 3.4 7.5 7.5 7.5 2 0 3.8-.7 5.2-2 .1.2.1.3.1.5 0 5.4-4.5 9.7-10 9.7z" />
-          </svg>
-        </button>
-      </div>
-    </header>
-
-    <main>
-      <div id="input-pane">
-        <div class="panel-header">Code</div>
-        <div id="code-container">
-          <textarea id="code-input"></textarea>
-        </div>
-
-        <div id="query-container" style="visibility: hidden; position: absolute;">
-          <div class="panel-header">Query</div>
-          <textarea id="query-input"></textarea>
-        </div>
-      </div>
-
-      <div id="output-container-scroll">
-        <div class="panel-header">Tree</div>
-        <pre id="output-container" class="highlight"></pre>
-      </div>
-    </main>
-  </div>
-
-  <script src="https://code.jquery.com/jquery-3.3.1.min.js" crossorigin="anonymous">
-  </script>
-
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.js"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.19.0/clusterize.min.js"></script>
-
-  <script>LANGUAGE_BASE_URL = "";</script>
-  <script type="module" src="playground.js"></script>
-  <script type="module">
-    import * as TreeSitter from './tree-sitter.js';
-    window.TreeSitter = TreeSitter;
-    setTimeout(() => window.initializePlayground({local: true}), 1)
-  </script>
-
-  <style>
-    /* Base Variables */
-    :root {
-      --light-bg: #f9f9f9;
-      --light-border: #e0e0e0;
-      --light-text: #333;
-      --light-hover-border: #c1c1c1;
-      --light-scrollbar-track: #f1f1f1;
-      --light-scrollbar-thumb: #c1c1c1;
-      --light-scrollbar-thumb-hover: #a8a8a8;
-
-      --dark-bg: #1d1f21;
-      --dark-border: #2d2d2d;
-      --dark-text: #c5c8c6;
-      --dark-panel-bg: #252526;
-      --dark-code-bg: #1e1e1e;
-      --dark-scrollbar-track: #25282c;
-      --dark-scrollbar-thumb: #4a4d51;
-      --dark-scrollbar-thumb-hover: #5a5d61;
-
-      --primary-color: #0550ae;
-      --primary-color-alpha: rgba(5, 80, 174, 0.1);
-      --primary-color-alpha-dark: rgba(121, 192, 255, 0.1);
-      --selection-color: rgba(39, 95, 255, 0.3);
-    }
-
-    /* Theme Colors */
-    [data-theme="dark"] {
-      --bg-color: var(--dark-bg);
-      --border-color: var(--dark-border);
-      --text-color: var(--dark-text);
-      --panel-bg: var(--dark-panel-bg);
-      --code-bg: var(--dark-code-bg);
-    }
-
-    [data-theme="light"] {
-      --bg-color: var(--light-bg);
-      --border-color: var(--light-border);
-      --text-color: var(--light-text);
-      --panel-bg: white;
-      --code-bg: white;
-    }
-
-    /* Base Styles */
-    body {
-      margin: 0;
-      padding: 0;
-      font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-      background-color: var(--bg-color);
-      color: var(--text-color);
-    }
-
-    /* Layout */
-    #playground-container {
-      width: 100%;
-      height: 100vh;
-      display: flex;
-      flex-direction: column;
-      background-color: var(--bg-color);
-    }
-
-    header {
-      padding: 16px 24px;
-      border-bottom: 1px solid var(--border-color);
-      display: flex;
-      align-items: center;
-      gap: 20px;
-      background-color: var(--panel-bg);
-      font-size: 14px;
-    }
-
-    .header-item {
-      display: flex;
-      align-items: center;
-      gap: 8px;
-    }
-
-    .language-name {
-      font-weight: 600;
-    }
-
-    main {
-      flex: 1;
-      display: flex;
-      overflow: hidden;
-    }
-
-    #input-pane {
-      width: 50%;
-      display: flex;
-      flex-direction: column;
-      border-right: 1px solid var(--border-color);
-      background-color: var(--panel-bg);
-      overflow: hidden;
-    }
-
-    #code-container {
-      flex: 1;
-      min-height: 0;
-      position: relative;
-      border-bottom: 1px solid var(--border-color);
-      display: flex;
-      flex-direction: column;
-    }
-
-    #query-container:not([style*="visibility: hidden"]) {
-      flex: 1;
-      min-height: 0;
-      display: flex;
-      flex-direction: column;
-    }
-
-    #query-container .panel-header {
-      flex: 0 0 auto;
-    }
-
-    #query-container .CodeMirror {
-      flex: 1;
-      position: relative;
-      min-height: 0;
-    }
-
-    #output-container-scroll {
-      width: 50%;
-      overflow: auto;
-      background-color: var(--panel-bg);
-      padding: 0;
-      display: flex;
-      flex-direction: column;
-    }
-
-    #output-container {
-      font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
-      line-height: 1.5;
-      margin: 0;
-      padding: 16px;
-    }
-
-    .panel-header {
-      padding: 8px 16px;
-      font-weight: 600;
-      font-size: 14px;
-      border-bottom: 1px solid var(--border-color);
-      background-color: var(--panel-bg);
-    }
-
-    .CodeMirror {
-      position: absolute;
-      top: 0;
-      left: 0;
-      right: 0;
-      bottom: 0;
-      height: 100%;
-      font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
-      font-size: 14px;
-      line-height: 1.6;
-      background-color: var(--code-bg) !important;
-      color: var(--text-color) !important;
-    }
-
-    .query-error {
-      text-decoration: underline red dashed;
-      -webkit-text-decoration: underline red dashed;
-    }
-
-    /* Scrollbars */
-    ::-webkit-scrollbar {
-      width: 8px;
-      height: 8px;
-    }
-
-    ::-webkit-scrollbar-track {
-      border-radius: 4px;
-      background: var(--light-scrollbar-track);
-    }
-
-    ::-webkit-scrollbar-thumb {
-      border-radius: 4px;
-      background: var(--light-scrollbar-thumb);
-    }
-
-    ::-webkit-scrollbar-thumb:hover {
-      background: var(--light-scrollbar-thumb-hover);
-    }
-
-    [data-theme="dark"] {
-      ::-webkit-scrollbar-track {
-        background: var(--dark-scrollbar-track) !important;
-      }
-
-      ::-webkit-scrollbar-thumb {
-        background: var(--dark-scrollbar-thumb) !important;
-      }
-
-      ::-webkit-scrollbar-thumb:hover {
-        background: var(--dark-scrollbar-thumb-hover) !important;
-      }
-    }
-
-    /* Theme Toggle */
-    .theme-toggle {
-      background: none;
-      border: 1px solid var(--border-color);
-      border-radius: 4px;
-      padding: 6px;
-      cursor: pointer;
-      color: var(--text-color);
-    }
-
-    .theme-toggle:hover {
-      background-color: var(--primary-color-alpha);
-    }
-
-    [data-theme="light"] .moon-icon,
-    [data-theme="dark"] .sun-icon {
-      display: none;
-    }
-
-    /* Form Elements */
-    input[type="checkbox"] {
-      margin-right: 6px;
-      vertical-align: middle;
-    }
-
-    label {
-      font-size: 14px;
-      margin-right: 16px;
-      cursor: pointer;
-    }
-
-    #output-container a {
-      cursor: pointer;
-      text-decoration: none;
-      color: #040404;
-      padding: 2px;
-    }
-
-    #output-container a:hover {
-      text-decoration: underline;
-    }
-
-    #output-container a.node-link.named {
-      color: #0550ae;
-    }
-
-    #output-container a.node-link.anonymous {
-      color: #116329;
-    }
-
-    #output-container a.node-link.anonymous:before {
-      content: '"';
-    }
-
-    #output-container a.node-link.anonymous:after {
-      content: '"';
-    }
-
-    #output-container a.node-link.error {
-      color: #cf222e;
-    }
-
-    #output-container a.highlighted {
-      background-color: #d9d9d9;
-      color: red;
-      border-radius: 3px;
-      text-decoration: underline;
-    }
-
-    /* Dark Theme Node Colors */
-    [data-theme="dark"] {
-      & #output-container a {
-        color: #d4d4d4;
-      }
-
-      & #output-container a.node-link.named {
-        color: #79c0ff;
-      }
-
-      & #output-container a.node-link.anonymous {
-        color: #7ee787;
-      }
-
-      & #output-container a.node-link.error {
-        color: #ff7b72;
-      }
-
-      & #output-container a.highlighted {
-        background-color: #373b41;
-        color: red;
-      }
-
-      & .CodeMirror {
-        background-color: var(--dark-code-bg) !important;
-        color: var(--dark-text) !important;
-      }
-
-      & .CodeMirror-gutters {
-        background-color: var(--dark-panel-bg) !important;
-        border-color: var(--dark-border) !important;
-      }
-
-      & .CodeMirror-cursor {
-        border-color: var(--dark-text) !important;
-      }
-
-      & .CodeMirror-selected {
-        background-color: rgba(255, 255, 255, 0.1) !important;
-      }
-    }
-  </style>
-</body>

cli/src/query.rs

@@ -1,148 +0,0 @@
-use std::{
-    fs,
-    io::{self, Write},
-    ops::Range,
-    path::Path,
-    time::Instant,
-};
-
-use anstyle::AnsiColor;
-use anyhow::{Context, Result};
-use streaming_iterator::StreamingIterator;
-use tree_sitter::{Language, Parser, Point, Query, QueryCursor};
-
-use crate::{
-    query_testing::{self, to_utf8_point},
-    test::paint,
-};
-
-#[allow(clippy::too_many_arguments)]
-pub fn query_file_at_path(
-    language: &Language,
-    path: &Path,
-    name: &str,
-    query_path: &Path,
-    ordered_captures: bool,
-    byte_range: Option<Range<usize>>,
-    point_range: Option<Range<Point>>,
-    should_test: bool,
-    quiet: bool,
-    print_time: bool,
-    stdin: bool,
-) -> Result<()> {
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    let query_source = fs::read_to_string(query_path)
-        .with_context(|| format!("Error reading query file {query_path:?}"))?;
-    let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?;
-
-    let mut query_cursor = QueryCursor::new();
-    if let Some(range) = byte_range {
-        query_cursor.set_byte_range(range);
-    }
-    if let Some(range) = point_range {
-        query_cursor.set_point_range(range);
-    }
-
-    let mut parser = Parser::new();
-    parser.set_language(language)?;
-
-    let mut results = Vec::new();
-
-    if !should_test && !stdin {
-        writeln!(&mut stdout, "{name}")?;
-    }
-
-    let source_code =
-        fs::read(path).with_context(|| format!("Error reading source file {path:?}"))?;
-    let tree = parser.parse(&source_code, None).unwrap();
-
-    let start = Instant::now();
-    if ordered_captures {
-        let mut captures = query_cursor.captures(&query, tree.root_node(), source_code.as_slice());
-        while let Some((mat, capture_index)) = captures.next() {
-            let capture = mat.captures[*capture_index];
-            let capture_name = &query.capture_names()[capture.index as usize];
-            if !quiet && !should_test {
-                writeln!(
-                        &mut stdout,
-                        "    pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`",
-                        mat.pattern_index,
-                        capture.index,
-                        capture.node.start_position(),
-                        capture.node.end_position(),
-                        capture.node.utf8_text(&source_code).unwrap_or("")
-                    )?;
-            }
-            results.push(query_testing::CaptureInfo {
-                name: (*capture_name).to_string(),
-                start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
-                end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
-            });
-        }
-    } else {
-        let mut matches = query_cursor.matches(&query, tree.root_node(), source_code.as_slice());
-        while let Some(m) = matches.next() {
-            if !quiet && !should_test {
-                writeln!(&mut stdout, "  pattern: {}", m.pattern_index)?;
-            }
-            for capture in m.captures {
-                let start = capture.node.start_position();
-                let end = capture.node.end_position();
-                let capture_name = &query.capture_names()[capture.index as usize];
-                if !quiet && !should_test {
-                    if end.row == start.row {
-                        writeln!(
-                                &mut stdout,
-                                "    capture: {} - {capture_name}, start: {start}, end: {end}, text: `{}`",
-                                capture.index,
-                                capture.node.utf8_text(&source_code).unwrap_or("")
-                            )?;
-                    } else {
-                        writeln!(
-                            &mut stdout,
-                            "    capture: {capture_name}, start: {start}, end: {end}",
-                        )?;
-                    }
-                }
-                results.push(query_testing::CaptureInfo {
-                    name: (*capture_name).to_string(),
-                    start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
-                    end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
-                });
-            }
-        }
-    }
-    if query_cursor.did_exceed_match_limit() {
-        writeln!(
-            &mut stdout,
-            "  WARNING: Query exceeded maximum number of in-progress captures!"
-        )?;
-    }
-    if should_test {
-        let path_name = if stdin {
-            "stdin"
-        } else {
-            Path::new(&path).file_name().unwrap().to_str().unwrap()
-        };
-        match query_testing::assert_expected_captures(&results, path, &mut parser, language) {
-            Ok(assertion_count) => {
-                println!(
-                    "  ✓ {} ({} assertions)",
-                    paint(Some(AnsiColor::Green), path_name),
-                    assertion_count
-                );
-            }
-            Err(e) => {
-                println!("  ✗ {}", paint(Some(AnsiColor::Red), path_name));
-                return Err(e);
-            }
-        }
-    }
-    if print_time {
-        writeln!(&mut stdout, "{:?}", start.elapsed())?;
-    }
-
-    Ok(())
-}

cli/src/templates/__init__.py

@@ -1,42 +0,0 @@
-"""PARSER_DESCRIPTION"""
-
-from importlib.resources import files as _files
-
-from ._binding import language
-
-
-def _get_query(name, file):
-    query = _files(f"{__package__}.queries") / file
-    globals()[name] = query.read_text()
-    return globals()[name]
-
-
-def __getattr__(name):
-    # NOTE: uncomment these to include any queries that this grammar contains:
-
-    # if name == "HIGHLIGHTS_QUERY":
-    #     return _get_query("HIGHLIGHTS_QUERY", "highlights.scm")
-    # if name == "INJECTIONS_QUERY":
-    #     return _get_query("INJECTIONS_QUERY", "injections.scm")
-    # if name == "LOCALS_QUERY":
-    #     return _get_query("LOCALS_QUERY", "locals.scm")
-    # if name == "TAGS_QUERY":
-    #     return _get_query("TAGS_QUERY", "tags.scm")
-
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
-__all__ = [
-    "language",
-    # "HIGHLIGHTS_QUERY",
-    # "INJECTIONS_QUERY",
-    # "LOCALS_QUERY",
-    # "TAGS_QUERY",
-]
-
-
-def __dir__():
-    return sorted(__all__ + [
-        "__all__", "__builtins__", "__cached__", "__doc__", "__file__",
-        "__loader__", "__name__", "__package__", "__path__", "__spec__",
-    ])

cli/src/templates/__init__.pyi

@@ -1,10 +0,0 @@
-from typing import Final
-
-# NOTE: uncomment these to include any queries that this grammar contains:
-
-# HIGHLIGHTS_QUERY: Final[str]
-# INJECTIONS_QUERY: Final[str]
-# LOCALS_QUERY: Final[str]
-# TAGS_QUERY: Final[str]
-
-def language() -> object: ...

cli/src/templates/binding_test.js

@@ -1,9 +0,0 @@
-const assert = require("node:assert");
-const { test } = require("node:test");
-
-const Parser = require("tree-sitter");
-
-test("can load grammar", () => {
-  const parser = new Parser();
-  assert.doesNotThrow(() => parser.setLanguage(require(".")));
-});

cli/src/templates/build.rs

@@ -1,21 +0,0 @@
-fn main() {
-    let src_dir = std::path::Path::new("src");
-
-    let mut c_config = cc::Build::new();
-    c_config.std("c11").include(src_dir);
-
-    #[cfg(target_env = "msvc")]
-    c_config.flag("-utf-8");
-
-    let parser_path = src_dir.join("parser.c");
-    c_config.file(&parser_path);
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
-
-    let scanner_path = src_dir.join("scanner.c");
-    if scanner_path.exists() {
-        c_config.file(&scanner_path);
-        println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    }
-
-    c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
-}

cli/src/templates/build.zig

@@ -1,79 +0,0 @@
-const std = @import("std");
-
-pub fn build(b: *std.Build) !void {
-    const target = b.standardTargetOptions(.{});
-    const optimize = b.standardOptimizeOption(.{});
-
-    const shared = b.option(bool, "build-shared", "Build a shared library") orelse true;
-    const reuse_alloc = b.option(bool, "reuse-allocator", "Reuse the library allocator") orelse false;
-
-    const lib: *std.Build.Step.Compile = if (shared) b.addSharedLibrary(.{
-        .name = "tree-sitter-PARSER_NAME",
-        .pic = true,
-        .target = target,
-        .optimize = optimize,
-        .link_libc = true,
-    }) else b.addStaticLibrary(.{
-        .name = "tree-sitter-PARSER_NAME",
-        .target = target,
-        .optimize = optimize,
-        .link_libc = true,
-    });
-
-    lib.addCSourceFile(.{
-        .file = b.path("src/parser.c"),
-        .flags = &.{"-std=c11"},
-    });
-    if (hasScanner(b.build_root.handle)) {
-        lib.addCSourceFile(.{
-            .file = b.path("src/scanner.c"),
-            .flags = &.{"-std=c11"},
-        });
-    }
-
-    if (reuse_alloc) {
-        lib.root_module.addCMacro("TREE_SITTER_REUSE_ALLOCATOR", "");
-    }
-    if (optimize == .Debug) {
-        lib.root_module.addCMacro("TREE_SITTER_DEBUG", "");
-    }
-
-    lib.addIncludePath(b.path("src"));
-
-    b.installArtifact(lib);
-    b.installFile("src/node-types.json", "node-types.json");
-    b.installDirectory(.{ .source_dir = b.path("queries"), .install_dir = .prefix, .install_subdir = "queries", .include_extensions = &.{"scm"} });
-
-    const module = b.addModule("tree-sitter-PARSER_NAME", .{
-        .root_source_file = b.path("bindings/zig/root.zig"),
-        .target = target,
-        .optimize = optimize,
-    });
-    module.linkLibrary(lib);
-
-    const ts_dep = b.dependency("tree-sitter", .{});
-    const ts_mod = ts_dep.module("tree-sitter");
-    module.addImport("tree-sitter", ts_mod);
-
-    // ╭─────────────────╮
-    // │      Tests      │
-    // ╰─────────────────╯
-
-    const tests = b.addTest(.{
-        .root_source_file = b.path("bindings/zig/root.zig"),
-        .target = target,
-        .optimize = optimize,
-    });
-    tests.linkLibrary(lib);
-    tests.root_module.addImport("tree-sitter", ts_mod);
-
-    const run_tests = b.addRunArtifact(tests);
-
-    const test_step = b.step("test", "Run unit tests");
-    test_step.dependOn(&run_tests.step);
-}
-
-inline fn hasScanner(dir: std.fs.Dir) bool {
-    dir.access("src/scanner.c", .{}) catch return false;
-    return true;
-}

cli/src/templates/build.zig.zon

@@ -1,17 +0,0 @@
-.{
-    .name = "tree-sitter-PARSER_NAME",
-    .version = "PARSER_VERSION",
-    .dependencies = .{ .@"tree-sitter" = .{
-        .url = "https://github.com/tree-sitter/zig-tree-sitter/archive/refs/tags/v0.25.0.tar.gz",
-        .hash = "12201a8d5e840678bbbf5128e605519c4024af422295d68e2ba2090e675328e5811d",
-    } },
-    .paths = .{
-        "build.zig",
-        "build.zig.zon",
-        "bindings/zig",
-        "src",
-        "queries",
-        "LICENSE",
-        "README.md",
-    },
-}

cli/src/templates/index.d.ts

@@ -1,27 +0,0 @@
-type BaseNode = {
-  type: string;
-  named: boolean;
-};
-
-type ChildNode = {
-  multiple: boolean;
-  required: boolean;
-  types: BaseNode[];
-};
-
-type NodeInfo =
-  | (BaseNode & {
-      subtypes: BaseNode[];
-    })
-  | (BaseNode & {
-      fields: { [name: string]: ChildNode };
-      children: ChildNode[];
-    });
-
-type Language = {
-  language: unknown;
-  nodeTypeInfo: NodeInfo[];
-};
-
-declare const language: Language;
-export = language;

cli/src/templates/index.js

@@ -1,11 +0,0 @@
-const root = require("path").join(__dirname, "..", "..");
-
-module.exports =
-  typeof process.versions.bun === "string"
-    // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time
-    ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-KEBAB_PARSER_NAME.node`)
-    : require("node-gyp-build")(root);
-
-try {
-  module.exports.nodeTypeInfo = require("../../src/node-types.json");
-} catch (_) {}

cli/src/templates/lib.rs

@@ -1,53 +0,0 @@
-//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library.
-//!
-//! Typically, you will use the [LANGUAGE][] constant to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
-//!
-//! ```
-//! let code = r#"
-//! "#;
-//! let mut parser = tree_sitter::Parser::new();
-//! let language = tree_sitter_PARSER_NAME::LANGUAGE;
-//! parser
-//!     .set_language(&language.into())
-//!     .expect("Error loading TITLE_PARSER_NAME parser");
-//! let tree = parser.parse(code, None).unwrap();
-//! assert!(!tree.root_node().has_error());
-//! ```
-//!
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
-//! [tree-sitter]: https://tree-sitter.github.io/
-
-use tree_sitter_language::LanguageFn;
-
-extern "C" {
-    fn tree_sitter_PARSER_NAME() -> *const ();
-}
-
-/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar.
-///
-/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
-pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) };
-
-/// The content of the [`node-types.json`][] file for this grammar.
-///
-/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
-pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
-
-// NOTE: uncomment these to include any queries that this grammar contains:
-
-// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn test_can_load_grammar() {
-        let mut parser = tree_sitter::Parser::new();
-        parser
-            .set_language(&super::LANGUAGE.into())
-            .expect("Error loading TITLE_PARSER_NAME parser");
-    }
-}

cli/src/templates/root.zig

@@ -1,19 +0,0 @@
-const testing = @import("std").testing;
-
-const ts = @import("tree-sitter");
-const Language = ts.Language;
-const Parser = ts.Parser;
-
-pub extern fn tree_sitter_PARSER_NAME() callconv(.C) *const Language;
-
-pub export fn language() *const Language {
-    return tree_sitter_PARSER_NAME();
-}
-
-test "can load grammar" {
-    const parser = Parser.create();
-    defer parser.destroy();
-    try testing.expectEqual(parser.setLanguage(language()), void{});
-    try testing.expectEqual(parser.getLanguage(), tree_sitter_PARSER_NAME());
-}
-

cli/src/tests/async_context_test.rs

@@ -1,278 +0,0 @@
-use std::{
-    future::Future,
-    pin::{pin, Pin},
-    ptr,
-    task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker},
-};
-
-use tree_sitter::Parser;
-
-use super::helpers::fixtures::get_language;
-
-#[test]
-fn test_node_in_fut() {
-    let (ret, pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("bash");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let fut_val_fn = || async {
-            yield_now().await;
-            root.child(0).unwrap().kind()
-        };
-
-        yield_now().await;
-
-        let fut_ref_fn = || async {
-            yield_now().await;
-            root_ref.child(0).unwrap().kind()
-        };
-
-        let f1 = fut_val_fn().await;
-        let f2 = fut_ref_fn().await;
-        assert_eq!(f1, f2);
-
-        let fut_val = async {
-            yield_now().await;
-            root.child(0).unwrap().kind()
-        };
-
-        let fut_ref = async {
-            yield_now().await;
-            root_ref.child(0).unwrap().kind()
-        };
-
-        let f1 = fut_val.await;
-        let f2 = fut_ref.await;
-        assert_eq!(f1, f2);
-
-        f1
-    })
-    .join();
-    assert_eq!(ret, "comment");
-    assert_eq!(pended, 5);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut() {
-    let ((), pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("c");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = async {
-            yield_now().await;
-            let _ = root.to_sexp();
-        };
-
-        yield_now().await;
-
-        let fut_ref = async {
-            yield_now().await;
-            let _ = root_ref.to_sexp();
-            cursor_ref.goto_first_child();
-        };
-
-        fut_val.await;
-        fut_ref.await;
-
-        cursor_ref.goto_first_child();
-    })
-    .join();
-    assert_eq!(pended, 3);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut_with_fut_fabrics() {
-    let ((), pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("javascript");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = || async {
-            yield_now().await;
-            let _ = root.to_sexp();
-        };
-
-        yield_now().await;
-
-        let fut_ref = || async move {
-            yield_now().await;
-            let _ = root_ref.to_sexp();
-            cursor_ref.goto_first_child();
-        };
-
-        fut_val().await;
-        fut_val().await;
-        fut_ref().await;
-    })
-    .join();
-    assert_eq!(pended, 4);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut_with_inner_spawns() {
-    let (ret, pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("rust");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = || {
-            let tree = tree.clone();
-            async move {
-                let root = tree.root_node();
-                let mut cursor = tree.walk();
-                let cursor_ref = &mut cursor;
-                yield_now().await;
-                let _ = root.to_sexp();
-                cursor_ref.goto_first_child();
-            }
-        };
-
-        yield_now().await;
-
-        let fut_ref = || {
-            let tree = tree.clone();
-            async move {
-                let root = tree.root_node();
-                let root_ref = &root;
-                let mut cursor = tree.walk();
-                let cursor_ref = &mut cursor;
-                yield_now().await;
-                let _ = root_ref.to_sexp();
-                cursor_ref.goto_first_child();
-            }
-        };
-
-        let ((), p1) = tokio_like_spawn(fut_val()).await.unwrap();
-        let ((), p2) = tokio_like_spawn(fut_ref()).await.unwrap();
-
-        cursor_ref.goto_first_child();
-
-        fut_val().await;
-        fut_val().await;
-        fut_ref().await;
-
-        cursor_ref.goto_first_child();
-
-        p1 + p2
-    })
-    .join();
-    assert_eq!(pended, 4);
-    assert_eq!(ret, 2);
-}
-
-fn tokio_like_spawn<T>(future: T) -> JoinHandle<(T::Output, usize)>
-where
-    T: Future + Send + 'static,
-    T::Output: Send + 'static,
-{
-    // No runtime, just noop waker
-
-    let waker = noop_waker();
-    let mut cx = task::Context::from_waker(&waker);
-
-    let mut pending = 0;
-    let mut future = pin!(future);
-    let ret = loop {
-        match future.as_mut().poll(&mut cx) {
-            Poll::Pending => pending += 1,
-            Poll::Ready(r) => {
-                break r;
-            }
-        }
-    };
-    JoinHandle::new((ret, pending))
-}
-
-async fn yield_now() {
-    struct SimpleYieldNow {
-        yielded: bool,
-    }
-
-    impl Future for SimpleYieldNow {
-        type Output = ();
-
-        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-            cx.waker().wake_by_ref();
-            if self.yielded {
-                return Poll::Ready(());
-            }
-            self.yielded = true;
-            Poll::Pending
-        }
-    }
-
-    SimpleYieldNow { yielded: false }.await;
-}
-
-pub fn noop_waker() -> Waker {
-    const VTABLE: RawWakerVTable = RawWakerVTable::new(
-        // Cloning just returns a new no-op raw waker
-        |_| RAW,
-        // `wake` does nothing
-        |_| {},
-        // `wake_by_ref` does nothing
-        |_| {},
-        // Dropping does nothing as we don't allocate anything
-        |_| {},
-    );
-    const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE);
-    unsafe { Waker::from_raw(RAW) }
-}
-
-struct JoinHandle<T> {
-    data: Option<T>,
-}
-
-impl<T> JoinHandle<T> {
-    #[must_use]
-    const fn new(data: T) -> Self {
-        Self { data: Some(data) }
-    }
-
-    fn join(&mut self) -> T {
-        self.data.take().unwrap()
-    }
-}
-
-impl<T: Unpin> Future for JoinHandle<T> {
-    type Output = std::result::Result<T, ()>;
-
-    fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let data = self.get_mut().data.take().unwrap();
-        Poll::Ready(Ok(data))
-    }
-}

cli/src/tests/helpers/allocations.rs

@@ -1,121 +0,0 @@
-use std::{
-    collections::HashMap,
-    os::raw::c_void,
-    sync::{
-        atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},
-        Mutex,
-    },
-};
-
-#[ctor::ctor]
-unsafe fn initialize_allocation_recording() {
-    tree_sitter::set_allocator(
-        Some(ts_record_malloc),
-        Some(ts_record_calloc),
-        Some(ts_record_realloc),
-        Some(ts_record_free),
-    );
-}
-
-#[derive(Debug, PartialEq, Eq, Hash)]
-struct Allocation(*const c_void);
-unsafe impl Send for Allocation {}
-unsafe impl Sync for Allocation {}
-
-#[derive(Default)]
-struct AllocationRecorder {
-    enabled: AtomicBool,
-    allocation_count: AtomicUsize,
-    outstanding_allocations: Mutex<HashMap<Allocation, usize>>,
-}
-
-thread_local! {
-    static RECORDER: AllocationRecorder = AllocationRecorder::default();
-}
-
-extern "C" {
-    fn malloc(size: usize) -> *mut c_void;
-    fn calloc(count: usize, size: usize) -> *mut c_void;
-    fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
-    fn free(ptr: *mut c_void);
-}
-
-pub fn record<T>(f: impl FnOnce() -> T) -> T {
-    RECORDER.with(|recorder| {
-        recorder.enabled.store(true, SeqCst);
-        recorder.allocation_count.store(0, SeqCst);
-        recorder.outstanding_allocations.lock().unwrap().clear();
-    });
-
-    let value = f();
-
-    let outstanding_allocation_indices = RECORDER.with(|recorder| {
-        recorder.enabled.store(false, SeqCst);
-        recorder.allocation_count.store(0, SeqCst);
-        recorder
-            .outstanding_allocations
-            .lock()
-            .unwrap()
-            .drain()
-            .map(|e| e.1)
-            .collect::<Vec<_>>()
-    });
-    assert!(
-        outstanding_allocation_indices.is_empty(),
-        "Leaked allocation indices: {outstanding_allocation_indices:?}"
-    );
-    value
-}
-
-fn record_alloc(ptr: *mut c_void) {
-    RECORDER.with(|recorder| {
-        if recorder.enabled.load(SeqCst) {
-            let count = recorder.allocation_count.fetch_add(1, SeqCst);
-            recorder
-                .outstanding_allocations
-                .lock()
-                .unwrap()
-                .insert(Allocation(ptr), count);
-        }
-    });
-}
-
-fn record_dealloc(ptr: *mut c_void) {
-    RECORDER.with(|recorder| {
-        if recorder.enabled.load(SeqCst) {
-            recorder
-                .outstanding_allocations
-                .lock()
-                .unwrap()
-                .remove(&Allocation(ptr));
-        }
-    });
-}
-
-unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
-    let result = malloc(size);
-    record_alloc(result);
-    result
-}
-
-unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
-    let result = calloc(count, size);
-    record_alloc(result);
-    result
-}
-
-unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
-    let result = realloc(ptr, size);
-    if ptr.is_null() {
-        record_alloc(result);
-    } else if ptr != result {
-        record_dealloc(ptr);
-        record_alloc(result);
-    }
-    result
-}
-
-unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
-    record_dealloc(ptr);
-    free(ptr);
-}

cli/src/tests/parser_hang_test.rs

@@ -1,104 +0,0 @@
-// For some reasons `Command::spawn` doesn't work in CI env for many exotic arches.
-#![cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
-
-use std::{
-    env::VarError,
-    process::{Command, Stdio},
-};
-
-use tree_sitter::Parser;
-use tree_sitter_generate::load_grammar_file;
-
-use super::generate_parser;
-use crate::tests::helpers::fixtures::{fixtures_dir, get_test_language};
-
-// The `sanitizing` cfg is required to don't run tests under specific sunitizer
-// because they don't work well with subprocesses _(it's an assumption)_.
-//
-// Below are two alternative examples of how to disable tests for some arches
-// if a way with excluding the whole mod from compilation wouldn't work well.
-//
-// XXX: Also may be it makes sense to keep such tests as ignored by default
-//      to omit surprises and enable them on CI by passing an extra option explicitly:
-//
-//        > cargo test -- --include-ignored
-//
-// #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
-// #[cfg_attr(not(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing))), ignore)]
-//
-#[test]
-fn test_grammar_that_should_hang_and_not_segfault() {
-    let parent_sleep_millis = 1000;
-    let test_name = "test_grammar_that_should_hang_and_not_segfault";
-    let test_var = "CARGO_HANG_TEST";
-
-    eprintln!("  {test_name}");
-
-    let tests_exec_path = std::env::args()
-        .next()
-        .expect("Failed to get tests executable path");
-
-    match std::env::var(test_var) {
-        Ok(v) if v == test_name => {
-            eprintln!("    child process id {}", std::process::id());
-            hang_test();
-        }
-
-        Err(VarError::NotPresent) => {
-            eprintln!("    parent process id {}", std::process::id());
-            let mut command = Command::new(tests_exec_path);
-            command.arg(test_name).env(test_var, test_name);
-
-            if std::env::args().any(|x| x == "--nocapture") {
-                command.arg("--nocapture");
-            } else {
-                command.stdout(Stdio::null()).stderr(Stdio::null());
-            }
-
-            match command.spawn() {
-                Ok(mut child) => {
-                    std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
-                    match child.try_wait() {
-                        Ok(Some(status)) if status.success() => {
-                            panic!("Child didn't hang and exited successfully")
-                        }
-                        Ok(Some(status)) => panic!(
-                            "Child didn't hang and exited with status code: {:?}",
-                            status.code()
-                        ),
-                        _ => (),
-                    }
-                    if let Err(e) = child.kill() {
-                        eprintln!(
-                            "Failed to kill hang test's process id: {}, error: {e}",
-                            child.id()
-                        );
-                    }
-                }
-                Err(e) => panic!("{e}"),
-            }
-        }
-
-        Err(e) => panic!("Env var error: {e}"),
-
-        _ => unreachable!(),
-    }
-}
-
-fn hang_test() {
-    let test_grammar_dir = fixtures_dir()
-        .join("test_grammars")
-        .join("get_col_should_hang_not_crash");
-
-    let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
-    let (parser_name, parser_code) = generate_parser(grammar_json.as_str()).unwrap();
-
-    let language = get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
-
-    let mut parser = Parser::new();
-    parser.set_language(&language).unwrap();
-
-    let code_that_should_hang = "\nHello";
-
-    parser.parse(code_that_should_hang, None).unwrap();
-}

cli/src/tests/test_highlight_test.rs

@@ -1,70 +0,0 @@
-use tree_sitter::Parser;
-use tree_sitter_highlight::{Highlight, Highlighter};
-
-use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
-use crate::{
-    query_testing::{parse_position_comments, Assertion, Utf8Point},
-    test_highlight::get_highlight_positions,
-};
-
-#[test]
-fn test_highlight_test_with_basic_test() {
-    let language = get_language("javascript");
-    let config = get_highlight_config(
-        "javascript",
-        Some("injections.scm"),
-        &[
-            "function".to_string(),
-            "variable".to_string(),
-            "keyword".to_string(),
-        ],
-    );
-    let source = [
-        "// hi",
-        "var abc = function(d) {",
-        "  // ^ function",
-        "  //       ^^^ keyword",
-        "  return d + e;",
-        "  //     ^ variable",
-        "  //       ^ !variable",
-        "};",
-        "var y̆y̆y̆y̆ = function() {}",
-        "  // ^ function",
-        "  //       ^ keyword",
-    ]
-    .join("\n");
-
-    let assertions =
-        parse_position_comments(&mut Parser::new(), &language, source.as_bytes()).unwrap();
-    assert_eq!(
-        assertions,
-        &[
-            Assertion::new(1, 5, 1, false, String::from("function")),
-            Assertion::new(1, 11, 3, false, String::from("keyword")),
-            Assertion::new(4, 9, 1, false, String::from("variable")),
-            Assertion::new(4, 11, 1, true, String::from("variable")),
-            Assertion::new(8, 5, 1, false, String::from("function")),
-            Assertion::new(8, 11, 1, false, String::from("keyword")),
-        ]
-    );
-
-    let mut highlighter = Highlighter::new();
-    let highlight_positions =
-        get_highlight_positions(test_loader(), &mut highlighter, &config, source.as_bytes())
-            .unwrap();
-    assert_eq!(
-        highlight_positions,
-        &[
-            (Utf8Point::new(1, 0), Utf8Point::new(1, 3), Highlight(2)), // "var"
-            (Utf8Point::new(1, 4), Utf8Point::new(1, 7), Highlight(0)), // "abc"
-            (Utf8Point::new(1, 10), Utf8Point::new(1, 18), Highlight(2)), // "function"
-            (Utf8Point::new(1, 19), Utf8Point::new(1, 20), Highlight(1)), // "d"
-            (Utf8Point::new(4, 2), Utf8Point::new(4, 8), Highlight(2)), // "return"
-            (Utf8Point::new(4, 9), Utf8Point::new(4, 10), Highlight(1)), // "d"
-            (Utf8Point::new(4, 13), Utf8Point::new(4, 14), Highlight(1)), // "e"
-            (Utf8Point::new(8, 0), Utf8Point::new(8, 3), Highlight(2)), // "var"
-            (Utf8Point::new(8, 4), Utf8Point::new(8, 8), Highlight(0)), // "y̆y̆y̆y̆"
-            (Utf8Point::new(8, 11), Utf8Point::new(8, 19), Highlight(2)), // "function"
-        ]
-    );
-}

cli/src/version.rs

@@ -1,264 +0,0 @@
-use std::{fs, path::PathBuf, process::Command};
-
-use anyhow::{anyhow, Context, Result};
-use regex::Regex;
-use tree_sitter_loader::TreeSitterJSON;
-
-pub struct Version {
-    pub version: String,
-    pub current_dir: PathBuf,
-}
-
-impl Version {
-    #[must_use]
-    pub const fn new(version: String, current_dir: PathBuf) -> Self {
-        Self {
-            version,
-            current_dir,
-        }
-    }
-
-    pub fn run(self) -> Result<()> {
-        let tree_sitter_json = self.current_dir.join("tree-sitter.json");
-
-        let tree_sitter_json =
-            serde_json::from_str::<TreeSitterJSON>(&fs::read_to_string(tree_sitter_json)?)?;
-
-        let is_multigrammar = tree_sitter_json.grammars.len() > 1;
-
-        self.update_treesitter_json().with_context(|| {
-            format!(
-                "Failed to update tree-sitter.json at {}",
-                self.current_dir.display()
-            )
-        })?;
-        self.update_cargo_toml().with_context(|| {
-            format!(
-                "Failed to update Cargo.toml at {}",
-                self.current_dir.display()
-            )
-        })?;
-        self.update_package_json().with_context(|| {
-            format!(
-                "Failed to update package.json at {}",
-                self.current_dir.display()
-            )
-        })?;
-        self.update_makefile(is_multigrammar).with_context(|| {
-            format!(
-                "Failed to update Makefile at {}",
-                self.current_dir.display()
-            )
-        })?;
-        self.update_cmakelists_txt().with_context(|| {
-            format!(
-                "Failed to update CMakeLists.txt at {}",
-                self.current_dir.display()
-            )
-        })?;
-        self.update_pyproject_toml().with_context(|| {
-            format!(
-                "Failed to update pyproject.toml at {}",
-                self.current_dir.display()
-            )
-        })?;
-
-        Ok(())
-    }
-
-    fn update_treesitter_json(&self) -> Result<()> {
-        let tree_sitter_json = &fs::read_to_string(self.current_dir.join("tree-sitter.json"))?;
-
-        let tree_sitter_json = tree_sitter_json
-            .lines()
-            .map(|line| {
-                if line.contains("\"version\":") {
-                    let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len();
-                    let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1;
-                    let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
-
-                    format!(
-                        "{}{}{}",
-                        &line[..start_quote],
-                        self.version,
-                        &line[end_quote..]
-                    )
-                } else {
-                    line.to_string()
-                }
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
-            + "\n";
-
-        fs::write(self.current_dir.join("tree-sitter.json"), tree_sitter_json)?;
-
-        Ok(())
-    }
-
-    fn update_cargo_toml(&self) -> Result<()> {
-        if !self.current_dir.join("Cargo.toml").exists() {
-            return Ok(());
-        }
-
-        let cargo_toml = fs::read_to_string(self.current_dir.join("Cargo.toml"))?;
-
-        let cargo_toml = cargo_toml
-            .lines()
-            .map(|line| {
-                if line.starts_with("version =") {
-                    format!("version = \"{}\"", self.version)
-                } else {
-                    line.to_string()
-                }
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
-            + "\n";
-
-        fs::write(self.current_dir.join("Cargo.toml"), cargo_toml)?;
-
-        if self.current_dir.join("Cargo.lock").exists() {
-            let Ok(cmd) = Command::new("cargo")
-                .arg("generate-lockfile")
-                .arg("--offline")
-                .current_dir(&self.current_dir)
-                .output()
-            else {
-                return Ok(()); // cargo is not `executable`, ignore
-            };
-
-            if !cmd.status.success() {
-                let stderr = String::from_utf8_lossy(&cmd.stderr);
-                return Err(anyhow!(
-                    "Failed to run `cargo generate-lockfile`:\n{stderr}"
-                ));
-            }
-        }
-
-        Ok(())
-    }
-
-    fn update_package_json(&self) -> Result<()> {
-        if !self.current_dir.join("package.json").exists() {
-            return Ok(());
-        }
-
-        let package_json = &fs::read_to_string(self.current_dir.join("package.json"))?;
-
-        let package_json = package_json
-            .lines()
-            .map(|line| {
-                if line.contains("\"version\":") {
-                    let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len();
-                    let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1;
-                    let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
-
-                    format!(
-                        "{}{}{}",
-                        &line[..start_quote],
-                        self.version,
-                        &line[end_quote..]
-                    )
-                } else {
-                    line.to_string()
-                }
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
-            + "\n";
-
-        fs::write(self.current_dir.join("package.json"), package_json)?;
-
-        if self.current_dir.join("package-lock.json").exists() {
-            let Ok(cmd) = Command::new("npm")
-                .arg("install")
-                .arg("--package-lock-only")
-                .current_dir(&self.current_dir)
-                .output()
-            else {
-                return Ok(()); // npm is not `executable`, ignore
-            };
-
-            if !cmd.status.success() {
-                let stderr = String::from_utf8_lossy(&cmd.stderr);
-                return Err(anyhow!("Failed to run `npm install`:\n{stderr}"));
-            }
-        }
-
-        Ok(())
-    }
-
-    fn update_makefile(&self, is_multigrammar: bool) -> Result<()> {
-        let makefile = if is_multigrammar {
-            if !self.current_dir.join("common").join("common.mak").exists() {
-                return Ok(());
-            }
-
-            fs::read_to_string(self.current_dir.join("Makefile"))?
-        } else {
-            if !self.current_dir.join("Makefile").exists() {
-                return Ok(());
-            }
-
-            fs::read_to_string(self.current_dir.join("Makefile"))?
-        };
-
-        let makefile = makefile
-            .lines()
-            .map(|line| {
-                if line.starts_with("VERSION") {
-                    format!("VERSION := {}", self.version)
-                } else {
-                    line.to_string()
-                }
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
-            + "\n";
-
-        fs::write(self.current_dir.join("Makefile"), makefile)?;
-
-        Ok(())
-    }
-
-    fn update_cmakelists_txt(&self) -> Result<()> {
-        if !self.current_dir.join("CMakeLists.txt").exists() {
-            return Ok(());
-        }
-
-        let cmake = fs::read_to_string(self.current_dir.join("CMakeLists.txt"))?;
-
-        let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#)?;
-        let cmake = re.replace(&cmake, format!(r#"$1"{}""#, self.version));
-
-        fs::write(self.current_dir.join("CMakeLists.txt"), cmake.as_bytes())?;
-
-        Ok(())
-    }
-
-    fn update_pyproject_toml(&self) -> Result<()> {
-        if !self.current_dir.join("pyproject.toml").exists() {
-            return Ok(());
-        }
-
-        let pyproject_toml = fs::read_to_string(self.current_dir.join("pyproject.toml"))?;
-
-        let pyproject_toml = pyproject_toml
-            .lines()
-            .map(|line| {
-                if line.starts_with("version =") {
-                    format!("version = \"{}\"", self.version)
-                } else {
-                    line.to_string()
-                }
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
-            + "\n";
-
-        fs::write(self.current_dir.join("pyproject.toml"), pyproject_toml)?;
-
-        Ok(())
-    }
-}

crates/cli/Cargo.toml

@@ -0,0 +1,115 @@
+[package]
+name = "tree-sitter-cli"
+version.workspace = true
+description = "CLI tool for developing, testing, and using Tree-sitter parsers"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+repository.workspace = true
+documentation = "https://docs.rs/tree-sitter-cli"
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+include = [ "build.rs", "README.md", "LICENSE", "benches/*", "src/**" ]
+
+[lints]
+workspace = true
+
+[lib]
+path = "src/tree_sitter_cli.rs"
+
+[[bin]]
+doc = false
+name = "tree-sitter"
+path = "src/main.rs"
+
+[[bench]]
+harness = false
+name = "benchmark"
+
+[features]
+default = [ "qjs-rt" ]
+qjs-rt = [ "tree-sitter-generate/qjs-rt" ]
+wasm = [ "tree-sitter/wasm", "tree-sitter-loader/wasm" ]
+
+[dependencies]
+ansi_colours.workspace = true
+anstyle.workspace = true
+anyhow.workspace = true
+bstr.workspace = true
+clap.workspace = true
+clap_complete.workspace = true
+clap_complete_nushell.workspace = true
+crc32fast.workspace = true
+ctor.workspace = true
+ctrlc.workspace = true
+dialoguer.workspace = true
+glob.workspace = true
+heck.workspace = true
+html-escape.workspace = true
+indoc.workspace = true
+log.workspace = true
+memchr.workspace = true
+rand.workspace = true
+regex.workspace = true
+schemars.workspace = true
+semver.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+similar.workspace = true
+streaming-iterator.workspace = true
+thiserror.workspace = true
+tiny_http.workspace = true
+walkdir.workspace = true
+wasmparser.workspace = true
+webbrowser.workspace = true
+
+tree-sitter.workspace = true
+tree-sitter-config.workspace = true
+tree-sitter-generate = { features = [ "load" ], workspace = true }
+tree-sitter-highlight.workspace = true
+tree-sitter-loader.workspace = true
+tree-sitter-tags.workspace = true
+
+[dev-dependencies]
+encoding_rs = "0.8.35"
+tree_sitter_proc_macro = { package = "tree-sitter-tests-proc-macro", path = "src/tests/proc_macro" }
+widestring = "1.2.1"
+
+pretty_assertions.workspace = true
+tempfile.workspace = true
+unindent.workspace = true
+
+[package.metadata.binstall]
+pkg-fmt = "zip"
+
+[package.metadata.binstall.overrides.aarch64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.armv7-unknown-linux-gnueabihf]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-arm{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.powerpc64-unknown-linux-gnu]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-linux-powerpc64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.i686-pc-windows-msvc]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-windows-x86{ archive-suffix }"
+
+[package.metadata.binstall.overrides.aarch64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-arm64{ archive-suffix }"
+
+[package.metadata.binstall.overrides.x86_64-apple-darwin]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-macos-x64{ archive-suffix }"

crates/cli/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/cli/README.md

@@ -7,20 +7,20 @@
 [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
 [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
 
-The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`.
+The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`,
+`Linux`, and `Windows`.
 
 ### Installation
 
-You can install the `tree-sitter-cli` with `cargo`:
+You can install the `tree-sitter-cli` with [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall):
 
 ```sh
-cargo install --locked tree-sitter-cli
+cargo binstall tree-sitter-cli
 ```
 
-or with `npm`:
-
+or you can build it from source:
 ```sh
-npm install tree-sitter-cli
+cargo install --locked tree-sitter-cli
 ```
 
 You can also download a pre-built binary for your platform from [the releases page].
@@ -34,9 +34,11 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
 
 ### Commands
 
-* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information.
+* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current
+  working directory. See [the documentation] for more information.
 
-* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information.
+* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory.
+  See [the documentation] for more information.
 
 * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.
 

crates/cli/benches/benchmark.rs

@@ -2,13 +2,15 @@ use std::{
     collections::BTreeMap,
     env, fs,
     path::{Path, PathBuf},
-    str,
     sync::LazyLock,
     time::Instant,
 };
 
 use anyhow::Context;
+use log::info;
 use tree_sitter::{Language, Parser, Query};
+#[cfg(feature = "wasm")]
+use tree_sitter::{WasmStore, wasmtime};
 use tree_sitter_loader::{CompileConfig, Loader};
 
 include!("../src/tests/helpers/dirs.rs");
@@ -18,14 +20,18 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
 static EXAMPLE_FILTER: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
 static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
-    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
-        .map(|s| s.parse::<usize>().unwrap())
-        .unwrap_or(5)
+    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
 });
+static WASM: LazyLock<bool> = LazyLock::new(|| env::var_os("TREE_SITTER_BENCHMARK_WASM").is_some());
 static TEST_LOADER: LazyLock<Loader> =
     LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
+#[cfg(feature = "wasm")]
+static WASM_ENGINE: LazyLock<wasmtime::Engine> = LazyLock::new(Default::default);
 
-#[allow(clippy::type_complexity)]
+#[expect(
+    clippy::type_complexity,
+    reason = "complex map type reflects benchmark data structure"
+)]
 static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
     BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>,
 > = LazyLock::new(|| {
@@ -37,22 +43,14 @@ static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
             if let Ok(example_files) = fs::read_dir(dir.join("examples")) {
                 example_paths.extend(example_files.filter_map(|p| {
                     let p = p.unwrap().path();
-                    if p.is_file() {
-                        Some(p)
-                    } else {
-                        None
-                    }
+                    if p.is_file() { Some(p) } else { None }
                 }));
             }
 
             if let Ok(query_files) = fs::read_dir(dir.join("queries")) {
                 query_paths.extend(query_files.filter_map(|p| {
                     let p = p.unwrap().path();
-                    if p.is_file() {
-                        Some(p)
-                    } else {
-                        None
-                    }
+                    if p.is_file() { Some(p) } else { None }
                 }));
             }
         } else {
@@ -71,6 +69,8 @@ static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
 });
 
 fn main() {
+    tree_sitter_cli::logger::init();
+
     let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR
         .values()
         .flat_map(|(e, q)| {
@@ -81,7 +81,7 @@ fn main() {
         .max()
         .unwrap_or(0);
 
-    eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT);
+    info!("Benchmarking with {} repetitions", *REPETITION_COUNT);
 
     let mut parser = Parser::new();
     let mut all_normal_speeds = Vec::new();
@@ -92,38 +92,42 @@ fn main() {
     {
         let language_name = language_path.file_name().unwrap().to_str().unwrap();
 
-        if let Some(filter) = LANGUAGE_FILTER.as_ref() {
-            if language_name != filter.as_str() {
-                continue;
-            }
+        if let Some(filter) = LANGUAGE_FILTER.as_ref()
+            && language_name != filter.as_str()
+        {
+            continue;
         }
 
-        eprintln!("\nLanguage: {language_name}");
-        let language = get_language(language_path);
+        info!("\nLanguage: {language_name}");
+        let language = if *WASM {
+            get_wasm_language(language_name, &mut parser)
+        } else {
+            get_language(language_path)
+        };
         parser.set_language(&language).unwrap();
 
-        eprintln!("  Constructing Queries");
+        info!("  Constructing Queries");
         for path in query_paths {
-            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
-                if !path.to_str().unwrap().contains(filter.as_str()) {
-                    continue;
-                }
+            if let Some(filter) = EXAMPLE_FILTER.as_ref()
+                && !path.to_str().unwrap().contains(filter.as_str())
+            {
+                continue;
             }
 
             parse(path, max_path_length, |source| {
                 Query::new(&language, str::from_utf8(source).unwrap())
-                    .with_context(|| format!("Query file path: {path:?}"))
+                    .with_context(|| format!("Query file path: {}", path.display()))
                     .expect("Failed to parse query");
             });
         }
 
-        eprintln!("  Parsing Valid Code:");
+        info!("  Parsing Valid Code:");
         let mut normal_speeds = Vec::new();
         for example_path in example_paths {
-            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
-                if !example_path.to_str().unwrap().contains(filter.as_str()) {
-                    continue;
-                }
+            if let Some(filter) = EXAMPLE_FILTER.as_ref()
+                && !example_path.to_str().unwrap().contains(filter.as_str())
+            {
+                continue;
             }
 
             normal_speeds.push(parse(example_path, max_path_length, |code| {
@@ -131,17 +135,17 @@ fn main() {
             }));
         }
 
-        eprintln!("  Parsing Invalid Code (mismatched languages):");
+        info!("  Parsing Invalid Code (mismatched languages):");
         let mut error_speeds = Vec::new();
         for (other_language_path, (example_paths, _)) in
             EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
         {
             if other_language_path != language_path {
                 for example_path in example_paths {
-                    if let Some(filter) = EXAMPLE_FILTER.as_ref() {
-                        if !example_path.to_str().unwrap().contains(filter.as_str()) {
-                            continue;
-                        }
+                    if let Some(filter) = EXAMPLE_FILTER.as_ref()
+                        && !example_path.to_str().unwrap().contains(filter.as_str())
+                    {
+                        continue;
                     }
 
                     error_speeds.push(parse(example_path, max_path_length, |code| {
@@ -152,30 +156,30 @@ fn main() {
         }
 
         if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) {
-            eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
-            eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+            info!("  Average Speed (normal): {average_normal} bytes/ms");
+            info!("  Worst Speed (normal):   {worst_normal} bytes/ms");
         }
 
         if let Some((average_error, worst_error)) = aggregate(&error_speeds) {
-            eprintln!("  Average Speed (errors): {average_error} bytes/ms");
-            eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
+            info!("  Average Speed (errors): {average_error} bytes/ms");
+            info!("  Worst Speed (errors):   {worst_error} bytes/ms");
         }
 
         all_normal_speeds.extend(normal_speeds);
         all_error_speeds.extend(error_speeds);
     }
 
-    eprintln!("\n  Overall");
+    info!("\n  Overall");
     if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) {
-        eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
-        eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+        info!("  Average Speed (normal): {average_normal} bytes/ms");
+        info!("  Worst Speed (normal):   {worst_normal} bytes/ms");
     }
 
     if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) {
-        eprintln!("  Average Speed (errors): {average_error} bytes/ms");
-        eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
+        info!("  Average Speed (errors): {average_error} bytes/ms");
+        info!("  Worst Speed (errors):   {worst_error} bytes/ms");
     }
-    eprintln!();
+    info!("");
 }
 
 fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
@@ -194,14 +198,8 @@ fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
 }
 
 fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize {
-    eprint!(
-        "    {:width$}\t",
-        path.file_name().unwrap().to_str().unwrap(),
-        width = max_path_length
-    );
-
     let source_code = fs::read(path)
-        .with_context(|| format!("Failed to read {path:?}"))
+        .with_context(|| format!("Failed to read {}", path.display()))
         .unwrap();
     let time = Instant::now();
     for _ in 0..*REPETITION_COUNT {
@@ -210,8 +208,9 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
     let duration = time.elapsed() / (*REPETITION_COUNT as u32);
     let duration_ns = duration.as_nanos();
     let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns;
-    eprintln!(
-        "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
+    info!(
+        "    {:max_path_length$}\ttime {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
+        path.file_name().unwrap().to_str().unwrap(),
         (duration_ns as f64) / 1e6,
     );
     speed as usize
@@ -221,6 +220,35 @@ fn get_language(path: &Path) -> Language {
     let src_path = GRAMMARS_DIR.join(path).join("src");
     TEST_LOADER
         .load_language_at_path(CompileConfig::new(&src_path, None, None))
-        .with_context(|| format!("Failed to load language at path {src_path:?}"))
+        .with_context(|| format!("Failed to load language at path {}", src_path.display()))
         .unwrap()
 }
+
+#[cfg(feature = "wasm")]
+fn get_wasm_language(language_name: &str, parser: &mut Parser) -> Language {
+    let wasm_language_name = language_name.replace('-', "_");
+    let wasm_path = ROOT_DIR
+        .join("target")
+        .join("release")
+        .join(format!("tree-sitter-{language_name}.wasm"));
+    let wasm = fs::read(&wasm_path)
+        .with_context(|| {
+            format!(
+                "Failed to read {}. Generate Wasm fixtures with `cargo xtask generate-fixtures --wasm`",
+                wasm_path.display()
+            )
+        })
+        .unwrap();
+    let mut store = WasmStore::new(&WASM_ENGINE).expect("Failed to create Wasm store");
+    let language = store
+        .load_language(&wasm_language_name, &wasm)
+        .with_context(|| format!("Failed to load Wasm language at {}", wasm_path.display()))
+        .unwrap();
+    parser.set_wasm_store(store).unwrap();
+    language
+}
+
+#[cfg(not(feature = "wasm"))]
+fn get_wasm_language(_language_name: &str, _parser: &mut Parser) -> Language {
+    panic!("Wasm benchmarking requires the `wasm` feature");
+}

crates/cli/build.rs

@@ -52,16 +52,14 @@ fn main() {
 
 fn web_playground_files_present() -> bool {
     let paths = [
-        "../docs/src/assets/js/playground.js",
-        "../lib/binding_web/tree-sitter.js",
-        "../lib/binding_web/tree-sitter.wasm",
+        "../../docs/src/assets/js/playground.js",
+        "../../lib/binding_web/web-tree-sitter.js",
+        "../../lib/binding_web/web-tree-sitter.wasm",
     ];
 
     paths.iter().all(|p| Path::new(p).exists())
 }
 
-// When updating this function, don't forget to also update generate/build.rs which has a
-// near-identical function.
 fn read_git_sha() -> Option<String> {
     let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
 

crates/cli/eslint/.gitignore

@@ -0,0 +1 @@
+LICENSE

crates/cli/eslint/package.json

@@ -4,7 +4,8 @@
   "description": "Eslint configuration for Tree-sitter grammar files",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tree-sitter/tree-sitter.git"
+    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
+    "directory": "crates/cli/eslint"
   },
   "license": "MIT",
   "author": "Amaan Qureshi <amaanq12@gmail.com>",
@@ -16,9 +17,13 @@
     "tree-sitter"
   ],
   "dependencies": {
-    "eslint-plugin-jsdoc": "^50.2.4"
+    "eslint-plugin-jsdoc": "^62.7.0"
   },
   "peerDependencies": {
     "eslint": ">= 9"
+  },
+  "scripts": {
+    "prepack": "cp ../../../LICENSE .",
+    "postpack": "rm LICENSE"
   }
 }

crates/cli/npm/dsl.d.ts

@@ -3,17 +3,22 @@ type BlankRule = { type: 'BLANK' };
 type ChoiceRule = { type: 'CHOICE'; members: Rule[] };
 type FieldRule = { type: 'FIELD'; name: string; content: Rule };
 type ImmediateTokenRule = { type: 'IMMEDIATE_TOKEN'; content: Rule };
-type PatternRule = { type: 'PATTERN'; value: string };
+type PatternRule = { type: 'PATTERN'; value: string; flags?: string };
+type PrecedenceValue = string | number;
 type PrecDynamicRule = { type: 'PREC_DYNAMIC'; content: Rule; value: number };
-type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: number };
-type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number };
-type PrecRule = { type: 'PREC'; content: Rule; value: number };
+type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: PrecedenceValue };
+type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: PrecedenceValue };
+type PrecRule = { type: 'PREC'; content: Rule; value: PrecedenceValue };
 type Repeat1Rule = { type: 'REPEAT1'; content: Rule };
 type RepeatRule = { type: 'REPEAT'; content: Rule };
+type ReservedRule = { type: 'RESERVED'; content: Rule; context_name: string };
 type SeqRule = { type: 'SEQ'; members: Rule[] };
 type StringRule = { type: 'STRING'; value: string };
 type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
+type PrecedenceEntry = StringRule | SymbolRule<string>;
 type TokenRule = { type: 'TOKEN'; content: Rule };
+type EOFRule = { type: 'EOF' };
+
 
 type Rule =
   | AliasRule
@@ -28,17 +33,17 @@ type Rule =
   | PrecRule
   | Repeat1Rule
   | RepeatRule
+  | ReservedRule
   | SeqRule
   | StringRule
   | SymbolRule<string>
-  | TokenRule;
+  | TokenRule
+  | EOFRule;
 
-class RustRegex {
+declare class RustRegex {
   value: string;
 
-  constructor(pattern: string) {
-    this.value = pattern;
-  }
+  constructor(pattern: string);
 }
 
 type RuleOrLiteral = Rule | RegExp | RustRegex | string;
@@ -86,8 +91,8 @@ interface Grammar<
    */
   precedences?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[][],
-  ) => RuleOrLiteral[][],
+    previous: PrecedenceEntry[][],
+  ) => (string | PrecedenceEntry)[][],
 
   /**
    * An array of arrays of rule names. Each inner array represents a set of
@@ -101,8 +106,8 @@ interface Grammar<
    */
   conflicts?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[][],
-  ) => RuleOrLiteral[][];
+    previous: SymbolRule<string>[][],
+  ) => SymbolRule<string>[][];
 
   /**
    * An array of token names which can be returned by an _external scanner_.
@@ -127,9 +132,11 @@ interface Grammar<
    * specify extras: `$ => []` in your grammar.
    *
    *  @param $ grammar rules
+   *  @param previous array of extras from the base grammar
    */
   extras?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
+    previous: Rule[],
   ) => RuleOrLiteral[];
 
   /**
@@ -142,8 +149,8 @@ interface Grammar<
    */
   inline?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[],
-  ) => RuleOrLiteral[];
+    previous: SymbolRule<string>[],
+  ) => SymbolRule<string>[];
 
   /**
    * A list of hidden rule names that should be considered supertypes in the
@@ -155,8 +162,8 @@ interface Grammar<
    */
   supertypes?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
-    previous: Rule[],
-  ) => RuleOrLiteral[];
+    previous: SymbolRule<string>[],
+  ) => SymbolRule<string>[];
 
   /**
    * The name of a token that will match keywords for the purpose of the
@@ -166,13 +173,47 @@ interface Grammar<
    *
    * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar#keyword-extraction
    */
-  word?: ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral;
+  word?: (
+    $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
+  ) => SymbolRule<string>;
+
+
+  /**
+   * Mapping of names to reserved word sets. The first reserved word set is the
+   * global word set, meaning it applies to every rule in every parse state.
+   * The other word sets can be used with the `reserved` function. Each callback
+   * receives the base grammar's reserved word set of the same name as its second
+   * argument, or `undefined` if no matching set exists.
+   */
+  reserved?: Record<
+    string,
+    (
+      $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
+      previous: Rule[] | undefined,
+    ) => RuleOrLiteral[]
+  >;
 }
 
+/**
+ * Return type of grammar(). The runtime evaluates and normalizes the grammar
+ * beneath a "grammar" key. Optional input fields become required output fields
+ * with default values when not provided.
+ */
 type GrammarSchema<RuleName extends string> = {
-  [K in keyof Grammar<RuleName>]: K extends 'rules'
-  ? Record<RuleName, Rule>
-  : Grammar<RuleName>[K];
+  grammar: {
+    name: string;
+    /** Base grammar name when extending; undefined for root grammars. */
+    inherits: string | undefined;
+    rules: Record<RuleName, Rule>;
+    precedences: PrecedenceEntry[][];
+    conflicts: string[][];
+    externals: Rule[];
+    extras: Rule[];
+    inline: string[];
+    supertypes: string[];
+    word: string | undefined;
+    reserved: Record<string, Rule[]>;
+  };
 };
 
 /**
@@ -251,7 +292,7 @@ declare function optional(rule: RuleOrLiteral): ChoiceRule;
  * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
  */
 declare const prec: {
-  (value: String | number, rule: RuleOrLiteral): PrecRule;
+  (value: string | number, rule: RuleOrLiteral): PrecRule;
 
   /**
    * Marks the given rule as left-associative (and optionally applies a
@@ -267,7 +308,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   left(rule: RuleOrLiteral): PrecLeftRule;
-  left(value: String | number, rule: RuleOrLiteral): PrecLeftRule;
+  left(value: string | number, rule: RuleOrLiteral): PrecLeftRule;
 
   /**
    * Marks the given rule as right-associative (and optionally applies a
@@ -283,7 +324,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   right(rule: RuleOrLiteral): PrecRightRule;
-  right(value: String | number, rule: RuleOrLiteral): PrecRightRule;
+  right(value: string | number, rule: RuleOrLiteral): PrecRightRule;
 
   /**
    * Marks the given rule with a numerical precedence which will be used to
@@ -300,7 +341,7 @@ declare const prec: {
    *
    * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
    */
-  dynamic(value: String | number, rule: RuleOrLiteral): PrecDynamicRule;
+  dynamic(value: number, rule: RuleOrLiteral): PrecDynamicRule;
 };
 
 /**
@@ -320,6 +361,15 @@ declare function repeat(rule: RuleOrLiteral): RepeatRule;
  */
 declare function repeat1(rule: RuleOrLiteral): Repeat1Rule;
 
+/**
+ * Overrides the global reserved word set for a given rule. The word set name
+ * should be defined in the `reserved` field in the grammar.
+ *
+ * @param wordset name of the reserved word set
+ * @param rule rule that will use the reserved word set
+ */
+declare function reserved(wordset: string, rule: RuleOrLiteral): ReservedRule;
+
 /**
  * Creates a rule that matches any number of other rules, one after another.
  * It is analogous to simply writing multiple symbols next to each other
@@ -338,7 +388,7 @@ declare function sym<Name extends string>(name: Name): SymbolRule<Name>;
 
 /**
  * Marks the given rule as producing only a single token. Tree-sitter's
- * default is to treat each String or RegExp literal in the grammar as a
+ * default is to treat each string or RegExp literal in the grammar as a
  * separate token. Each token is matched separately by the lexer and
  * returned as its own leaf node in the tree. The token function allows
  * you to express a complex rule using the DSL functions (rather
@@ -362,6 +412,19 @@ declare const token: {
   immediate(rule: RuleOrLiteral): ImmediateTokenRule;
 };
 
+/**
+ * Matches the end of input. May only appear as the final symbol of a
+ * (possibly nested) sequence; a production ending in `eof()` reduces only
+ * when the lookahead is end-of-input, rather than shifting a token.
+ *
+ * Choice branches that continue past `eof()` are dropped as unreachable,
+ * and `eof()` is not allowed inside `token()`.
+ *
+ * Useful when a rule should match either an explicit terminator (e.g. a
+ * newline) or the end of the file.
+ */
+declare function eof(): EOFRule;
+
 /**
  * Creates a new language grammar with the provided schema.
  *

crates/cli/npm/install.js

@@ -6,7 +6,8 @@ const http = require('http');
 const https = require('https');
 const packageJSON = require('./package.json');
 
-// Look to a results table in https://github.com/tree-sitter/tree-sitter/issues/2196
+https.globalAgent.keepAlive = false;
+
 const matrix = {
   platform: {
     'darwin': {