build(deps): bump rquickjs to v0.13.0

Also use `IoError` more consistently throughout the rest of the project
where its straightforward to do so.

BREAKING CHANGE: Changes public error types for config, generate, and
loader crates.

.dockerignore

@@ -1,2 +0,0 @@
-target
-.git

.editorconfig

@@ -10,6 +10,9 @@ insert_final_newline = true
 [*.rs]
 indent_size = 4
 
+[*.{zig,zon}]
+indent_size = 4
+
 [Makefile]
 indent_style = tab
 indent_size = 8

.envrc

@@ -0,0 +1,4 @@
+source_up_if_exists
+if [ -z "${IN_NIX_SHELL:-}" ]; then
+  use flake .
+fi

.gitattributes

@@ -3,5 +3,4 @@
 /lib/src/unicode/*.h linguist-vendored
 /lib/src/unicode/LICENSE linguist-vendored
 
-/cli/src/generate/prepare_grammar/*.json -diff
 Cargo.lock -diff

.github/FUNDING.yml

@@ -0,0 +1,15 @@
+# These are supported funding model platforms
+
+github: tree-sitter
+patreon: # Replace with a single Patreon username
+open_collective: tree-sitter # Replace with a single Open Collective username
+ko_fi: amaanq
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+polar: # Replace with a single Polar username
+buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
+thanks_dev: # Replace with a single thanks.dev username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,7 +1,15 @@
 name: Bug Report
 description: Report a problem
-labels: [bug]
+type: Bug
 body:
+  - type: checkboxes
+    attributes:
+      label: AI Policy
+      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
+      options:
+        - label: I have read the AI Policy and this issue complies with it.
+          required: true
+
   - type: textarea
     attributes:
       label: "Problem"
@@ -13,9 +21,11 @@ body:
     attributes:
       label: "Steps to reproduce"
       placeholder: |
+        ```sh
         git clone --depth=1 https://github.com/tree-sitter/tree-sitter-ruby
         cd tree-sitter-ruby
         tree-sitter generate
+        ```
     validations:
       required: true
 

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,7 +1,15 @@
 name: Feature request
 description: Request an enhancement
-labels: [enhancement]
+type: Feature
 body:
+  - type: checkboxes
+    attributes:
+      label: AI Policy
+      description: Review our [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy).
+      options:
+        - label: I have read the AI Policy and this issue complies with it.
+          required: true
+
   - type: markdown
     attributes:
       value: |

.github/actions/cache/action.yml

@@ -1,24 +1,29 @@
-name: 'Cache'
-description: "This action caches fixtures"
+name: Cache
+
+description: This action caches fixtures
+
 outputs:
   cache-hit:
-    description: 'Cache hit'
-    value: ${{ steps.cache_output.outputs.cache-hit }}
+    description: Cache hit
+    value: ${{ steps.cache.outputs.cache-hit }}
+
 runs:
-  using: "composite"
+  using: composite
   steps:
-    - uses: actions/cache@v4
-      id: cache_fixtures
+    - uses: actions/cache@v5
+      id: cache
       with:
         path: |
           test/fixtures/grammars
           target/release/tree-sitter-*.wasm
         key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles(
-          'cli/src/generate/**',
-          'script/generate-fixtures*',
+          'crates/generate/src/**',
+          'lib/src/parser.h',
+          'lib/src/array.h',
+          'lib/src/alloc.h',
+          'lib/src/wasm-stdlib/external_scanner_stdlib.h',
+          'crates/loader/wasi-sdk-version',
+          'crates/loader/binaryen-version',
           'test/fixtures/grammars/*/**/src/*.c',
+          'test/fixtures/fixtures.json',
           '.github/actions/cache/action.yml') }}
-
-    - run: echo "cache-hit=${{ steps.cache_fixtures.outputs.cache-hit }}" >> $GITHUB_OUTPUT
-      shell: bash
-      id: cache_output

.github/cliff.toml

@@ -16,13 +16,13 @@ body = """
     {% for commit in commits%}\
         {% if not commit.scope %}\
             - {{ commit.message | upper_first }}\
-              {% if commit.github.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.github.pr_number }}>){%- endif %}
+              {% if commit.remote.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.remote.pr_number }}>){%- endif %}
         {% endif %}\
     {% endfor %}\
     {% for group, commits in commits | group_by(attribute="scope") %}\
         {% for commit in commits %}\
             - **{{commit.scope}}**: {{ commit.message | upper_first }}\
-                {% if commit.github.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.github.pr_number }}>){%- endif %}
+                {% if commit.remote.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.remote.pr_number }}>){%- endif %}
         {% endfor %}\
     {% endfor %}
 {% endfor %}
@@ -43,16 +43,16 @@ commit_preprocessors = [
 ]
 # regex for parsing and grouping commits
 commit_parsers = [
-  { message = "!:", group = "<!-- 0 -->Breaking" },
-  { message = "^feat", group = "<!-- 1 -->Features" },
-  { message = "^fix", group = "<!-- 2 -->Bug Fixes" },
-  { message = "^perf", group = "<!-- 3 -->Performance" },
-  { message = "^doc", group = "<!-- 4 -->Documentation" },
-  { message = "^refactor", group = "<!-- 5 -->Refactor" },
-  { message = "^test", group = "<!-- 6 -->Testing" },
-  { message = "^build", group = "<!-- 7 -->Build System and CI" },
-  { message = "^ci", group = "<!-- 7 -->Build System and CI" },
-  { message = ".*", group = "<!-- 8 -->Other" },
+  { group = "<!-- 0 -->Breaking", message = "!:" },
+  { group = "<!-- 1 -->Features", message = "^feat" },
+  { group = "<!-- 2 -->Bug Fixes", message = "^fix" },
+  { group = "<!-- 3 -->Performance", message = "^perf" },
+  { group = "<!-- 4 -->Documentation", message = "^doc" },
+  { group = "<!-- 5 -->Refactor", message = "^refactor" },
+  { group = "<!-- 6 -->Testing", message = "^test" },
+  { group = "<!-- 7 -->Build System and CI", message = "^build" },
+  { group = "<!-- 7 -->Build System and CI", message = "^ci" },
+  { group = "<!-- 8 -->Other", message = ".*" },
 ]
 # filter out the commits that are not matched by commit parsers
 filter_commits = false

.github/dependabot.yml

@@ -4,6 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 3
     commit-message:
       prefix: "build(deps)"
     labels:
@@ -12,10 +14,16 @@ updates:
     groups:
       cargo:
         patterns: ["*"]
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major", "version-update:semver-minor"]
+
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 3
     commit-message:
       prefix: "ci"
     labels:
@@ -24,3 +32,22 @@ updates:
     groups:
       actions:
         patterns: ["*"]
+
+  - package-ecosystem: "npm"
+    versioning-strategy: increase
+    directories:
+      - "/crates/npm"
+      - "/crates/eslint"
+      - "/lib/binding_web"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 3
+    commit-message:
+      prefix: "build(deps)"
+    labels:
+      - "dependencies"
+      - "npm"
+    groups:
+      npm:
+        patterns: ["*"]

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+### AI Policy
+
+- [ ] I have read the [AI Policy](https://tree-sitter.github.io/tree-sitter/6-contributing.html#ai-policy) and this PR complies with it.
+- [ ] If AI tools were used: I have disclosed the tool and extent of usage below.
+
+<!-- If you used AI tools, state which tool and how it was used. Delete this section if not applicable. -->

.github/scripts/close_spam.js

@@ -0,0 +1,29 @@
+module.exports = async ({ github, context }) => {
+  let target = context.payload.issue;
+  if (target) {
+    await github.rest.issues.update({
+      ...context.repo,
+      issue_number: target.number,
+      state: "closed",
+      state_reason: "not_planned",
+      title: "[spam]",
+      body: "",
+      type: null,
+    });
+  } else {
+    target = context.payload.pull_request;
+    await github.rest.pulls.update({
+      ...context.repo,
+      pull_number: target.number,
+      state: "closed",
+      title: "[spam]",
+      body: "",
+    });
+  }
+
+  await github.rest.issues.lock({
+    ...context.repo,
+    issue_number: target.number,
+    lock_reason: "spam",
+  });
+};

.github/scripts/cross.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# set -x
-set -e
-
-if [ "$BUILD_CMD" != "cross" ]; then
-    echo "cross.sh - is a helper to assist only in cross compiling environments" >&2
-    echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2
-    exit 111
-fi
-
-if [ -z "$CROSS_IMAGE" ]; then
-    echo "The CROSS_IMAGE env var should be provided" >&2
-    exit 111
-fi
-
-docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/scripts/make.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# set -x
-set -e
-
-if [ "$BUILD_CMD" == "cross" ]; then
-    if [ -z "$CC" ]; then
-        echo "make.sh: CC is not set" >&2
-        exit 111
-    fi
-    if [ -z "$AR" ]; then
-        echo "make.sh: AR is not set" >&2
-        exit 111
-    fi
-
-    cross.sh make CC=$CC AR=$AR "$@"
-else
-    make "$@"
-fi

.github/scripts/reviewers_remove.js

@@ -1,16 +0,0 @@
-module.exports = async ({ github, context }) => {
-  const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
-    owner: context.repo.owner,
-    repo: context.repo.repo,
-    pull_number: context.issue.number,
-  });
-
-  const reviewers = requestedReviewers.data.users.map((e) => e.login);
-
-  github.rest.pulls.removeRequestedReviewers({
-    owner: context.repo.owner,
-    repo: context.repo.repo,
-    pull_number: context.issue.number,
-    reviewers: reviewers,
-  });
-};

.github/scripts/tree-sitter.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# set -x
-set -e
-
-if [ -z "$ROOT" ]; then
-    echo "The ROOT env var should be set to absolute path of a repo root folder" >&2
-    exit 111
-fi
-
-if [ -z "$TARGET" ]; then
-    echo "The TARGET env var should be equal to a \`cargo build --target <TARGET>\` command value" >&2
-    exit 111
-fi
-
-tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
-
-if [ "$BUILD_CMD" == "cross" ]; then
-    if [ -z "$CROSS_RUNNER" ]; then
-        echo "The CROSS_RUNNER env var should be set to a CARGO_TARGET_*_RUNNER env var value" >&2
-        echo "that is available in a docker image used by the cross tool under the hood" >&2
-        exit 111
-    fi
-
-    cross.sh $CROSS_RUNNER "$tree_sitter" "$@"
-else
-    "$tree_sitter" "$@"
-fi

.github/scripts/wasm_stdlib.js

@@ -0,0 +1,35 @@
+module.exports = async ({ github, context, core }) => {
+  if (context.eventName !== 'pull_request') return;
+
+  const prNumber = context.payload.pull_request.number;
+  const owner = context.repo.owner;
+  const repo = context.repo.repo;
+
+  const files = await github.paginate(github.rest.pulls.listFiles, {
+    owner,
+    repo,
+    pull_number: prNumber,
+    per_page: 100
+  });
+
+  const changedFiles = files.map(file => file.filename);
+
+  const wasmStdLibSources = [
+    'lib/src/wasm-stdlib/external_scanner_allocator.c',
+    'lib/src/wasm-stdlib/imports.txt',
+    'lib/src/wasm-stdlib/libc.c',
+    'lib/src/wasm-stdlib/stdio.c'
+  ];
+  const dirChanged = changedFiles.some(file =>
+    wasmStdLibSources.includes(file) ||
+    file.startsWith('lib/src/wasm-stdlib/libc/ctype/') ||
+    file.startsWith('lib/src/wasm-stdlib/libc/string/')
+  );
+
+  if (!dirChanged) return;
+
+  const wasmStdLibHeader = 'lib/src/wasm-stdlib/external_scanner_stdlib.h';
+  const requiredChanged = changedFiles.includes(wasmStdLibHeader);
+
+  if (!requiredChanged) core.setFailed(`Changes detected in the Wasm stdlib sources but ${wasmStdLibHeader} was not modified.`);
+};

.github/workflows/backport.yml

@@ -1,27 +1,33 @@
-name: backport
+name: Backport Pull Request
+
 on:
   pull_request_target:
     types: [closed, labeled]
+
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   backport:
-    permissions:
-      contents: write
-      pull-requests: write
-    name: Backport Pull Request
     if: github.event.pull_request.merged
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+        with:
+          persist-credentials: true
+          ref: ${{ github.event.pull_request.base.ref }}
 
-      - uses: actions/create-github-app-token@v1
+      - name: Create app token
+        uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ vars.BACKPORT_APP }}
           private-key: ${{ secrets.BACKPORT_KEY }}
 
       - name: Create backport PR
-        id: backport
-        uses: korthout/backport-action@v3
+        uses: korthout/backport-action@v4.6.0
         with:
           pull_title: "${pull_title}"
           label_pattern: "^ci:backport ([^ ]+)$"

.github/workflows/bindgen.yml

@@ -0,0 +1,30 @@
+name: Check Bindgen Output
+
+on:
+  pull_request:
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_rust/bindings.rs
+  push:
+    branches: [master]
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_rust/bindings.rs
+
+jobs:
+  check-bindgen:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Generate bindings
+        run: cargo xtask generate-bindings
+
+      - name: Check if the bindgen output changed
+        run: git diff --exit-code lib/binding_rust/bindings.rs

.github/workflows/build.yml

@@ -1,14 +1,9 @@
 name: Build & Test
 
-env:
-  CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-D warnings"
-  CROSS_DEBUG: 1
-
 on:
   workflow_call:
     inputs:
-      run_test:
+      run-test:
         default: true
         type: boolean
 
@@ -21,168 +16,323 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-        - linux-arm64       #
-        - linux-arm         #
-        - linux-x64         #
-        - linux-x86         #
-        - linux-powerpc64   #
-        - windows-arm64     #
-        - windows-x64       # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler
-        - windows-x86       #     -- // --
-        - macos-arm64       #
-        - macos-x64         #
+          - linux-arm64
+          - linux-arm
+          - linux-x64
+          - linux-x86
+          - linux-powerpc64
+          - windows-arm64
+          - windows-x64
+          - windows-x86
+          - macos-arm64
+          - macos-x64
+          - illumos-x64
+          - wasm32
 
         include:
-        # When adding a new `target`:
-        # 1. Define a new platform alias above
-        # 2. Add a new record to a matrix map in `cli/npm/install.js`
-        - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu           , os: ubuntu-latest  , use-cross: true    }
-        - { platform: linux-arm         , target: arm-unknown-linux-gnueabi           , os: ubuntu-latest  , use-cross: true    }
-        - { platform: linux-x64         , target: x86_64-unknown-linux-gnu            , os: ubuntu-20.04   , cli_features: wasm } #2272
-        - { platform: linux-x86         , target: i686-unknown-linux-gnu              , os: ubuntu-latest  , use-cross: true    }
-        - { platform: linux-powerpc64   , target: powerpc64-unknown-linux-gnu         , os: ubuntu-latest  , use-cross: true    }
-        - { platform: windows-arm64     , target: aarch64-pc-windows-msvc             , os: windows-latest                      }
-        - { platform: windows-x64       , target: x86_64-pc-windows-msvc              , os: windows-latest , cli_features: wasm }
-        - { platform: windows-x86       , target: i686-pc-windows-msvc                , os: windows-latest                      }
-        - { platform: macos-arm64       , target: aarch64-apple-darwin                , os: macos-14       , cli_features: wasm }
-        - { platform: macos-x64         , target: x86_64-apple-darwin                 , os: macos-12       , cli_features: wasm }
+          # When adding a new `target`:
+          # 1. Define a new platform alias above
+          # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
+          # 3. Consider adding the mapping at the end of 'crates/cli/Cargo.toml' for cargo-binstall support
+          - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
+          - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
+          - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
+          - { platform: linux-x86       , target: i686-unknown-linux-gnu        , os: ubuntu-24.04     }
+          - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu   , os: ubuntu-24.04     }
+          - { platform: windows-arm64   , target: aarch64-pc-windows-msvc       , os: windows-11-arm   }
+          - { platform: windows-x64     , target: x86_64-pc-windows-msvc        , os: windows-2025     }
+          - { platform: windows-x86     , target: i686-pc-windows-msvc          , os: windows-2025     }
+          - { platform: macos-arm64     , target: aarch64-apple-darwin          , os: macos-15         }
+          - { platform: macos-x64       , target: x86_64-apple-darwin           , os: macos-15-intel   }
+          - { platform: wasm32          , target: wasm32-unknown-unknown        , os: ubuntu-24.04     }
 
-        # Cross compilers for C library
-        - { platform: linux-arm64       , cc: aarch64-linux-gnu-gcc             , ar: aarch64-linux-gnu-ar           }
-        - { platform: linux-arm         , cc: arm-linux-gnueabi-gcc             , ar: arm-linux-gnueabi-ar           }
-        - { platform: linux-x86         , cc: i686-linux-gnu-gcc                , ar: i686-linux-gnu-ar              }
-        - { platform: linux-powerpc64   , cc: powerpc64-linux-gnu-gcc           , ar: powerpc64-linux-gnu-ar         }
+          # illumos is not supported OOTB, it runs in a vm
+          - { platform: illumos-x64     , target: x86_64-unknown-illumos        , os: ubuntu-24.04     , vm: true , no-run: true }
 
-        # See #2041 tree-sitter issue
-        - { platform: windows-x64   , rust-test-threads: 1 }
-        - { platform: windows-x86   , rust-test-threads: 1 }
+          # Extra features
+          - { platform: linux-arm64     , features: wasm }
+          - { platform: linux-x64       , features: wasm , run-wasm-test: true }
+          - { platform: macos-arm64     , features: wasm , run-wasm-test: true }
+          - { platform: macos-x64       , features: wasm }
 
-        # CLI only build
-        - { platform: windows-arm64 , cli-only: true }
+          # Cross-compilation
+          - { platform: linux-arm       , cross: true }
+          - { platform: linux-x86       , cross: true }
+          - { platform: linux-powerpc64 , cross: true }
+
+          # Compile-only
+          - { platform: wasm32          , no-run: true }
 
     env:
-      BUILD_CMD: cargo
-      EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
+      CARGO_TERM_COLOR: always
+      RUSTFLAGS: -D warnings
 
     defaults:
       run:
         shell: bash
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout repository
+      uses: actions/checkout@v7.0.1
 
-    - name: Read Emscripten version
-      run: echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV
+    - name: Set up cross-compilation
+      if: matrix.cross
+      run: |
+        for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do
+          camel_target=${target//-/_}; target_cc=${target/-unknown/}
+          printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
+          printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
+          printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
+        done >> $GITHUB_ENV
+        {
+          printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
+          printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
+        } >> $GITHUB_ENV
+
+    - name: Get emscripten version
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
+
+    - name: Cache Emscripten SDK
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/cache@v6
+      with:
+        path: emsdk
+        key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
 
     - name: Install Emscripten
-      if: ${{ !matrix.cli-only && !matrix.use-cross }}
-      uses: mymindstorm/setup-emsdk@v14
-      with:
-        version: ${{ env.EMSCRIPTEN_VERSION }}
-
-    - run: rustup toolchain install stable --profile minimal
-    - run: rustup target add ${{ matrix.target }}
-    - uses: Swatinem/rust-cache@v2
-
-    - name: Install cross
-      if: ${{ matrix.use-cross }}
-      uses: taiki-e/install-action@v2
-      with:
-        tool: cross
-
-    - name: Build custom cross image
-      if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }}
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
       run: |
-        target="${{ matrix.target }}"
-        image=ghcr.io/cross-rs/$target:custom
-        echo "CROSS_IMAGE=$image"                              >> $GITHUB_ENV
-
-        echo "[target.$target]"                                >> Cross.toml
-        echo "image = \"$image\""                              >> Cross.toml
-        echo "CROSS_CONFIG=$PWD/Cross.toml"                    >> $GITHUB_ENV
-
-        echo "FROM ghcr.io/cross-rs/$target:edge"                            >> Dockerfile
-        echo "RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -" >> Dockerfile
-        echo "RUN apt-get update && apt-get -y install nodejs"               >> Dockerfile
-        docker build -t $image .
-
-    - name: Setup env extras
-      env:
-        RUST_TEST_THREADS: ${{ matrix.rust-test-threads }}
-        USE_CROSS: ${{ matrix.use-cross }}
-        TARGET: ${{ matrix.target }}
-        CC: ${{ matrix.cc }}
-        AR: ${{ matrix.ar }}
-      run: |
-        PATH="$PWD/.github/scripts:$PATH"
-        echo "$PWD/.github/scripts" >> $GITHUB_PATH
-
-        echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
-        echo "TARGET=$TARGET" >> $GITHUB_ENV
-        echo "ROOT=$PWD" >> $GITHUB_ENV
-
-        [ -n "$RUST_TEST_THREADS" ] && \
-        echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV
-
-        [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
-        [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
-
-        if [ "$USE_CROSS" == "true" ]; then
-          echo "BUILD_CMD=cross" >> $GITHUB_ENV
-          runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
-          [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
+        if [[ ! -d emsdk ]]; then
+          git clone --depth 1 https://github.com/emscripten-core/emsdk.git
         fi
+        cd emsdk
+        ./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
+        ./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
+        echo "$PWD" >> "$GITHUB_PATH"
+        echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
+        echo "EMSDK=$PWD" >> "$GITHUB_ENV"
+        echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
 
-    - name: Build C library
-      if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
-      run: make.sh -j CFLAGS="-Werror"
+    - name: Set up Node.js
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      uses: actions/setup-node@v7.0.0
+      with:
+        node-version: 24
+        cache: npm
+        cache-dependency-path: lib/binding_web/package-lock.json
 
-    - name: Build wasm library
-      if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times
-      run: script/build-wasm
+    - name: Set up Rust
+      if: ${{ !matrix.vm }}
+      uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        target: ${{ matrix.target }}
 
-    - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
+    - name: Install Rust Wasm test target
+      if: matrix.run-wasm-test
+      run: rustup toolchain install nightly --profile minimal --component rust-src
 
-    - run: script/fetch-fixtures
+    - name: Install cross-compilation toolchain
+      if: matrix.cross
+      run: |
+        sudo apt-get update -qy
+        if [[ $PLATFORM == linux-arm ]]; then
+          sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user
+        elif [[ $PLATFORM == linux-x86 ]]; then
+          sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu
+        elif [[ $PLATFORM == linux-powerpc64 ]]; then
+          sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user
+        fi
+      env:
+        PLATFORM: ${{ matrix.platform }}
 
-    - uses: ./.github/actions/cache
+    - name: Install MinGW and Clang (Windows x64 MSYS2)
+      if: matrix.platform == 'windows-x64'
+      uses: msys2/setup-msys2@v2
+      with:
+        install: |
+          mingw-w64-x86_64-toolchain
+          mingw-w64-x86_64-clang
+          mingw-w64-x86_64-make
+          mingw-w64-x86_64-cmake
+
+    - name: Build C library (Windows x64 MSYS2 CMake)
+      if: matrix.platform == 'windows-x64'
+      shell: msys2 {0}
+      run: |
+        cmake -G Ninja -S . -B build/static \
+          -DBUILD_SHARED_LIBS=OFF \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM \
+          -DCMAKE_C_COMPILER=clang
+        cmake --build build/static
+
+        cmake -G Ninja -S . -B build/shared \
+          -DBUILD_SHARED_LIBS=ON \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM \
+          -DCMAKE_C_COMPILER=clang
+        cmake --build build/shared
+        rm -rf \
+          build/{static,shared} \
+          "${CMAKE_PREFIX_PATH}/artifacts" \
+          target/wasmtime-${WASMTIME_VERSION}
+      env:
+        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
+
+    - name: Download wasmtime C API
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      run: |
+        mkdir -p target
+        WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
+                           jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
+        case '${{ matrix.target }}' in
+          x86_64-unknown-linux-gnu)  WT_TARGET=x86_64-linux ;;
+          aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
+          x86_64-apple-darwin)       WT_TARGET=x86_64-macos ;;
+          aarch64-apple-darwin)      WT_TARGET=aarch64-macos ;;
+        esac
+        curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
+          | tar xJf - -C target
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
+      env:
+        WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
+
+    - name: Build C library (make)
+      if: runner.os != 'Windows' && !matrix.vm
+      run: |
+        if [[ $PLATFORM == linux-arm ]]; then
+          CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar
+        elif [[ $PLATFORM == linux-x86 ]]; then
+          CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar
+        elif [[ $PLATFORM == linux-powerpc64 ]]; then
+          CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar
+        else
+          CC=gcc; AR=ar
+        fi
+        make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
+      env:
+        PLATFORM: ${{ matrix.platform }}
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
+
+    - name: Build C library (CMake)
+      if: "!matrix.cross && !matrix.vm"
+      run: |
+        cmake -S . -B build/static \
+          -DBUILD_SHARED_LIBS=OFF \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM
+        cmake --build build/static --verbose
+
+        cmake -S . -B build/shared \
+          -DBUILD_SHARED_LIBS=ON \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM
+        cmake --build build/shared --verbose
+      env:
+        CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
+        WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
+
+    - name: Build C library and Rust crate (illumos gmake)
+      if: matrix.platform == 'illumos-x64'
+      uses: vmactions/omnios-vm@v1.3.6
+      with:
+        release: r151056-build
+        copyback: false
+        prepare: |
+          pkg install -q build-essential || [ $? -eq 4 ]
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
+        run: |
+          . "$HOME/.cargo/env"
+          gmake -j
+          cargo build -p tree-sitter
+
+    - name: Build Wasm library
+      if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
+      shell: bash
+      run: |
+        cd lib/binding_web
+        npm ci
+        CJS=true npm run build
+        CJS=true npm run build:debug
+        npm run build
+        npm run build:debug
+
+    - name: Check no_std builds
+      if: inputs.run-test && !matrix.no-run
+      working-directory: lib
+      shell: bash
+      run: cargo check --no-default-features --target='${{ matrix.target }}'
+
+    - name: Build target
+      if: "!inputs.run-test && !matrix.vm"
+      run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
+      env:
+        PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
+
+    - name: Cache fixtures
       id: cache
+      if: inputs.run-test && !matrix.no-run
+      uses: ./.github/actions/cache
+
+    - name: Fetch fixtures
+      if: inputs.run-test && !matrix.no-run
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures
 
     - name: Generate fixtures
-      if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host
-      run: script/generate-fixtures
+      if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true'
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
 
-    - name: Generate WASM fixtures
-      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step
-      run: script/generate-fixtures-wasm
+    - name: Generate Wasm fixtures
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
 
     - name: Run main tests
-      if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host
-      run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
+      if: inputs.run-test && !matrix.no-run
+      run: cargo test --workspace --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
 
-    - name: Run wasm tests
-      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step
-      run: script/test-wasm
+    - name: Run Wasm tests
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
 
-    - name: Run benchmarks
-      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense
-      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
+    - name: Run Rust Wasm web test
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-rust-wasm-web
 
     - name: Upload CLI artifact
-      uses: actions/upload-artifact@v4
+      if: "!inputs.run-test && !matrix.no-run"
+      uses: actions/upload-artifact@v7
       with:
         name: tree-sitter.${{ matrix.platform }}
-        path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
         if-no-files-found: error
         retention-days: 7
 
-    - name: Upload WASM artifacts
-      if: ${{ matrix.platform == 'linux-x64' }}
-      uses: actions/upload-artifact@v4
+    - name: Upload Wasm artifacts
+      if: "!inputs.run-test && matrix.platform == 'linux-x64'"
+      uses: actions/upload-artifact@v7
       with:
         name: tree-sitter.wasm
         path: |
-          lib/binding_web/tree-sitter.js
-          lib/binding_web/tree-sitter.wasm
+          lib/binding_web/web-tree-sitter.js
+          lib/binding_web/web-tree-sitter.js.map
+          lib/binding_web/web-tree-sitter.cjs
+          lib/binding_web/web-tree-sitter.cjs.map
+          lib/binding_web/web-tree-sitter.wasm
+          lib/binding_web/web-tree-sitter.wasm.map
+          lib/binding_web/debug/web-tree-sitter.cjs
+          lib/binding_web/debug/web-tree-sitter.cjs.map
+          lib/binding_web/debug/web-tree-sitter.js
+          lib/binding_web/debug/web-tree-sitter.js.map
+          lib/binding_web/debug/web-tree-sitter.wasm
+          lib/binding_web/debug/web-tree-sitter.wasm.map
+          lib/binding_web/lib/*.c
+          lib/binding_web/lib/*.h
+          lib/binding_web/lib/*.ts
+          lib/binding_web/src/*.ts
         if-no-files-found: error
         retention-days: 7

.github/workflows/ci.yml

@@ -1,9 +1,14 @@
 name: CI
+
 on:
-  pull_request:
   push:
     branches:
       - 'master'
+      - 'release-[0-9]+.[0-9]+'
+  pull_request:
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -13,15 +18,34 @@ jobs:
   checks:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - run: rustup toolchain install stable --profile minimal
-    - run: rustup toolchain install nightly --profile minimal
-    - run: rustup component add --toolchain nightly rustfmt
-    - uses: Swatinem/rust-cache@v2
-    - run: make lint
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+          components: clippy, rustfmt
+
+      - name: Lint Rust files
+        run: make lint
+
+      - name: Install Taplo
+        uses: taiki-e/install-action@v2
+        with:
+          tool: taplo@0.10.0
+
+      - name: Lint TOML files
+        run: make lint-toml
+
+      - name: Lint web files
+        run: make lint-web
 
   sanitize:
     uses: ./.github/workflows/sanitize.yml
 
   build:
     uses: ./.github/workflows/build.yml
+
+  check-wasm-stdlib:
+    uses: ./.github/workflows/wasm_stdlib.yml

.github/workflows/crate_versions.yml

@@ -0,0 +1,22 @@
+name: Crate Versions Check
+
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+  workflow_dispatch:
+
+jobs:
+  check-crates:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Check crates against crates.io
+        uses: katyo/publish-crates@v2
+        with:
+          dry-run: true

.github/workflows/docs.yml

@@ -0,0 +1,44 @@
+name: Deploy Docs
+on:
+  push:
+    branches: [master]
+    paths: [docs/**]
+  workflow_dispatch:
+
+jobs:
+  deploy-docs:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Install mdbook
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
+          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.5.4 --jq "$jq_expr")
+          mkdir mdbook
+          curl -sSL "$url" | tar -xz -C mdbook
+          printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
+
+      - name: Build Book
+        run: mdbook build docs
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v6
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v5
+        with:
+          path: docs/book
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5

.github/workflows/nvim_ts.yml

@@ -0,0 +1,96 @@
+name: nvim-treesitter parser tests
+
+on:
+  pull_request:
+    paths:
+      - 'lib/**'
+      - 'crates/cli/**'
+      - 'crates/config/**'
+      - 'crates/generate/**'
+      - 'crates/loader/**'
+      - '.github/workflows/nvim_ts.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check_compilation:
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        type: [generate, build]
+    name: ${{ matrix.os }} - ${{ matrix.type }}
+    runs-on: ${{ matrix.os }}
+    env:
+      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim.exe' || 'nvim' }}
+      NVIM_TAG: stable
+      NVIM_DIR: neovim
+      NVIM_TS_DIR: nvim-treesitter
+    steps:
+      - uses: actions/checkout@v7.0.1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build --profile optimize
+
+      - name: Clone Neovim
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: neovim/neovim
+          ref: ${{ env.NVIM_TAG }}
+          path: ${{ env.NVIM_DIR }}
+
+      - if: runner.os != 'Windows'
+        name: Setup environment (Posix)
+        run: |
+          echo ${{ github.workspace }}/target/optimize >> "$GITHUB_PATH"
+          echo ${{ github.workspace }}/neovim/build/bin >> "$GITHUB_PATH"
+          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$GITHUB_ENV"
+
+      - if: runner.os == 'Windows'
+        name: Setup environment (why can't you just be normal?!)
+        run: |
+          ${{ env.NVIM_DIR }}/.github/scripts/env.ps1
+          echo ${{ github.workspace }}/target/optimize >> "$env:GITHUB_PATH"
+          echo ${{ github.workspace }}/neovim/build/bin >> "$env:GITHUB_PATH"
+          echo "VIMRUNTIME=${{ github.workspace }}/neovim/runtime" >> "$env:GITHUB_ENV"
+
+      - name: Build Neovim
+        working-directory: ${{ env.NVIM_DIR }}
+        run: |
+          cmake -S cmake.deps -B .deps -G Ninja -D CMAKE_BUILD_TYPE=Release -D TREESITTER_URL=https://github.com/tree-sitter/tree-sitter/archive/${{ github.event.pull_request.head.sha }}.tar.gz -D DEPS_IGNORE_SHA=TRUE
+          cmake --build .deps --config Release
+          cmake -B build -G Ninja -D CMAKE_BUILD_TYPE=Release
+          cmake --build build --config Release
+
+      - name: Clone nvim-treesitter
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: nvim-treesitter/nvim-treesitter
+          path: ${{ env.NVIM_TS_DIR }}
+
+      - if: matrix.type == 'generate'
+        name: Generate and compile parsers
+        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=10
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: matrix.type == 'build'
+        name: Compile parsers
+        run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: "!cancelled()"
+        name: Test parsers
+        run: $NVIM -l ./scripts/check-parsers.lua
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: "!cancelled()"
+        name: Test queries
+        run: $NVIM -l ./scripts/check-queries.lua
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash

.github/workflows/release.yml

@@ -1,6 +1,14 @@
 name: Release
+
 on:
+  schedule:
+    - cron: '5 5 * * *'
   workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Tag name for release'
+        required: false
+        default: nightly
   push:
     tags:
       - v[0-9]+.[0-9]+.[0-9]+
@@ -9,19 +17,33 @@ jobs:
   build:
     uses: ./.github/workflows/build.yml
     with:
-      run_test: false
+      run-test: false
 
   release:
-    name: Release
+    name: Release on GitHub
     runs-on: ubuntu-latest
     needs: build
     permissions:
+      id-token: write
+      attestations: write
       contents: write
     steps:
-      - uses: actions/checkout@v4
+      - if: github.event_name == 'workflow_dispatch'
+        env:
+          TAG_NAME: ${{ github.event.inputs.tag_name }}
+        run: echo "TAG_NAME=${TAG_NAME}" >> $GITHUB_ENV
+
+      - if: github.event_name == 'schedule'
+        run: echo 'TAG_NAME=nightly' >> $GITHUB_ENV
+
+      - if: github.event_name == 'push'
+        run: echo "TAG_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           path: artifacts
 
@@ -31,70 +53,109 @@ jobs:
 
       - name: Prepare release artifacts
         run: |
-          mkdir -p target
-          mv artifacts/tree-sitter.wasm/* target/
+          mkdir -p target web
+          mv artifacts/tree-sitter.wasm/* web/
+
+          tar -czf target/web-tree-sitter.tar.gz -C web .
+
           rm -r artifacts/tree-sitter.wasm
+
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
+            chmod +x $exe
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
+            zip -j9 target/tree-sitter-cli-$platform.zip $exe
           done
           rm -rf artifacts
           ls -l target/
 
-      - name: Create release
-        uses: softprops/action-gh-release@v2
+      - name: Generate attestations
+        uses: actions/attest-build-provenance@v4
         with:
-          name: ${{ github.ref_name }}
-          tag_name: ${{ github.ref_name }}
-          fail_on_unmatched_files: true
-          files: |
+          subject-path: |
             target/tree-sitter-*.gz
-            target/tree-sitter.wasm
-            target/tree-sitter.js
+            target/tree-sitter-cli-*.zip
+            target/web-tree-sitter.tar.gz
+
+      - if: env.TAG_NAME == 'nightly'
+        run: |
+          echo 'PRERELEASE=--prerelease' >> $GITHUB_ENV
+          gh release delete nightly --yes || true
+          git push https://${GITHUB_ACTOR}:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY} :nightly || true
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Create release
+        run: |-
+          gh release create ${{ env.TAG_NAME }} $PRERELEASE \
+            target/tree-sitter-*.gz \
+            target/tree-sitter-cli-*.zip \
+            target/web-tree-sitter.tar.gz
+        env:
+          GH_TOKEN: ${{ github.token }}
 
   crates_io:
-    name: Publish CLI to Crates.io
+    name: Publish packages to Crates.io
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
+    environment: crates
+    permissions:
+      id-token: write
+      contents: read
     needs: release
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
 
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Set up registry token
+        id: auth
+        uses: rust-lang/crates-io-auth-action@v1
 
       - name: Publish crates to Crates.io
         uses: katyo/publish-crates@v2
         with:
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          registry-token: ${{ steps.auth.outputs.token }}
 
   npm:
-    name: Publish lib to npmjs.com
+    name: Publish packages to npmjs.com
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tag_name != 'nightly')
     runs-on: ubuntu-latest
+    environment: npm
+    permissions:
+      id-token: write
+      contents: read
     needs: release
     strategy:
       fail-fast: false
       matrix:
-        directory: ["cli/npm", "lib/binding_web"]
+        directory: [crates/cli/npm, lib/binding_web]
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up Node
+        uses: actions/setup-node@v7.0.0
+        with:
+          node-version: 24
+          registry-url: https://registry.npmjs.org
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
 
       - name: Build wasm
         if: matrix.directory == 'lib/binding_web'
-        run: ./script/build-wasm
-
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-          registry-url: "https://registry.npmjs.org"
-
-      - name: Publish lib to npmjs.com
-        env:
-          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
         run: |
           cd ${{ matrix.directory }}
-          npm publish
+          npm ci
+          npm run build
+          npm run build:debug
+          CJS=true npm run build
+          CJS=true npm run build:debug
+          npm run build:dts
+
+      - name: Publish to npmjs.com
+        working-directory: ${{ matrix.directory }}
+        run: npm publish

.github/workflows/response.yml

@@ -1,34 +1,47 @@
-name: no_response
+name: No response
+
 on:
   schedule:
-    - cron: '30 1 * * *' # Run every day at 01:30
+    - cron: "30 1 * * *" # Run every day at 01:30
   workflow_dispatch:
   issue_comment:
 
+permissions:
+  issues: write
+  pull-requests: write
+
 jobs:
   close:
+    name: Close issues with no response
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/github-script@v7
+      - name: Checkout script
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts/close_unresponsive.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/close_unresponsive.js')
             await script({github, context})
 
   remove_label:
+    name: Remove response label
     if: github.event_name == 'issue_comment'
     runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/github-script@v7
+      - name: Checkout script
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts/remove_response_label.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v9
         with:
           script: |
             const script = require('./.github/scripts/remove_response_label.js')

.github/workflows/reviewers_remove.yml

@@ -1,17 +1,31 @@
-name: "reviewers: remove"
+name: Remove Reviewers
+
 on:
   pull_request_target:
     types: [converted_to_draft, closed]
+
+permissions:
+  pull-requests: write
+
 jobs:
   remove-reviewers:
     runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
     steps:
-      - uses: actions/checkout@v4
-      - name: 'Remove reviewers'
-        uses: actions/github-script@v7
+      - name: Remove reviewers
+        uses: actions/github-script@v9
         with:
           script: |
-            const script = require('./.github/scripts/reviewers_remove.js')
-            await script({github, context})
+            const requestedReviewers = await github.rest.pulls.listRequestedReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+
+            const reviewers = requestedReviewers.data.users.map((e) => e.login);
+
+            github.rest.pulls.removeRequestedReviewers({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+              reviewers: reviewers,
+            });

.github/workflows/sanitize.yml

@@ -8,39 +8,44 @@ on:
   workflow_call:
 
 jobs:
-  check_undefined_behaviour:
-    name: Sanitizer checks
+  check-undefined-behaviour:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     env:
       TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
     steps:
-    - name: Checkout source code
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
 
-    - name: Install UBSAN library
-      run: sudo apt-get update -y && sudo apt-get install -y libubsan1
+      - name: Install UBSAN library
+        run: sudo apt-get update -y && sudo apt-get install -y libubsan1
 
-    - run: rustup toolchain install stable --profile minimal
-    - uses: Swatinem/rust-cache@v2
-    - run: cargo build --release
-    - run: script/fetch-fixtures
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
 
-    - uses: ./.github/actions/cache
-      id: cache
+      - name: Build project
+        run: cargo build --release
 
-    - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-      run: script/generate-fixtures
+      - name: Cache fixtures
+        uses: ./.github/actions/cache
+        id: cache
 
-    - name: Run main tests with undefined behaviour sanitizer (UBSAN)
-      env:
-        CFLAGS: -fsanitize=undefined
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
-      run: cargo test -- --test-threads 1
+      - name: Fetch fixtures
+        run: cargo xtask fetch-fixtures
 
-    - name: Run main tests with address sanitizer (ASAN)
-      env:
-        ASAN_OPTIONS: verify_asan_link_order=0
-        CFLAGS: -fsanitize=address
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing
-      run: cargo test -- --test-threads 1
+      - name: Generate fixtures
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: cargo xtask generate-fixtures
+
+      - name: Run main tests with undefined behaviour sanitizer (UBSAN)
+        run: cargo test -- --test-threads 1
+        env:
+          CFLAGS: -fsanitize=undefined
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
+
+      - name: Run main tests with address sanitizer (ASAN)
+        run: cargo test -- --test-threads 1
+        env:
+          ASAN_OPTIONS: verify_asan_link_order=0
+          CFLAGS: -fsanitize=address
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing

.github/workflows/spam.yml

@@ -0,0 +1,29 @@
+name: Close as spam
+
+on:
+  issues:
+    types: [labeled]
+  pull_request_target:
+    types: [labeled]
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  spam:
+    runs-on: ubuntu-latest
+    if: github.event.label.name == 'spam'
+    steps:
+      - name: Checkout script
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts/close_spam.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const script = require('./.github/scripts/close_spam.js')
+            await script({github, context})

.github/workflows/wasm_exports.yml

@@ -0,0 +1,41 @@
+name: Check Wasm Exports
+
+on:
+  pull_request:
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_web/**
+      - xtask/src/**
+  push:
+    branches: [master]
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_rust/bindings.rs
+      - CMakeLists.txt
+
+jobs:
+  check-wasm-exports:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Install wasm-objdump
+        run: sudo apt-get update -y && sudo apt-get install -y wabt
+
+      - name: Build C library (make)
+        run: make -j CFLAGS="$CFLAGS"
+        env:
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
+
+      - name: Build Wasm Library
+        working-directory: lib/binding_web
+        run: npm ci && npm run build:debug
+
+      - name: Check Wasm exports
+        run: cargo xtask check-wasm-exports

.github/workflows/wasm_stdlib.yml

@@ -0,0 +1,19 @@
+name: Check Wasm Stdlib build
+
+on:
+  workflow_call:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+
+      - name: Check directory changes
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
+            const script = require(scriptPath);
+            return script({ github, context, core });

.gitignore

@@ -1,10 +1,12 @@
 log*.html
+.direnv
 
 .idea
 *.xcodeproj
 .vscode
 .cache
 .zig-cache
+.direnv
 
 profile*
 fuzz-results
@@ -12,7 +14,6 @@ test/fuzz/out
 test/fixtures/grammars/*
 !test/fixtures/grammars/.gitkeep
 
-package-lock.json
 node_modules
 
 docs/assets/js/tree-sitter.js
@@ -25,6 +26,7 @@ docs/assets/js/tree-sitter.js
 *.dylib
 *.so
 *.so.[0-9]*
+*.dll
 *.o
 *.obj
 *.exp
@@ -34,3 +36,5 @@ docs/assets/js/tree-sitter.js
 .build
 build
 zig-*
+
+/result

.taplo.toml

@@ -0,0 +1,19 @@
+[formatting]
+column_width = 100
+compact_arrays = false
+reorder_inline_tables = true
+reorder_keys = true
+
+[[rule]]
+include = [ "**/Cargo.toml" ]
+keys = [ "package" ]
+
+[rule.formatting]
+reorder_keys = false
+
+[[rule]]
+include = [ "**/Cargo.toml" ]
+keys = [ "profile" ]
+
+[rule.formatting]
+reorder_keys = false

CHANGELOG.md

@@ -1,470 +0,0 @@
-# Changelog
-
-## [0.23.0] - 2024-08-26
-
-### Breaking
-
-- Introduce tree-sitter-language crate for grammar crates to depend on (<https://github.com/tree-sitter/tree-sitter/pull/3069>)
-- Revert interning of a sequence or choice of a single rule (<https://github.com/tree-sitter/tree-sitter/pull/3548>)
-- **bindings**: Use capsules in python (<https://github.com/tree-sitter/tree-sitter/pull/3384>)
-- **dsl**: Support other JS runtimes (<https://github.com/tree-sitter/tree-sitter/pull/3355>)
-
-### Features
-
-- Add `fuzz` subcommand (<https://github.com/tree-sitter/tree-sitter/pull/3385>)
-- Allow external scanners to use the logger (<https://github.com/tree-sitter/tree-sitter/pull/3204>)
-- **bindings**: Add query constants to python
-- **bindings**: Add node, python, swift tests (<https://github.com/tree-sitter/tree-sitter/pull/3178>)
-- **bindings**: Update npm scripts (<https://github.com/tree-sitter/tree-sitter/pull/3210>)
-- **cli**: Bump unicode data to v15.1.0
-- **cli**: Add debug build flag (<https://github.com/tree-sitter/tree-sitter/pull/3279>)
-- **cli**: Attach helpful context when `grammar.json` cannot be found (<https://github.com/tree-sitter/tree-sitter/pull/3405>)
-- **cli**: Add `--show-fields` flag to `test` command (<https://github.com/tree-sitter/tree-sitter/pull/3502>)
-- **lib**: Add `ts_query_end_byte_for_pattern` (<https://github.com/tree-sitter/tree-sitter/pull/3451>)
-- **lib**: Support no_std
-- **zig**: Update outdated path syntax (<https://github.com/tree-sitter/tree-sitter/pull/3383>)
-
-### Bug Fixes
-
-- Always reset to the first language when iterating over language attributes (<https://github.com/tree-sitter/tree-sitter/pull/3375>)
-- Better error when a supertype rule is invalid (<https://github.com/tree-sitter/tree-sitter/pull/3400>)
-- Intern a sequence or choice of a single element the same as the element itself
-- Do not "absorb" rules that consist of a single terminal if the rule is hidden (<https://github.com/tree-sitter/tree-sitter/pull/2577>)
-- **bindings**: Update go bindings (<https://github.com/tree-sitter/tree-sitter/pull/3544>)
-- **cli**: Installation via authenticated proxy (<https://github.com/tree-sitter/tree-sitter/pull/3414>)
-- **cli**: Dedup `preceding_auxiliary_symbols` (<https://github.com/tree-sitter/tree-sitter/pull/3550>)
-- **dsl**: Improve error message when a rule function returns undefined (<https://github.com/tree-sitter/tree-sitter/pull/3452>)
-- **generate**: Rename `cargo.toml` template (<https://github.com/tree-sitter/tree-sitter/pull/3532>)
-- **go**: Update parser name in binding files, add to docs (<https://github.com/tree-sitter/tree-sitter/pull/3547>)
-- **lib**: A null clock must have `tv_nsec` be 0 as well (<https://github.com/tree-sitter/tree-sitter/pull/3372>)
-- **lib**: Restrict pattern_map optimization when a wildcard step has an immediate first child (<https://github.com/tree-sitter/tree-sitter/pull/3440>)
-- **lib**: An empty root node should not precede an empty range (<https://github.com/tree-sitter/tree-sitter/pull/3450>)
-- **lib**: Fix api header C++ interop (<https://github.com/tree-sitter/tree-sitter/pull/3534>)
-- **make**: Fail properly on Windows (<https://github.com/tree-sitter/tree-sitter/pull/3418>)
-- **rust**: Fetch `CARGO_MANIFEST_DIR` at runtime in build script (<https://github.com/tree-sitter/tree-sitter/pull/3352>)
-- **rust**: Fix new clippy warnings (<https://github.com/tree-sitter/tree-sitter/pull/3491>)
-- **test**: Multi-grammar corpus tests are now in the repo root (<https://github.com/tree-sitter/tree-sitter/pull/3342>)
-- **wasm**: Update test
-
-### Performance
-
-- Hoist out common subexpressions in satisfies_text_predicates (<https://github.com/tree-sitter/tree-sitter/pull/3397>)
-
-### Documentation
-
-- Update changelog
-- Remove duplicate pr # in changelog
-- Add note for bullet
-- Fix syntax highlighting unit testing example (<https://github.com/tree-sitter/tree-sitter/pull/3434>)
-- Add tsserver annotation to example (<https://github.com/tree-sitter/tree-sitter/pull/3460>)
-- Fix tree cursor documentation (<https://github.com/tree-sitter/tree-sitter/pull/3324>)
-- Document rust library features (<https://github.com/tree-sitter/tree-sitter/pull/3395>)
-- Clean up binding & parser lists (<https://github.com/tree-sitter/tree-sitter/pull/3443>)
-
-### Refactor
-
-- Remove ansi_term dependency (<https://github.com/tree-sitter/tree-sitter/pull/3387>)
-- Remove difference dependency (<https://github.com/tree-sitter/tree-sitter/pull/3388>)
-- **scripts**: Clean up bash scripts (<https://github.com/tree-sitter/tree-sitter/pull/3231>)
-
-### Testing
-
-- Modernize scanner files (<https://github.com/tree-sitter/tree-sitter/pull/3340>)
-
-### Build System and CI
-
-- **deps**: bump wasmtime, cc, and wasmparser (<https://github.com/tree-sitter/tree-sitter/pull/3529>
-- **bindings**: Use language version in soname (<https://github.com/tree-sitter/tree-sitter/pull/3308>)
-- **lib**: Include the minor in the soname
-- **loader**: Make dependencies optional (<https://github.com/tree-sitter/tree-sitter/pull/1638>)
-- **swift**: Declare header search path (<https://github.com/tree-sitter/tree-sitter/pull/3474>)
-- **wasm**: Don't minify JS (<https://github.com/tree-sitter/tree-sitter/pull/3380>)
-- **wasm**: Bump emscripten to 3.1.64 (<https://github.com/tree-sitter/tree-sitter/pull/3497>)
-- **wasm**: Support big endian machines (<https://github.com/tree-sitter/tree-sitter/pull/3492>)
-- **zig**: Git ignore updated Zig cache directory (<https://github.com/tree-sitter/tree-sitter/pull/3408>)
-
-### Other
-
-- Swap `sprintf()` for `snprintf()` (<https://github.com/tree-sitter/tree-sitter/pull/3430>)
-- Add `.build` to gitignore (<https://github.com/tree-sitter/tree-sitter/pull/3498>)
-- Reset language when resetting wasm store (<https://github.com/tree-sitter/tree-sitter/pull/3495>)
-- Clone wasm store engine (<https://github.com/tree-sitter/tree-sitter/pull/3542>)
-- **bindings**: Fix indent & line endings (<https://github.com/tree-sitter/tree-sitter/pull/3284>)
-
-## [0.22.6] — 2024-05-05
-
-### Features
-
-- Improve handling of serialization buffer overflows (<https://github.com/tree-sitter/tree-sitter/pull/3318>)
-- Reverse iteration through node parents (<https://github.com/tree-sitter/tree-sitter/pull/3214>)
-- **cli**: Support `NO_COLOR` (<https://github.com/tree-sitter/tree-sitter/pull/3299>)
-- **cli**: Add test listing and allow users to parse a specific test number (<https://github.com/tree-sitter/tree-sitter/pull/3067>)
-- **grammar**: Add "inherits" field if available (<https://github.com/tree-sitter/tree-sitter/pull/3295>)
-
-### Bug Fixes
-
-- Correctly load field data from wasm languages
-- Improve error message when the `tree-sitter` field is malformed
-- Don't error out on package.json lookup errors if `--no-bindings` is passed (<https://github.com/tree-sitter/tree-sitter/pull/3323>)
-- **cli**: Keep default cc flags in build
-- **cli**: Properly account for multi-grammar repos when using docker to build a wasm parser (<https://github.com/tree-sitter/tree-sitter/pull/3337>)
-- **generate**: Don't check arbitrarily named dirs
-- **generate**: Take `AsRef<Path>` for the path parameter to avoid clones (<https://github.com/tree-sitter/tree-sitter/pull/3322>)
-- **highlight**: Correct signature of `ts_highlighter_add_language`
-- **lib**: Do not return field names for extras (<https://github.com/tree-sitter/tree-sitter/pull/3330>)
-- **lib**: Advance the lookahead end byte by 4 when there's an invalid code point (<https://github.com/tree-sitter/tree-sitter/pull/3305>)
-- **rust**: Update README example (<https://github.com/tree-sitter/tree-sitter/pull/3307>)
-- **rust**: Use unix + wasi cfg instead of not windows for fd (<https://github.com/tree-sitter/tree-sitter/pull/3304>)
-- **test**: Allow newlines in between test name and attribute
-- **wasm**: Correct `childrenFromFieldXXX` method signatures (<https://github.com/tree-sitter/tree-sitter/pull/3301>)
-- **xtask**: Always bump every crate in tandem
-- **zig**: Make usable as a zig dependency (<https://github.com/tree-sitter/tree-sitter/pull/3315>)
-
-### Documentation
-
-- Mention build command variables
-- Swap `\s` for `\\s` in query example
-- **highlight**: Typo (<https://github.com/tree-sitter/tree-sitter/pull/3290>)
-
-### Refactor
-
-- **tests**: Migrate remaining `grammar.json` tests to `grammar.js` (<https://github.com/tree-sitter/tree-sitter/pull/3325>)
-
-### Build System and CI
-
-- Add nightly rustfmt to workflow for linting (<https://github.com/tree-sitter/tree-sitter/pull/3333>)
-- Fix address sanitizer step (<https://github.com/tree-sitter/tree-sitter/pull/3188>)
-- **deps**: Bump cc from 1.0.92 to 1.0.94 in the cargo group (<https://github.com/tree-sitter/tree-sitter/pull/3298>)
-- **deps**: Bump the cargo group with 6 updates (<https://github.com/tree-sitter/tree-sitter/pull/3313>)
-- **xtask**: Bump `build.zig.zon` version when bumping versions
-
-## [0.22.5] — 2024-04-14
-
-### Bug Fixes
-
-- Avoid generating unused character set constants
-- **cli**: Test parsing on windows (<https://github.com/tree-sitter/tree-sitter/pull/3289>)
-- **rust**: Compilation on wasm32-wasi (<https://github.com/tree-sitter/tree-sitter/pull/3293>)
-
-## [0.22.4] — 2024-04-12
-
-### Bug Fixes
-
-- Fix sorting of transitions within a lex state
-- Include 2-character ranges in array-based state transitions
-
-### Build System and CI
-
-- Always bump at least the patch version in bump xtask
-
-## [0.22.3] — 2024-04-12
-
-### Features
-
-- Add strncat to wasm stdlib
-- Generate simpler code for matching large character sets (<https://github.com/tree-sitter/tree-sitter/pull/3234>)
-- When loading languages via WASM, gracefully handle memory errors and leaks in external scanners (<https://github.com/tree-sitter/tree-sitter/pull/3181>)
-
-### Bug Fixes
-
-- **bindings**: Add utf-8 flag to python & node (<https://github.com/tree-sitter/tree-sitter/pull/3278>)
-- **bindings**: Generate parser.c if missing (<https://github.com/tree-sitter/tree-sitter/pull/3277>)
-- **bindings**: Remove required platforms for swift (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
-- **cli**: Fix mismatched parenthesis when accounting for `&&` (<https://github.com/tree-sitter/tree-sitter/pull/3274>)
-- **lib**: Do not consider childless nodes for ts_node_parent (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **lib**: Properly account for aliased root nodes and root nodes with
-  children in `ts_subtree_string` (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **lib**: Account for the root node of a tree cursor being an alias (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **lib**: Use correct format specifier in log message (<https://github.com/tree-sitter/tree-sitter/pull/3255>)
-- **parser**: Fix variadic macro (<https://github.com/tree-sitter/tree-sitter/pull/3229>)
-- render: Proper function prototypes (<https://github.com/tree-sitter/tree-sitter/pull/3277>)
-- **windows**: Add `/utf-8` flag for parsers using unicode symbols (<https://github.com/tree-sitter/tree-sitter/pull/3223>)
-- Add a semicolon after SKIP macros (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
-- Add back `build-wasm` temporarily (<https://github.com/tree-sitter/tree-sitter/pull/3203>)
-- Add lifetime to matches function (<https://github.com/tree-sitter/tree-sitter/pull/3254>)
-- Default output directory for `build --wasm` should use current_dir (<https://github.com/tree-sitter/tree-sitter/pull/3203>)
-- Fix sorting of wasm stdlib symbols
-- Insert "tree-sitter" section in current directory's package.json if it exists (<https://github.com/tree-sitter/tree-sitter/pull/3224>)
-- Tie the lifetime of the cursor to the query in `QueryCursor::captures()` (<https://github.com/tree-sitter/tree-sitter/pull/3266>)
-- Wrong flag check in `build.rs`
-
-### Performance
-
-- **cli**: Reduced the compile time of generated parsers by generating C code with fewer conditionals (<https://github.com/tree-sitter/tree-sitter/pull/3234>)
-
-### Documentation
-
-- Add NGINX grammar
-
-### Refactor
-
-- **parser**: Make REDUCE macro non-variadic (<https://github.com/tree-sitter/tree-sitter/pull/3280>)
-- **js**: Misc fixes & tidying
-- **rust**: Misc fixes & tidying
-
-### Testing
-
-- Add regression test for node parent + string bug (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **test**: Allow colons in test names (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
-
-### Build System and CI
-
-- Upgrade wasmtime
-- Update emscripten version (<https://github.com/tree-sitter/tree-sitter/pull/3272>)
-- **dependabot**: Improve PR labels (<https://github.com/tree-sitter/tree-sitter/pull/3282>)
-
-## [0.22.2] — 2024-03-17
-
-### Breaking
-
-- **cli**: Add a separate build command to compile parsers
-
-### Features
-
-- **bindings/rust**: Expose `Parser::included_ranges`
-- Lower the lib's MSRV (<https://github.com/tree-sitter/tree-sitter/pull/3169>)
-- **lib**: Implement Display for Node (<https://github.com/tree-sitter/tree-sitter/pull/3177>)
-
-### Bug Fixes
-
-- **bindings/wasm**: Fix `Parser.getIncludedRanges()` (<https://github.com/tree-sitter/tree-sitter/pull/3164>)
-- **lib**: Makefile installation on macOS (<https://github.com/tree-sitter/tree-sitter/pull/3167>)
-- **lib**: Makefile installation (<https://github.com/tree-sitter/tree-sitter/pull/3173>)
-- **lib**: Avoid possible UB of calling memset on a null ptr when 0 is passed into `array_grow_by` (<https://github.com/tree-sitter/tree-sitter/pull/3176>)
-- **lib**: Allow hiding symbols (<https://github.com/tree-sitter/tree-sitter/pull/3180>)
-
-### Documentation
-
-- Fix typo (<https://github.com/tree-sitter/tree-sitter/pull/3158>)
-- **licensfe**: Update year (<https://github.com/tree-sitter/tree-sitter/pull/3183>)
-
-### Refactor
-
-- Remove dependency on which crate (<https://github.com/tree-sitter/tree-sitter/pull/3172>)
-- Turbofish styling
-
-### Testing
-
-- Fix header writes (<https://github.com/tree-sitter/tree-sitter/pull/3174>)
-
-### Build System and CI
-
-- Simplify workflows (<https://github.com/tree-sitter/tree-sitter/pull/3002>)
-- **lib**: Allow overriding CFLAGS on the commandline (<https://github.com/tree-sitter/tree-sitter/pull/3159>)
-
-## [0.22.1] — 2024-03-10
-
-### Bug Fixes
-
-- Cli build script behavior on release
-
-## [0.22.0] — 2024-03-10
-
-### Breaking
-
-- Remove top-level `corpus` dir for tests
-  The cli will now only look in `test/corpus` for tests
-- Remove redundant escape regex & curly brace regex preprocessing (<https://github.com/tree-sitter/tree-sitter/pull/2838>)
-- **bindings**: Convert node bindings to NAPI (<https://github.com/tree-sitter/tree-sitter/pull/3077>)
-- **wasm**: Make `current*`, `is*`, and `has*` methods properties (<https://github.com/tree-sitter/tree-sitter/pull/3103>)
-- **wasm**: Keep API in-line with upstream and start aligning with node (<https://github.com/tree-sitter/tree-sitter/pull/3149>)
-
-### Features
-
-- Add xtasks to assist with bumping crates (<https://github.com/tree-sitter/tree-sitter/pull/3065>)
-- Improve language bindings (<https://github.com/tree-sitter/tree-sitter/pull/2438>)
-- Expose the allocator and array header files for external scanners (<https://github.com/tree-sitter/tree-sitter/pull/3063>)
-- Add typings for the node bindings
-- Replace `nan` with `node-addon-api` and conditionally print logs
-- **bindings**: Add more make targets
-- **bindings**: Add peerDependencies for npm
-- **bindings**: Add prebuildify to node
-- **bindings**: Remove dsl types file (<https://github.com/tree-sitter/tree-sitter/pull/3126>)
-- **node**: Type tag the language (<https://github.com/tree-sitter/tree-sitter/pull/3109>)
-- **test**: Add attributes for corpus tests
-
-### Bug Fixes
-
-- Apply some `scan-build` suggestions (unused assignment/garbage access) (<https://github.com/tree-sitter/tree-sitter/pull/3056>)
-- Wrap `||` comparison in parentheses when `&&` is used (<https://github.com/tree-sitter/tree-sitter/pull/3070>)
-- Ignore unused variables in the array macros (<https://github.com/tree-sitter/tree-sitter/pull/3083>)
-- `binding.cc` overwrite should replace `PARSER_NAME` (<https://github.com/tree-sitter/tree-sitter/pull/3116>)
-- Don't use `__declspec(dllexport)` on windows (<https://github.com/tree-sitter/tree-sitter/pull/3128>)
-- Parsers should export the language function on windows
-- Allow the regex `v` flag (<https://github.com/tree-sitter/tree-sitter/pull/3154>)
-- **assertions**: Case shouldn't matter for comment node detection
-- **bindings**: Editorconfig and setup.py fixes (<https://github.com/tree-sitter/tree-sitter/pull/3082>)
-- **bindings**: Insert `types` after `main` if it exists (<https://github.com/tree-sitter/tree-sitter/pull/3122>)
-- **bindings**: Fix template oversights (<https://github.com/tree-sitter/tree-sitter/pull/3155>)
-- **cli**: Only output the sources with `--no-bindings` (<https://github.com/tree-sitter/tree-sitter/pull/3123>)
-- **generate**: Add `.npmignore`, populate Swift's exclude list (<https://github.com/tree-sitter/tree-sitter/pull/3085>)
-- **generate**: Extern allocator functions for the template don't need to be "exported" (<https://github.com/tree-sitter/tree-sitter/pull/3132>)
-- **generate**: Camel case name in `Cargo.toml` description (<https://github.com/tree-sitter/tree-sitter/pull/3140>)
-- **lib**: Include `api.h` so `ts_set_allocator` is visible (<https://github.com/tree-sitter/tree-sitter/pull/3092>)
-
-### Documentation
-
-- Add GitHub user and PR info to the changelog
-- Add css for inline code (<https://github.com/tree-sitter/tree-sitter/pull/2844>)
-- Document test attributes
-- Add `Ohm` language parser
-- Remove duplicate `the`'s (<https://github.com/tree-sitter/tree-sitter/pull/3120>)
-- Add discord and matrix badges (<https://github.com/tree-sitter/tree-sitter/pull/3148>)
-
-### Refactor
-
-- Rename TS_REUSE_ALLOCATOR flag (<https://github.com/tree-sitter/tree-sitter/pull/3088>)
-- Remove extern/const where possible
-- **array**: Use pragma GCC in clang too
-- **bindings**: Remove npmignore (<https://github.com/tree-sitter/tree-sitter/pull/3089>)
-
-### Testing
-
-- Don't use TS_REUSE_ALLOCATOR on Darwin systems (<https://github.com/tree-sitter/tree-sitter/pull/3087>)
-- Add test case for parse stack merging with incorrect error cost bug (<https://github.com/tree-sitter/tree-sitter/pull/3098>)
-
-### Build System and CI
-
-- Improve changelog settings (<https://github.com/tree-sitter/tree-sitter/pull/3064>)
-- Unify crate versions via workspace (<https://github.com/tree-sitter/tree-sitter/pull/3074>)
-- Update `cc` to remove annoying debug output (<https://github.com/tree-sitter/tree-sitter/pull/3075>)
-- Adjust dependabot settings (<https://github.com/tree-sitter/tree-sitter/pull/3079>)
-- Use c11 everywhere
-- Add uninstall command
-- Don't skip tests on failing lint (<https://github.com/tree-sitter/tree-sitter/pull/3102>)
-- Remove unused deps, bump deps, and bump MSRV to 1.74.1 (<https://github.com/tree-sitter/tree-sitter/pull/3153>)
-- **bindings**: Metadata improvements
-- **bindings**: Make everything c11 (<https://github.com/tree-sitter/tree-sitter/pull/3099>)
-- **dependabot**: Update weekly instead of daily (<https://github.com/tree-sitter/tree-sitter/pull/3112>)
-- **deps**: Bump the cargo group with 1 update (<https://github.com/tree-sitter/tree-sitter/pull/3081>)
-- **deps**: Bump the cargo group with 1 update (<https://github.com/tree-sitter/tree-sitter/pull/3097>)
-- **deps**: Bump deps & lockfile (<https://github.com/tree-sitter/tree-sitter/pull/3060>)
-- **deps**: Bump the cargo group with 4 updates (<https://github.com/tree-sitter/tree-sitter/pull/3134>)
-- **lint**: Detect if `Cargo.lock` needs to be updated (<https://github.com/tree-sitter/tree-sitter/pull/3066>)
-- **lint**: Make lockfile check quiet (<https://github.com/tree-sitter/tree-sitter/pull/3078>)
-- **swift**: Move 'cLanguageStandard' behind 'targets' (<https://github.com/tree-sitter/tree-sitter/pull/3101>)
-
-### Other
-
-- Make Node.js language bindings context aware (<https://github.com/tree-sitter/tree-sitter/pull/2841>)
-  They don't have any dynamic global data, so all it takes is just declaring them as such
-- Fix crash when attempting to load ancient languages via wasm (<https://github.com/tree-sitter/tree-sitter/pull/3068>)
-- Use workspace dependencies for internal crates like Tree-sitter (<https://github.com/tree-sitter/tree-sitter/pull/3076>)
-- Remove vendored wasmtime headers (<https://github.com/tree-sitter/tree-sitter/pull/3084>)
-  When building rust binding, use wasmtime headers provided via cargo
-  by the wasmtime-c-api crate.
-- Fix invalid parse stack recursive merging with mismatched error cost (<https://github.com/tree-sitter/tree-sitter/pull/3086>)
-  Allowing this invalid merge caused an invariant to be violated
-  later on during parsing, when handling a later error.
-- Fix regression in `subtree_compare` (<https://github.com/tree-sitter/tree-sitter/pull/3111>)
-- docs: Add `Ohm` language parser (<https://github.com/tree-sitter/tree-sitter/pull/3114>)
-- Delete `binding_files.rs` (<https://github.com/tree-sitter/tree-sitter/pull/3106>)
-- **bindings**: Consistent wording (<https://github.com/tree-sitter/tree-sitter/pull/3096>)
-- **bindings**: Ignore more artifacts (<https://github.com/tree-sitter/tree-sitter/pull/3119>)
-
-## [0.21.0] — 2024-02-21
-
-### Breaking
-
-- Remove the apply-all-captures flag, make last-wins precedence the default
-
-  **NOTE**: This change might cause breakage in your grammar's highlight tests.
-  Just flip the order around of the relevant queries, and keep in mind that the
-  last query that matches will win.
-
-### Features
-
-- Use lockfiles to dedup recompilation
-- Improve error message for files with an unknown grammar path (<https://github.com/tree-sitter/tree-sitter/pull/2475>)
-- Implement first-line-regex (<https://github.com/tree-sitter/tree-sitter/pull/2479>)
-- Error out if an empty string is in the `extras` array
-- Allow specifying an external scanner's files (<https://github.com/tree-sitter/tree-sitter/pull/3031>)
-- Better error info when a scanner is missing required symbols
-- **cli**: Add an optional `grammar-path` argument for the playground (<https://github.com/tree-sitter/tree-sitter/pull/3014>)
-- **cli**: Add optional `config-path` argument (<https://github.com/tree-sitter/tree-sitter/pull/3050>)
-- **loader**: Add more commonly used default parser directories
-
-### Bug Fixes
-
-- Prettify xml output and add node position info (<https://github.com/tree-sitter/tree-sitter/pull/2970>)
-- Inherited grammar generation
-- Properly error out when the word property is an invalid rule
-- Update schema for regex flags (<https://github.com/tree-sitter/tree-sitter/pull/3006>)
-- Properly handle `Query.matches` when filtering out results (<https://github.com/tree-sitter/tree-sitter/pull/3013>)
-- Sexp format edge case with quoted closed parenthesis (<https://github.com/tree-sitter/tree-sitter/pull/3016>)
-- Always push the default files if there's no `externals`
-- Don't log NUL characters (<https://github.com/tree-sitter/tree-sitter/pull/3037>)
-- Don't throw an error if the user uses `map` in the grammar (<https://github.com/tree-sitter/tree-sitter/pull/3041>)
-- Remove redundant imports (<https://github.com/tree-sitter/tree-sitter/pull/3047>)
-- **cli**: Installation via a HTTP tunnel proxy (<https://github.com/tree-sitter/tree-sitter/pull/2824>)
-- **cli**: Don't update tests automatically if parse errors are detected (<https://github.com/tree-sitter/tree-sitter/pull/3033>)
-- **cli**: Don't use `long` for `grammar_path`
-- **test**: Allow writing updates to tests without erroneous nodes instead of denying all of them if a single error is found
-- **test**: Edge case when parsing `UNEXPECTED`/`MISSING` nodes with an indentation level greater than 0
-- **wasm**: Remove C++ mangled symbols (<https://github.com/tree-sitter/tree-sitter/pull/2971>)
-
-### Documentation
-
-- Create issue template (<https://github.com/tree-sitter/tree-sitter/pull/2978>)
-- Document regex limitations
-- Mention that `token($.foo)` is illegal
-- Explicitly mention behavior of walking outside the given "root" node for a `TSTreeCursor` (<https://github.com/tree-sitter/tree-sitter/pull/3021>)
-- Small fixes (<https://github.com/tree-sitter/tree-sitter/pull/2987>)
-- Add `Tact` language parser (<https://github.com/tree-sitter/tree-sitter/pull/3030>)
-- **web**: Provide deno usage information (<https://github.com/tree-sitter/tree-sitter/pull/2498>)
-
-### Refactor
-
-- Extract regex check into a function and lower its precedence
-- `&PathBuf` -> `&Path` (<https://github.com/tree-sitter/tree-sitter/pull/3035>)
-- Name anonymous types in api.h (<https://github.com/tree-sitter/tree-sitter/pull/1659>)
-
-### Testing
-
-- Add quotes around bash variables (<https://github.com/tree-sitter/tree-sitter/pull/3023>)
-- Update html tests
-
-### Build System and CI
-
-- Only create release for normal semver tags (<https://github.com/tree-sitter/tree-sitter/pull/2973>)
-- Add useful development targets to makefile (<https://github.com/tree-sitter/tree-sitter/pull/2979>)
-- Remove minimum glibc information in summary page (<https://github.com/tree-sitter/tree-sitter/pull/2988>)
-- Use the native m1 mac runner (<https://github.com/tree-sitter/tree-sitter/pull/2995>)
-- Add editorconfig (<https://github.com/tree-sitter/tree-sitter/pull/2998>)
-- Remove symbolic links from repository (<https://github.com/tree-sitter/tree-sitter/pull/2997>)
-- Move common Cargo.toml keys into the workspace and inherit them (<https://github.com/tree-sitter/tree-sitter/pull/3019>)
-- Remove reviewers when drafting or closing a PR (<https://github.com/tree-sitter/tree-sitter/pull/2963>)
-- Enable creating changelogs with git-cliff (<https://github.com/tree-sitter/tree-sitter/pull/3040>)
-- Cache fixtures (<https://github.com/tree-sitter/tree-sitter/pull/3038>)
-- Don't cancel jobs on master (<https://github.com/tree-sitter/tree-sitter/pull/3052>)
-- Relax caching requirements (<https://github.com/tree-sitter/tree-sitter/pull/3051>)
-- **deps**: Bump clap from 4.4.18 to 4.5.0 (<https://github.com/tree-sitter/tree-sitter/pull/3007>)
-- **deps**: Bump wasmtime from v16.0.0 to v17.0.1 (<https://github.com/tree-sitter/tree-sitter/pull/3008>)
-- **deps**: Bump wasmtime to v18.0.1 (<https://github.com/tree-sitter/tree-sitter/pull/3057>)
-- **sanitize**: Add a timeout of 60 minutes (<https://github.com/tree-sitter/tree-sitter/pull/3017>)
-- **sanitize**: Reduce timeout to 20 minutes (<https://github.com/tree-sitter/tree-sitter/pull/3054>)
-
-### Other
-
-- Document preferred language for scanner (<https://github.com/tree-sitter/tree-sitter/pull/2972>)
-- Add java and tsx to corpus tests (<https://github.com/tree-sitter/tree-sitter/pull/2992>)
-- Provide a CLI flag to open `log.html` (<https://github.com/tree-sitter/tree-sitter/pull/2996>)
-- Some more clippy lints (<https://github.com/tree-sitter/tree-sitter/pull/3010>)
-- Remove deprecated query parsing mechanism (<https://github.com/tree-sitter/tree-sitter/pull/3011>)
-- Print out full compiler arguments ran when it fails (<https://github.com/tree-sitter/tree-sitter/pull/3018>)
-- Deprecate C++ scanners (<https://github.com/tree-sitter/tree-sitter/pull/3020>)
-- Add some documentation to the playground page (<https://github.com/tree-sitter/tree-sitter/pull/1495>)
-- Update relevant rust tests (<https://github.com/tree-sitter/tree-sitter/pull/2947>)
-- Clippy lints (<https://github.com/tree-sitter/tree-sitter/pull/3032>)
-- Error out when multiple arguments are passed to `token`/`token.immediate` (<https://github.com/tree-sitter/tree-sitter/pull/3036>)
-- Tidying
-- Prefer turbofish syntax where possible (<https://github.com/tree-sitter/tree-sitter/pull/3048>)
-- Use published wasmtime crates
-- Cleaner cast
-- Update `Cargo.lock`
-- Get rid of `github_issue_test` file (<https://github.com/tree-sitter/tree-sitter/pull/3055>)
-- **cli**: Use spawn to display `emcc`'s stdout and stderr (<https://github.com/tree-sitter/tree-sitter/pull/2494>)
-- **cli**: Warn users when a query path needed for a subcommand isn't specified in a grammar's package.json
-- **generate**: Dedup and warn about duplicate or invalid rules (<https://github.com/tree-sitter/tree-sitter/pull/2994>)
-- **test**: Use different languages for async tests (<https://github.com/tree-sitter/tree-sitter/pull/2953>)
-- **wasm**: Use `SIDE_MODULE=2` to silence warning (<https://github.com/tree-sitter/tree-sitter/pull/3003>)

CMakeLists.txt

@@ -0,0 +1,96 @@
+cmake_minimum_required(VERSION 3.13)
+
+project(tree-sitter
+        VERSION "0.28.0"
+        DESCRIPTION "An incremental parsing system for programming tools"
+        HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
+        LANGUAGES C)
+
+option(BUILD_SHARED_LIBS "Build using shared libraries" ON)
+option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF)
+option(AMALGAMATED "Build using an amalgamated source" OFF)
+
+if(AMALGAMATED)
+  set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c")
+else()
+  file(GLOB TS_SOURCE_FILES lib/src/*.c)
+  list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c")
+endif()
+
+add_library(tree-sitter ${TS_SOURCE_FILES})
+
+target_include_directories(tree-sitter PRIVATE lib/src lib/src/wasm PUBLIC lib/include)
+
+if(MSVC)
+  target_compile_options(tree-sitter PRIVATE
+                         /wd4018 # disable 'signed/unsigned mismatch'
+                         /wd4232 # disable 'nonstandard extension used'
+                         /wd4244 # disable 'possible loss of data'
+                         /wd4267 # disable 'possible loss of data (size_t)'
+                         /wd4701 # disable 'potentially uninitialized local variable'
+                         /we4022 # treat 'incompatible types' as an error
+                         /W4)
+else()
+  target_compile_options(tree-sitter PRIVATE
+                         -Wall -Wextra -Wshadow -Wpedantic
+                         -Werror=incompatible-pointer-types
+                         -Werror=strict-aliasing -Wstrict-aliasing=2)
+endif()
+
+if(TREE_SITTER_FEATURE_WASM)
+  if(NOT DEFINED CACHE{WASMTIME_INCLUDE_DIR})
+    message(CHECK_START "Looking for wasmtime headers")
+    find_path(WASMTIME_INCLUDE_DIR wasmtime.h
+              PATHS ENV DEP_WASMTIME_C_API_INCLUDE)
+    if(NOT WASMTIME_INCLUDE_DIR)
+      unset(WASMTIME_INCLUDE_DIR CACHE)
+      message(FATAL_ERROR "Could not find wasmtime headers.\nDid you forget to set CMAKE_INCLUDE_PATH?")
+    endif()
+    message(CHECK_PASS "found")
+  endif()
+
+  if(NOT DEFINED CACHE{WASMTIME_LIBRARY})
+    message(CHECK_START "Looking for wasmtime library")
+    find_library(WASMTIME_LIBRARY wasmtime)
+    if(NOT WASMTIME_LIBRARY)
+      unset(WASMTIME_LIBRARY CACHE)
+      message(FATAL_ERROR "Could not find wasmtime library.\nDid you forget to set CMAKE_LIBRARY_PATH?")
+    endif()
+    message(CHECK_PASS "found")
+  endif()
+
+  target_compile_definitions(tree-sitter PUBLIC TREE_SITTER_FEATURE_WASM)
+  target_include_directories(tree-sitter SYSTEM PRIVATE "${WASMTIME_INCLUDE_DIR}")
+  target_link_libraries(tree-sitter PUBLIC "${WASMTIME_LIBRARY}")
+  set_property(TARGET tree-sitter PROPERTY C_STANDARD_REQUIRED ON)
+
+  if(NOT BUILD_SHARED_LIBS)
+    if(WIN32)
+      target_compile_definitions(tree-sitter PRIVATE WASM_API_EXTERN= WASI_API_EXTERN=)
+      target_link_libraries(tree-sitter INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt)
+    elseif(NOT APPLE)
+      target_link_libraries(tree-sitter INTERFACE pthread dl m)
+    endif()
+  endif()
+endif()
+
+set_target_properties(tree-sitter
+                      PROPERTIES
+                      C_STANDARD 11
+                      C_VISIBILITY_PRESET hidden
+                      POSITION_INDEPENDENT_CODE ON
+                      SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
+                      DEFINE_SYMBOL "")
+
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
+
+include(GNUInstallDirs)
+
+configure_file(lib/tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
+
+install(FILES lib/include/tree_sitter/api.h
+        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter")
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc"
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+install(TARGETS tree-sitter
+        LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")

CONTRIBUTING.md

@@ -1 +1 @@
-See [section-6-contributing.md](./docs/section-6-contributing.md)
+See [docs/src/6-contributing.md](./docs/src/6-contributing.md)

Cargo.toml

@@ -1,34 +1,68 @@
 [workspace]
-default-members = ["cli"]
+default-members = [ "crates/cli" ]
 members = [
-  "cli",
-  "cli/config",
-  "cli/loader",
+  "crates/cli",
+  "crates/config",
+  "crates/generate",
+  "crates/highlight",
+  "crates/loader",
+  "crates/tags",
+  "crates/xtask",
+  "crates/language",
   "lib",
-  "lib/language",
-  "tags",
-  "highlight",
-  "xtask",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.24.1"
-authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
-edition = "2021"
-rust-version = "1.74.1"
+authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>" ]
+categories = [ "command-line-utilities", "parsing" ]
+edition = "2024"
 homepage = "https://tree-sitter.github.io/tree-sitter"
-repository = "https://github.com/tree-sitter/tree-sitter"
+keywords = [ "incremental", "parsing" ]
 license = "MIT"
-keywords = ["incremental", "parsing"]
-categories = ["command-line-utilities", "parsing"]
+repository = "https://github.com/tree-sitter/tree-sitter"
+rust-version = "1.90"
+version = "0.28.0"
+
+[workspace.lints.clippy]
+cargo = { level = "warn", priority = -1 }
+dbg_macro = "deny"
+nursery = { level = "warn", priority = -1 }
+pedantic = { level = "warn", priority = -1 }
+todo = "deny"
+
+# The lints below are a specific subset of the pedantic+nursery lints
+# that we explicitly allow in the tree-sitter codebase because they either:
+#
+# 1. Contain false positives,
+# 2. Are unnecessary, or
+# 3. Worsen the code
+
+cast_possible_truncation = "allow"
+cast_possible_wrap = "allow"
+cast_precision_loss = "allow"
+cast_sign_loss = "allow"
+if_not_else = "allow"
+items_after_statements = "allow"
+missing_errors_doc = "allow"
+missing_panics_doc = "allow"
+multiple_crate_versions = "allow"
+option_if_let_else = "allow"
+similar_names = "allow"
+string_lit_as_bytes = "allow"
+struct_excessive_bools = "allow"
+too_many_lines = "allow"
+tuple_array_conversions = "allow"
+
+[workspace.lints.rust]
+mismatched_lifetime_syntaxes = "allow"
 
 [profile.optimize]
 inherits = "release"
-strip = true         # Automatically strip symbols from the binary.
+codegen-units = 1    # Maximum size reduction optimizations.
 lto = true           # Link-time optimization.
 opt-level = 3        # Optimization level 3.
-codegen-units = 1    # Maximum size reduction optimizations.
+strip = true         # Automatically strip symbols from the binary.
 
 [profile.size]
 inherits = "optimize"
@@ -36,69 +70,69 @@ opt-level = "s"       # Optimize for size.
 
 [profile.release-dev]
 inherits = "release"
-lto = false
+codegen-units = 256
 debug = true
 debug-assertions = true
-overflow-checks = true
 incremental = true
-codegen-units = 256
+lto = false
+overflow-checks = true
 
 [workspace.dependencies]
-anstyle = "1.0.8"
-anyhow = "1.0.89"
-bstr = "1.10.0"
-cc = "1.1.22"
-clap = { version = "4.5.18", features = [
+ansi_colours = "1.2.3"
+anstyle = "1.0.14"
+anyhow = "1.0.102"
+bstr = "1.12.1"
+cc = "1.2.63"
+clap = { features = [
   "cargo",
   "derive",
   "env",
   "help",
+  "string",
   "unstable-styles",
-] }
-clap_complete = "4.5.29"
-ctor = "0.2.8"
-ctrlc = { version = "3.4.5", features = ["termination"] }
-dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
-dirs = "5.0.1"
-filetime = "0.2.25"
-fs4 = "0.9.1"
-git2 = "0.19.0"
-glob = "0.3.1"
+], version = "4.5.58" }
+clap_complete = "4.6.3"
+clap_complete_nushell = "4.5.10"
+crc32fast = "1.5.0"
+ctor = "0.6.3"
+ctrlc = { features = [ "termination" ], version = "3.5.2" }
+dialoguer = { features = [ "fuzzy-select" ], version = "0.12.0" }
+etcetera = "0.11.0"
+glob = "0.3.3"
+hashbrown = { default-features = false, version = "0.17.1" }
 heck = "0.5.0"
 html-escape = "0.2.13"
-indexmap = "2.5.0"
-indoc = "2.0.5"
-lazy_static = "1.5.0"
-libloading = "0.8.5"
-log = { version = "0.4.22", features = ["std"] }
-memchr = "2.7.4"
-once_cell = "1.19.0"
-path-slash = "0.2.1"
+indexmap = "2.13.0"
+indoc = "2.0.7"
+libloading = "0.9.0"
+log = { features = [ "std" ], version = "0.4.30" }
+memchr = "2.8.1"
+once_cell = "1.21.3"
 pretty_assertions = "1.4.1"
-rand = "0.8.5"
-regex = "1.10.6"
-regex-syntax = "0.8.4"
-rustc-hash = "2.0.0"
-semver = { version = "1.0.23", features = ["serde"] }
-serde = { version = "1.0.210", features = ["derive"] }
-serde_derive = "1.0.210"
-serde_json = { version = "1.0.128", features = ["preserve_order"] }
-similar = "2.6.0"
-smallbitvec = "2.5.3"
+rand = "0.10.1"
+regex = "1.12.3"
+regex-syntax = "0.8.9"
+rustc-hash = "2.1.1"
+schemars = "1.2.1"
+semver = { features = [ "serde" ], version = "1.0.27" }
+serde = { features = [ "derive" ], version = "1.0.228" }
+serde_json = { features = [ "preserve_order" ], version = "1.0.150" }
+similar = "2.7.0"
 streaming-iterator = "0.1.9"
-tempfile = "3.12.0"
-thiserror = "1.0.64"
+tempfile = "3.25.0"
+thiserror = "2.0.18"
 tiny_http = "0.12.0"
-toml = "0.8.19"
-unindent = "0.2.3"
-url = { version = "2.5.2", features = ["serde"] }
+topological-sort = "0.2.2"
+unindent = "0.2.4"
 walkdir = "2.5.0"
-wasmparser = "0.217.0"
-webbrowser = "1.0.2"
+wasmparser = "0.244.0"
+webbrowser = "1.2.1"
 
-tree-sitter = { version = "0.24.0", path = "./lib" }
-tree-sitter-generate = { version = "0.24.0", path = "./cli/generate" }
-tree-sitter-loader = { version = "0.24.0", path = "./cli/loader" }
-tree-sitter-config = { version = "0.24.0", path = "./cli/config" }
-tree-sitter-highlight = { version = "0.24.0", path = "./highlight" }
-tree-sitter-tags = { version = "0.24.0", path = "./tags" }
+tree-sitter = { path = "./lib", version = "0.28.0" }
+tree-sitter-config = { path = "./crates/config", version = "0.28.0" }
+tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.28.0" }
+tree-sitter-highlight = { path = "./crates/highlight", version = "0.28.0" }
+tree-sitter-loader = { path = "./crates/loader", version = "0.28.0" }
+tree-sitter-tags = { path = "./crates/tags", version = "0.28.0" }
+
+tree-sitter-language = { path = "./crates/language", version = "0.1.8" }

Dockerfile

@@ -1,10 +0,0 @@
-FROM rust:1.76-buster
-
-WORKDIR /app
-
-RUN apt-get update
-RUN apt-get install -y nodejs
-
-COPY . .
-
-CMD cargo test --all-features

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2024 Max Brunsfeld
+Copyright (c) 2018 Max Brunsfeld
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,8 +1,4 @@
-ifeq ($(OS),Windows_NT)
-$(error Windows is not supported)
-endif
-
-VERSION := 0.24.1
+VERSION := 0.28.0
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -10,6 +6,7 @@ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 PREFIX ?= /usr/local
 INCLUDEDIR ?= $(PREFIX)/include
 LIBDIR ?= $(PREFIX)/lib
+BINDIR ?= $(PREFIX)/bin
 PCLIBDIR ?= $(LIBDIR)/pkgconfig
 
 # collect sources
@@ -25,8 +22,9 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -pedantic
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types -Werror=strict-aliasing -Wstrict-aliasing=2
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -34,20 +32,25 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
 SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
 
 # OS-specific bits
-ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),)
+MACHINE := $(shell $(CC) -dumpmachine)
+
+ifneq ($(findstring darwin,$(MACHINE)),)
 	SOEXT = dylib
 	SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
 	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
 	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER)
+else ifneq ($(findstring mingw32,$(MACHINE)),)
+	SOEXT = dll
+	LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a
 else
 	SOEXT = so
 	SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
 	SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
 	LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER)
-endif
 ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
 	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
 endif
+endif
 
 all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc
 
@@ -60,25 +63,39 @@ ifneq ($(STRIP),)
 	$(STRIP) $@
 endif
 
+ifneq ($(findstring mingw32,$(MACHINE)),)
+libtree-sitter.dll.a: libtree-sitter.$(SOEXT)
+endif
+
 tree-sitter.pc: lib/tree-sitter.pc.in
 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
-		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR)|' \
-		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR)|' \
+		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
+		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \
 		-e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \
 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
 
+shared: libtree-sitter.$(SOEXT)
+
+static: libtree-sitter.a
+
 clean:
-	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT)
+	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a
 
 install: all
 	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
 	install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h
 	install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
 	install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a
+ifneq ($(findstring mingw32,$(MACHINE)),)
+	install -d '$(DESTDIR)$(BINDIR)'
+	install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll
+	install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a
+else
 	install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER)
-	ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR)
-	ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT)
+	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR)
+	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT)
+endif
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \
@@ -87,31 +104,40 @@ uninstall:
 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \
 		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \
 		'$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
+	rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
 
-.PHONY: all install uninstall clean
+.PHONY: all shared static install uninstall clean
 
 
 ##### Dev targets #####
 
 test:
-	script/fetch-fixtures
-	script/generate-fixtures
-	script/test
+	cargo xtask fetch-fixtures
+	cargo xtask generate-fixtures
+	cargo xtask test
 
-test_wasm:
-	script/generate-fixtures-wasm
-	script/test-wasm
+test-wasm:
+	cargo xtask generate-fixtures --wasm
+	cargo xtask test-wasm
 
 lint:
 	cargo update --workspace --locked --quiet
-	cargo check --workspace --all-targets
-	cargo +nightly fmt --all --check
+	cargo fmt --all --check
 	cargo clippy --workspace --all-targets -- -D warnings
 
+lint-web:
+	npm --prefix lib/binding_web ci
+	npm --prefix lib/binding_web run lint
+
+lint-toml:
+	taplo check
+	taplo format --check --diff
+
 format:
-	cargo +nightly fmt --all
+	cargo fmt --all
+	taplo format
 
 changelog:
-	@git-cliff --config script/cliff.toml --output CHANGELOG.md --latest --github-token $(shell gh auth token)
+	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)
 
-.PHONY: test test_wasm lint format changelog
+.PHONY: test test-wasm lint format changelog

Package.swift

@@ -1,21 +0,0 @@
-// swift-tools-version: 5.8
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
-import PackageDescription
-
-let package = Package(
-    name: "TreeSitter",
-    products: [
-        // Products define the executables and libraries a package produces, and make them visible to other packages.
-        .library(
-            name: "TreeSitter",
-            targets: ["TreeSitter"]),
-    ],
-    targets: [
-        .target(name: "TreeSitter",
-                path: "lib",
-                sources: ["src/lib.c"],
-                cSettings: [.headerSearchPath("src")]),
-    ],
-    cLanguageStandard: .c11
-)

README.md

@@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca
 ## Links
 - [Documentation](https://tree-sitter.github.io)
 - [Rust binding](lib/binding_rust/README.md)
-- [WASM binding](lib/binding_web/README.md)
-- [Command-line interface](cli/README.md)
+- [Wasm binding](lib/binding_web/README.md)
+- [Command-line interface](crates/cli/README.md)
 
 [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord
 [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix

build.zig

@@ -1,18 +1,147 @@
 const std = @import("std");
 
-pub fn build(b: *std.Build) void {
-    var lib = b.addStaticLibrary(.{
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    var threaded: std.Io.Threaded = .init(b.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
+    const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
+    const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
+
+    var tree_sitter = b.createModule(.{
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .pic = if (shared) true else null,
+    });
+    const lib: *std.Build.Step.Compile = b.addLibrary(.{
         .name = "tree-sitter",
-        .target = b.standardTargetOptions(.{}),
-        .optimize = b.standardOptimizeOption(.{}),
+        .linkage = if (shared) .dynamic else .static,
+        .root_module = tree_sitter,
     });
 
-    lib.linkLibC();
-    lib.addCSourceFile(.{ .file = b.path("lib/src/lib.c"), .flags = &.{"-std=c11"} });
-    lib.addIncludePath(b.path("lib/include"));
-    lib.addIncludePath(b.path("lib/src"));
+    if (amalgamated) {
+        tree_sitter.addCSourceFile(.{
+            .file = b.path("lib/src/lib.c"),
+            .flags = &.{"-std=c11"},
+        });
+    } else {
+        const files = try findSourceFiles(b, io);
+        defer b.allocator.free(files);
+        tree_sitter.addCSourceFiles(.{
+            .root = b.path("lib/src"),
+            .files = files,
+            .flags = &.{"-std=c11"},
+        });
+    }
+
+    tree_sitter.addIncludePath(b.path("lib/include"));
+    tree_sitter.addIncludePath(b.path("lib/src"));
+    tree_sitter.addIncludePath(b.path("lib/src/wasm"));
+
+    tree_sitter.addCMacro("_POSIX_C_SOURCE", "200112L");
+    tree_sitter.addCMacro("_DEFAULT_SOURCE", "");
+    tree_sitter.addCMacro("_BSD_SOURCE", "");
+    tree_sitter.addCMacro("_DARWIN_C_SOURCE", "");
+
+    if (wasm) {
+        if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
+            tree_sitter.addCMacro("TREE_SITTER_FEATURE_WASM", "");
+            tree_sitter.addSystemIncludePath(wasmtime.path("include"));
+            tree_sitter.addLibraryPath(wasmtime.path("lib"));
+            if (shared) tree_sitter.linkSystemLibrary("wasmtime", .{});
+        }
+    }
 
     lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
 
     b.installArtifact(lib);
 }
+
+/// Get the name of the wasmtime dependency for this target.
+pub fn wasmtimeDep(target: std.Target) []const u8 {
+    const arch = target.cpu.arch;
+    const os = target.os.tag;
+    const abi = target.abi;
+    return @as(?[]const u8, switch (os) {
+        .linux => switch (arch) {
+            .x86_64 => switch (abi) {
+                .gnu => "wasmtime_c_api_x86_64_linux",
+                .musl => "wasmtime_c_api_x86_64_musl",
+                .android => "wasmtime_c_api_x86_64_android",
+                else => null,
+            },
+            .aarch64 => switch (abi) {
+                .gnu => "wasmtime_c_api_aarch64_linux",
+                .musl => "wasmtime_c_api_aarch64_musl",
+                .android => "wasmtime_c_api_aarch64_android",
+                else => null,
+            },
+            .x86 => switch (abi) {
+                .gnu => "wasmtime_c_api_i686_linux",
+                else => null,
+            },
+            .arm => switch (abi) {
+                .gnueabi => "wasmtime_c_api_armv7_linux",
+                else => null,
+            },
+            .s390x => switch (abi) {
+                .gnu => "wasmtime_c_api_s390x_linux",
+                else => null,
+            },
+            .riscv64 => switch (abi) {
+                .gnu => "wasmtime_c_api_riscv64gc_linux",
+                else => null,
+            },
+            else => null,
+        },
+        .windows => switch (arch) {
+            .x86_64 => switch (abi) {
+                .gnu => "wasmtime_c_api_x86_64_mingw",
+                .msvc => "wasmtime_c_api_x86_64_windows",
+                else => null,
+            },
+            .aarch64 => switch (abi) {
+                .msvc => "wasmtime_c_api_aarch64_windows",
+                else => null,
+            },
+            .x86 => switch (abi) {
+                .msvc => "wasmtime_c_api_i686_windows",
+                else => null,
+            },
+            else => null,
+        },
+        .macos => switch (arch) {
+            .x86_64 => "wasmtime_c_api_x86_64_macos",
+            .aarch64 => "wasmtime_c_api_aarch64_macos",
+            else => null,
+        },
+        else => null,
+    }) orelse std.debug.panic(
+        "Unsupported target for wasmtime: {s}-{s}-{s}",
+        .{ @tagName(arch), @tagName(os), @tagName(abi) },
+    );
+}
+
+fn findSourceFiles(b: *std.Build, io: std.Io) ![]const []const u8 {
+    var sources: std.ArrayListUnmanaged([]const u8) = .empty;
+
+    var dir = try b.build_root.handle.openDir(io, "lib/src", .{ .iterate = true });
+    var iter = dir.iterate();
+    defer dir.close(io);
+
+    while (try iter.next(io)) |entry| {
+        if (entry.kind != .file) continue;
+        const file = entry.name;
+        const ext = std.fs.path.extension(file);
+        if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
+            try sources.append(b.allocator, b.dupe(file));
+        }
+    }
+
+    return sources.toOwnedSlice(b.allocator);
+}

build.zig.zon

@@ -1,10 +1,96 @@
 .{
-    .name = "tree-sitter",
-    .version = "0.24.1",
+    .name = .tree_sitter,
+    .fingerprint = 0x841224b447ac0d4f,
+    .version = "0.28.0",
+    .minimum_zig_version = "0.16.0",
     .paths = .{
         "build.zig",
         "build.zig.zon",
         "lib/src",
         "lib/include",
+        "README.md",
+        "LICENSE",
+    },
+    .dependencies = .{
+        .wasmtime_c_api_aarch64_android = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAIp_mQVzQOITXcYcWxYLJkvB1W1SvLlrdiU2G7fj",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAMztsgU5Aj4oI3MRHXJVe5rW72op-kT_78I3kZVM",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_macos = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AANZxOwT27sdrKxDDGGKsiwtcZlHy204xAWNIgDBH",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_musl = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAJL1zQW9yxC98uc60lSuVUHhH77QHTto0zwIQnBj",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-aarch64-windows-c-api.zip",
+            .hash = "N-V-__8AAHRCtQU93hJcRFOgVcof3IQpRV9stT2Pp54wpJc2",
+            .lazy = true,
+        },
+        .wasmtime_c_api_armv7_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-armv7-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAJaW6gT8QdULOU0jxX4a_DOCA5YD6cxWBC8IqhQF",
+            .lazy = true,
+        },
+        .wasmtime_c_api_i686_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-linux-c-api.tar.xz",
+            .hash = "N-V-__8AANguMgVX4XMhdOVkdj4yfFKXrG8RTgZDs3nQB8J8",
+            .lazy = true,
+        },
+        .wasmtime_c_api_i686_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-i686-windows-c-api.zip",
+            .hash = "N-V-__8AANY9ggXg4rK2_1o3EIlrCq124l5RfykPv-DPforq",
+            .lazy = true,
+        },
+        .wasmtime_c_api_riscv64gc_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-riscv64gc-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAPDtCAdQ0dD9Rs-qWl-kPr2c7L3PVsUVjwy12Iz1",
+            .lazy = true,
+        },
+        .wasmtime_c_api_s390x_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-s390x-linux-c-api.tar.xz",
+            .hash = "N-V-__8AANA3BwY1ZOoGCWCR_tTY9G1vfIX128RGVxufN3ov",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_android = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAF4AIgY0ltjevj1ybGfvMU1ErPRnNve5X1TmvCru",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAIR0cAbjf3DkrTbu81Oq_zociz-0lCpb5DR0lIC9",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_macos = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAFJ4lgRgCBnYdz8-Yfc4hLve45Hv-0RICAEuCp4s",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_mingw = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-mingw-c-api.zip",
+            .hash = "N-V-__8AAMxZxQZUpp1cU8J5zgLiMNq4e4dy0hcchlQFy03J",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_musl = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAN5pWgZrZBt8VYWkN82WjyFe3DkGcE7uLn3jpt38",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.1/wasmtime-v48.0.1-x86_64-windows-c-api.zip",
+            .hash = "N-V-__8AAEIJkgaVHFETgakbognNUpFELuV17vpjw6NjsWhQ",
+            .lazy = true,
+        },
     },
 }

cli/Cargo.toml

@@ -1,75 +0,0 @@
-[package]
-name = "tree-sitter-cli"
-version.workspace = true
-description = "CLI tool for developing, testing, and using Tree-sitter parsers"
-authors.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-readme = "README.md"
-homepage.workspace = true
-repository.workspace = true
-license.workspace = true
-keywords.workspace = true
-categories.workspace = true
-
-[[bin]]
-name = "tree-sitter"
-path = "src/main.rs"
-doc = false
-
-[[bench]]
-name = "benchmark"
-harness = false
-
-[features]
-wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
-
-[dependencies]
-anstyle.workspace = true
-anyhow.workspace = true
-bstr.workspace = true
-clap.workspace = true
-clap_complete.workspace = true
-ctor.workspace = true
-ctrlc.workspace = true
-dialoguer.workspace = true
-dirs.workspace = true
-filetime.workspace = true
-glob.workspace = true
-heck.workspace = true
-html-escape.workspace = true
-indexmap.workspace = true
-indoc.workspace = true
-lazy_static.workspace = true
-log.workspace = true
-memchr.workspace = true
-rand.workspace = true
-regex.workspace = true
-regex-syntax.workspace = true
-rustc-hash.workspace = true
-semver.workspace = true
-serde.workspace = true
-serde_derive.workspace = true
-serde_json.workspace = true
-similar.workspace = true
-smallbitvec.workspace = true
-streaming-iterator.workspace = true
-tiny_http.workspace = true
-url.workspace = true
-walkdir.workspace = true
-wasmparser.workspace = true
-webbrowser.workspace = true
-
-tree-sitter.workspace = true
-tree-sitter-generate.workspace = true
-tree-sitter-config.workspace = true
-tree-sitter-highlight.workspace = true
-tree-sitter-loader.workspace = true
-tree-sitter-tags.workspace = true
-
-[dev-dependencies]
-tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
-
-tempfile.workspace = true
-pretty_assertions.workspace = true
-unindent.workspace = true

cli/benches/benchmark.rs

@@ -1,222 +0,0 @@
-use std::{
-    collections::BTreeMap,
-    env, fs,
-    path::{Path, PathBuf},
-    str,
-    time::Instant,
-};
-
-use anyhow::Context;
-use lazy_static::lazy_static;
-use tree_sitter::{Language, Parser, Query};
-use tree_sitter_loader::{CompileConfig, Loader};
-
-include!("../src/tests/helpers/dirs.rs");
-
-lazy_static! {
-    static ref LANGUAGE_FILTER: Option<String> =
-        env::var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER").ok();
-    static ref EXAMPLE_FILTER: Option<String> =
-        env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok();
-    static ref REPETITION_COUNT: usize = env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
-        .map(|s| s.parse::<usize>().unwrap())
-        .unwrap_or(5);
-    static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
-    static ref EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)> = {
-        fn process_dir(result: &mut BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>, dir: &Path) {
-            if dir.join("grammar.js").exists() {
-                let relative_path = dir.strip_prefix(GRAMMARS_DIR.as_path()).unwrap();
-                let (example_paths, query_paths) =
-                    result.entry(relative_path.to_owned()).or_default();
-
-                if let Ok(example_files) = fs::read_dir(dir.join("examples")) {
-                    example_paths.extend(example_files.filter_map(|p| {
-                        let p = p.unwrap().path();
-                        if p.is_file() {
-                            Some(p)
-                        } else {
-                            None
-                        }
-                    }));
-                }
-
-                if let Ok(query_files) = fs::read_dir(dir.join("queries")) {
-                    query_paths.extend(query_files.filter_map(|p| {
-                        let p = p.unwrap().path();
-                        if p.is_file() {
-                            Some(p)
-                        } else {
-                            None
-                        }
-                    }));
-                }
-            } else {
-                for entry in fs::read_dir(dir).unwrap() {
-                    let entry = entry.unwrap().path();
-                    if entry.is_dir() {
-                        process_dir(result, &entry);
-                    }
-                }
-            }
-        }
-
-        let mut result = BTreeMap::new();
-        process_dir(&mut result, &GRAMMARS_DIR);
-        result
-    };
-}
-
-fn main() {
-    let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR
-        .values()
-        .flat_map(|(e, q)| {
-            e.iter()
-                .chain(q.iter())
-                .map(|s| s.file_name().unwrap().to_str().unwrap().len())
-        })
-        .max()
-        .unwrap_or(0);
-
-    eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT);
-
-    let mut parser = Parser::new();
-    let mut all_normal_speeds = Vec::new();
-    let mut all_error_speeds = Vec::new();
-
-    for (language_path, (example_paths, query_paths)) in
-        EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
-    {
-        let language_name = language_path.file_name().unwrap().to_str().unwrap();
-
-        if let Some(filter) = LANGUAGE_FILTER.as_ref() {
-            if language_name != filter.as_str() {
-                continue;
-            }
-        }
-
-        eprintln!("\nLanguage: {language_name}");
-        let language = get_language(language_path);
-        parser.set_language(&language).unwrap();
-
-        eprintln!("  Constructing Queries");
-        for path in query_paths {
-            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
-                if !path.to_str().unwrap().contains(filter.as_str()) {
-                    continue;
-                }
-            }
-
-            parse(path, max_path_length, |source| {
-                Query::new(&language, str::from_utf8(source).unwrap())
-                    .with_context(|| format!("Query file path: {path:?}"))
-                    .expect("Failed to parse query");
-            });
-        }
-
-        eprintln!("  Parsing Valid Code:");
-        let mut normal_speeds = Vec::new();
-        for example_path in example_paths {
-            if let Some(filter) = EXAMPLE_FILTER.as_ref() {
-                if !example_path.to_str().unwrap().contains(filter.as_str()) {
-                    continue;
-                }
-            }
-
-            normal_speeds.push(parse(example_path, max_path_length, |code| {
-                parser.parse(code, None).expect("Failed to parse");
-            }));
-        }
-
-        eprintln!("  Parsing Invalid Code (mismatched languages):");
-        let mut error_speeds = Vec::new();
-        for (other_language_path, (example_paths, _)) in
-            EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
-        {
-            if other_language_path != language_path {
-                for example_path in example_paths {
-                    if let Some(filter) = EXAMPLE_FILTER.as_ref() {
-                        if !example_path.to_str().unwrap().contains(filter.as_str()) {
-                            continue;
-                        }
-                    }
-
-                    error_speeds.push(parse(example_path, max_path_length, |code| {
-                        parser.parse(code, None).expect("Failed to parse");
-                    }));
-                }
-            }
-        }
-
-        if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) {
-            eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
-            eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
-        }
-
-        if let Some((average_error, worst_error)) = aggregate(&error_speeds) {
-            eprintln!("  Average Speed (errors): {average_error} bytes/ms");
-            eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
-        }
-
-        all_normal_speeds.extend(normal_speeds);
-        all_error_speeds.extend(error_speeds);
-    }
-
-    eprintln!("\n  Overall");
-    if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) {
-        eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
-        eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
-    }
-
-    if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) {
-        eprintln!("  Average Speed (errors): {average_error} bytes/ms");
-        eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
-    }
-    eprintln!();
-}
-
-fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
-    if speeds.is_empty() {
-        return None;
-    }
-    let mut total = 0;
-    let mut max = usize::MAX;
-    for speed in speeds.iter().copied() {
-        total += speed;
-        if speed < max {
-            max = speed;
-        }
-    }
-    Some((total / speeds.len(), max))
-}
-
-fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize {
-    eprint!(
-        "    {:width$}\t",
-        path.file_name().unwrap().to_str().unwrap(),
-        width = max_path_length
-    );
-
-    let source_code = fs::read(path)
-        .with_context(|| format!("Failed to read {path:?}"))
-        .unwrap();
-    let time = Instant::now();
-    for _ in 0..*REPETITION_COUNT {
-        action(&source_code);
-    }
-    let duration = time.elapsed() / (*REPETITION_COUNT as u32);
-    let duration_ns = duration.as_nanos();
-    let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns;
-    eprintln!(
-        "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
-        (duration_ns as f64) / 1e6,
-    );
-    speed as usize
-}
-
-fn get_language(path: &Path) -> Language {
-    let src_path = GRAMMARS_DIR.join(path).join("src");
-    TEST_LOADER
-        .load_language_at_path(CompileConfig::new(&src_path, None, None))
-        .with_context(|| format!("Failed to load language at path {src_path:?}"))
-        .unwrap()
-}

cli/build.rs

@@ -1,142 +0,0 @@
-use std::{
-    env,
-    ffi::OsStr,
-    fs,
-    path::{Path, PathBuf},
-    time::SystemTime,
-};
-
-fn main() {
-    if let Some(git_sha) = read_git_sha() {
-        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
-    }
-
-    println!("cargo:rustc-check-cfg=cfg(sanitizing)");
-    println!("cargo:rustc-check-cfg=cfg(TREE_SITTER_EMBED_WASM_BINDING)");
-
-    if web_playground_files_present() {
-        println!("cargo:rustc-cfg=TREE_SITTER_EMBED_WASM_BINDING");
-    }
-
-    let build_time = SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
-        .as_secs_f64();
-    println!("cargo:rustc-env=BUILD_TIME={build_time}");
-
-    #[cfg(any(
-        target_os = "linux",
-        target_os = "android",
-        target_os = "freebsd",
-        target_os = "openbsd",
-        target_os = "netbsd",
-        target_os = "dragonfly",
-    ))]
-    {
-        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()).join("dynamic-symbols.txt");
-        std::fs::write(
-            &out_dir,
-            "{
-                ts_current_malloc;
-                ts_current_calloc;
-                ts_current_realloc;
-                ts_current_free;
-            };",
-        )
-        .unwrap();
-        println!(
-            "cargo:rustc-link-arg=-Wl,--dynamic-list={}",
-            out_dir.display()
-        );
-    }
-}
-
-fn web_playground_files_present() -> bool {
-    let paths = [
-        "../docs/assets/js/playground.js",
-        "../lib/binding_web/tree-sitter.js",
-        "../lib/binding_web/tree-sitter.wasm",
-    ];
-
-    paths.iter().all(|p| Path::new(p).exists())
-}
-
-fn read_git_sha() -> Option<String> {
-    let mut repo_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
-
-    let mut git_path;
-    loop {
-        git_path = repo_path.join(".git");
-        if git_path.exists() {
-            break;
-        }
-        if !repo_path.pop() {
-            return None;
-        }
-    }
-
-    let git_dir_path;
-    if git_path.is_dir() {
-        git_dir_path = git_path;
-    } else if let Ok(git_path_content) = fs::read_to_string(&git_path) {
-        git_dir_path = repo_path.join(git_path_content.get("gitdir: ".len()..).unwrap().trim_end());
-    } else {
-        return None;
-    }
-    let git_head_path = git_dir_path.join("HEAD");
-    if let Some(path) = git_head_path.to_str() {
-        println!("cargo:rerun-if-changed={path}");
-    }
-    if let Ok(mut head_content) = fs::read_to_string(&git_head_path) {
-        if head_content.ends_with('\n') {
-            head_content.pop();
-        }
-
-        // If we're on a branch, read the SHA from the ref file.
-        if head_content.starts_with("ref: ") {
-            head_content.replace_range(0.."ref: ".len(), "");
-            let ref_filename = {
-                // Go to real non-worktree gitdir
-                let git_dir_path = git_dir_path
-                    .parent()
-                    .and_then(|p| {
-                        p.file_name()
-                            .map(|n| n == OsStr::new("worktrees"))
-                            .and_then(|x| x.then(|| p.parent()))
-                    })
-                    .flatten()
-                    .unwrap_or(&git_dir_path);
-
-                let file = git_dir_path.join(&head_content);
-                if file.is_file() {
-                    file
-                } else {
-                    let packed_refs = git_dir_path.join("packed-refs");
-                    if let Ok(packed_refs_content) = fs::read_to_string(&packed_refs) {
-                        for line in packed_refs_content.lines() {
-                            if let Some((hash, r#ref)) = line.split_once(' ') {
-                                if r#ref == head_content {
-                                    if let Some(path) = packed_refs.to_str() {
-                                        println!("cargo:rerun-if-changed={path}");
-                                    }
-                                    return Some(hash.to_string());
-                                }
-                            }
-                        }
-                    }
-                    return None;
-                }
-            };
-            if let Some(path) = ref_filename.to_str() {
-                println!("cargo:rerun-if-changed={path}");
-            }
-            return fs::read_to_string(&ref_filename).ok();
-        }
-        // If we're on a detached commit, then the `HEAD` file itself contains the sha.
-        if head_content.len() == 40 {
-            return Some(head_content);
-        }
-    }
-
-    None
-}

cli/generate/Cargo.toml

@@ -1,31 +0,0 @@
-[package]
-name = "tree-sitter-generate"
-version.workspace = true
-description = "Library for generating C source code from a tree-sitter grammar"
-authors.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-readme = "README.md"
-homepage.workspace = true
-repository.workspace = true
-license.workspace = true
-keywords.workspace = true
-categories.workspace = true
-
-[dependencies]
-anyhow.workspace = true
-heck.workspace = true
-indexmap.workspace = true
-indoc.workspace = true
-lazy_static.workspace = true
-log.workspace = true
-regex.workspace = true
-regex-syntax.workspace = true
-rustc-hash.workspace = true
-semver.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-smallbitvec.workspace = true
-url.workspace = true
-
-tree-sitter.workspace = true

cli/generate/src/build_tables/coincident_tokens.rs

@@ -1,79 +0,0 @@
-use std::fmt;
-
-use crate::{
-    grammars::LexicalGrammar,
-    rules::Symbol,
-    tables::{ParseStateId, ParseTable},
-};
-
-pub struct CoincidentTokenIndex<'a> {
-    entries: Vec<Vec<ParseStateId>>,
-    grammar: &'a LexicalGrammar,
-    n: usize,
-}
-
-impl<'a> CoincidentTokenIndex<'a> {
-    pub fn new(table: &ParseTable, lexical_grammar: &'a LexicalGrammar) -> Self {
-        let n = lexical_grammar.variables.len();
-        let mut result = Self {
-            n,
-            grammar: lexical_grammar,
-            entries: vec![Vec::new(); n * n],
-        };
-        for (i, state) in table.states.iter().enumerate() {
-            for symbol in state.terminal_entries.keys() {
-                if symbol.is_terminal() {
-                    for other_symbol in state.terminal_entries.keys() {
-                        if other_symbol.is_terminal() {
-                            let index = result.index(symbol.index, other_symbol.index);
-                            if result.entries[index].last().copied() != Some(i) {
-                                result.entries[index].push(i);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        result
-    }
-
-    pub fn states_with(&self, a: Symbol, b: Symbol) -> &[ParseStateId] {
-        &self.entries[self.index(a.index, b.index)]
-    }
-
-    pub fn contains(&self, a: Symbol, b: Symbol) -> bool {
-        !self.entries[self.index(a.index, b.index)].is_empty()
-    }
-
-    #[must_use]
-    const fn index(&self, a: usize, b: usize) -> usize {
-        if a < b {
-            a * self.n + b
-        } else {
-            b * self.n + a
-        }
-    }
-}
-
-impl<'a> fmt::Debug for CoincidentTokenIndex<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "CoincidentTokenIndex {{")?;
-
-        writeln!(f, "  entries: {{")?;
-        for i in 0..self.n {
-            writeln!(f, "    {}: {{", self.grammar.variables[i].name)?;
-            for j in 0..self.n {
-                writeln!(
-                    f,
-                    "      {}: {:?},",
-                    self.grammar.variables[j].name,
-                    self.entries[self.index(i, j)].len()
-                )?;
-            }
-            writeln!(f, "    }},")?;
-        }
-        write!(f, "  }},")?;
-        write!(f, "}}")?;
-        Ok(())
-    }
-}

cli/generate/src/build_tables/item.rs

@@ -1,416 +0,0 @@
-use std::{
-    cmp::Ordering,
-    fmt,
-    hash::{Hash, Hasher},
-};
-
-use lazy_static::lazy_static;
-
-use crate::{
-    grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
-    rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
-};
-
-lazy_static! {
-    static ref START_PRODUCTION: Production = Production {
-        dynamic_precedence: 0,
-        steps: vec![ProductionStep {
-            symbol: Symbol {
-                index: 0,
-                kind: SymbolType::NonTerminal,
-            },
-            precedence: Precedence::None,
-            associativity: None,
-            alias: None,
-            field_name: None,
-        }],
-    };
-}
-
-/// A [`ParseItem`] represents an in-progress match of a single production in a grammar.
-#[derive(Clone, Copy, Debug)]
-pub struct ParseItem<'a> {
-    /// The index of the parent rule within the grammar.
-    pub variable_index: u32,
-    /// The number of symbols that have already been matched.
-    pub step_index: u32,
-    /// The production being matched.
-    pub production: &'a Production,
-    /// A boolean indicating whether any of the already-matched children were
-    /// hidden nodes and had fields. Ordinarily, a parse item's behavior is not
-    /// affected by the symbols of its preceding children; it only needs to
-    /// keep track of their fields and aliases.
-    ///
-    /// Take for example these two items:
-    ///   X -> a b • c
-    ///   X -> a g • c
-    ///
-    /// They can be considered equivalent, for the purposes of parse table
-    /// generation, because they entail the same actions. But if this flag is
-    /// true, then the item's set of inherited fields may depend on the specific
-    /// symbols of its preceding children.
-    pub has_preceding_inherited_fields: bool,
-}
-
-/// A [`ParseItemSet`] represents a set of in-progress matches of productions in a
-/// grammar, and for each in-progress match, a set of "lookaheads" - tokens that
-/// are allowed to *follow* the in-progress rule. This object corresponds directly
-/// to a state in the final parse table.
-#[derive(Clone, Debug, PartialEq, Eq, Default)]
-pub struct ParseItemSet<'a> {
-    pub entries: Vec<(ParseItem<'a>, TokenSet)>,
-}
-
-/// A [`ParseItemSetCore`] is like a [`ParseItemSet`], but without the lookahead
-/// information. Parse states with the same core are candidates for merging.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ParseItemSetCore<'a> {
-    pub entries: Vec<ParseItem<'a>>,
-}
-
-pub struct ParseItemDisplay<'a>(
-    pub &'a ParseItem<'a>,
-    pub &'a SyntaxGrammar,
-    pub &'a LexicalGrammar,
-);
-
-pub struct TokenSetDisplay<'a>(
-    pub &'a TokenSet,
-    pub &'a SyntaxGrammar,
-    pub &'a LexicalGrammar,
-);
-
-pub struct ParseItemSetDisplay<'a>(
-    pub &'a ParseItemSet<'a>,
-    pub &'a SyntaxGrammar,
-    pub &'a LexicalGrammar,
-);
-
-impl<'a> ParseItem<'a> {
-    pub fn start() -> Self {
-        ParseItem {
-            variable_index: u32::MAX,
-            production: &START_PRODUCTION,
-            step_index: 0,
-            has_preceding_inherited_fields: false,
-        }
-    }
-
-    pub fn step(&self) -> Option<&'a ProductionStep> {
-        self.production.steps.get(self.step_index as usize)
-    }
-
-    pub fn symbol(&self) -> Option<Symbol> {
-        self.step().map(|step| step.symbol)
-    }
-
-    pub fn associativity(&self) -> Option<Associativity> {
-        self.prev_step().and_then(|step| step.associativity)
-    }
-
-    pub fn precedence(&self) -> &Precedence {
-        self.prev_step()
-            .map_or(&Precedence::None, |step| &step.precedence)
-    }
-
-    pub fn prev_step(&self) -> Option<&'a ProductionStep> {
-        if self.step_index > 0 {
-            Some(&self.production.steps[self.step_index as usize - 1])
-        } else {
-            None
-        }
-    }
-
-    #[must_use]
-    pub fn is_done(&self) -> bool {
-        self.step_index as usize == self.production.steps.len()
-    }
-
-    #[must_use]
-    pub const fn is_augmented(&self) -> bool {
-        self.variable_index == u32::MAX
-    }
-
-    /// Create an item like this one, but advanced by one step.
-    #[must_use]
-    pub const fn successor(&self) -> Self {
-        ParseItem {
-            variable_index: self.variable_index,
-            production: self.production,
-            step_index: self.step_index + 1,
-            has_preceding_inherited_fields: self.has_preceding_inherited_fields,
-        }
-    }
-
-    /// Create an item identical to this one, but with a different production.
-    /// This is used when dynamically "inlining" certain symbols in a production.
-    pub const fn substitute_production(&self, production: &'a Production) -> Self {
-        let mut result = *self;
-        result.production = production;
-        result
-    }
-}
-
-impl<'a> ParseItemSet<'a> {
-    pub fn with(elements: impl IntoIterator<Item = (ParseItem<'a>, TokenSet)>) -> Self {
-        let mut result = Self::default();
-        for (item, lookaheads) in elements {
-            result.insert(item, &lookaheads);
-        }
-        result
-    }
-
-    pub fn insert(&mut self, item: ParseItem<'a>, lookaheads: &TokenSet) -> &mut TokenSet {
-        match self.entries.binary_search_by(|(i, _)| i.cmp(&item)) {
-            Err(i) => {
-                self.entries.insert(i, (item, lookaheads.clone()));
-                &mut self.entries[i].1
-            }
-            Ok(i) => {
-                self.entries[i].1.insert_all(lookaheads);
-                &mut self.entries[i].1
-            }
-        }
-    }
-
-    pub fn core(&self) -> ParseItemSetCore<'a> {
-        ParseItemSetCore {
-            entries: self.entries.iter().map(|e| e.0).collect(),
-        }
-    }
-}
-
-impl<'a> fmt::Display for ParseItemDisplay<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        if self.0.is_augmented() {
-            write!(f, "START →")?;
-        } else {
-            write!(
-                f,
-                "{} →",
-                &self.1.variables[self.0.variable_index as usize].name
-            )?;
-        }
-
-        for (i, step) in self.0.production.steps.iter().enumerate() {
-            if i == self.0.step_index as usize {
-                write!(f, " •")?;
-                if let Some(associativity) = step.associativity {
-                    if step.precedence.is_none() {
-                        write!(f, " ({associativity:?})")?;
-                    } else {
-                        write!(f, " ({} {associativity:?})", step.precedence)?;
-                    }
-                } else if !step.precedence.is_none() {
-                    write!(f, " ({})", step.precedence)?;
-                }
-            }
-
-            write!(f, " ")?;
-            if step.symbol.is_terminal() {
-                if let Some(variable) = self.2.variables.get(step.symbol.index) {
-                    write!(f, "{}", &variable.name)?;
-                } else {
-                    write!(f, "terminal-{}", step.symbol.index)?;
-                }
-            } else if step.symbol.is_external() {
-                write!(f, "{}", &self.1.external_tokens[step.symbol.index].name)?;
-            } else {
-                write!(f, "{}", &self.1.variables[step.symbol.index].name)?;
-            }
-
-            if let Some(alias) = &step.alias {
-                write!(f, "@{}", alias.value)?;
-            }
-        }
-
-        if self.0.is_done() {
-            write!(f, " •")?;
-            if let Some(step) = self.0.production.steps.last() {
-                if let Some(associativity) = step.associativity {
-                    if step.precedence.is_none() {
-                        write!(f, " ({associativity:?})")?;
-                    } else {
-                        write!(f, " ({} {associativity:?})", step.precedence)?;
-                    }
-                } else if !step.precedence.is_none() {
-                    write!(f, " ({})", step.precedence)?;
-                }
-            }
-        }
-
-        Ok(())
-    }
-}
-
-impl<'a> fmt::Display for TokenSetDisplay<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        write!(f, "[")?;
-        for (i, symbol) in self.0.iter().enumerate() {
-            if i > 0 {
-                write!(f, ", ")?;
-            }
-
-            if symbol.is_terminal() {
-                if let Some(variable) = self.2.variables.get(symbol.index) {
-                    write!(f, "{}", &variable.name)?;
-                } else {
-                    write!(f, "terminal-{}", symbol.index)?;
-                }
-            } else if symbol.is_external() {
-                write!(f, "{}", &self.1.external_tokens[symbol.index].name)?;
-            } else {
-                write!(f, "{}", &self.1.variables[symbol.index].name)?;
-            }
-        }
-        write!(f, "]")?;
-        Ok(())
-    }
-}
-
-impl<'a> fmt::Display for ParseItemSetDisplay<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        for (item, lookaheads) in &self.0.entries {
-            writeln!(
-                f,
-                "{}\t{}",
-                ParseItemDisplay(item, self.1, self.2),
-                TokenSetDisplay(lookaheads, self.1, self.2)
-            )?;
-        }
-        Ok(())
-    }
-}
-
-impl<'a> Hash for ParseItem<'a> {
-    fn hash<H: Hasher>(&self, hasher: &mut H) {
-        hasher.write_u32(self.variable_index);
-        hasher.write_u32(self.step_index);
-        hasher.write_i32(self.production.dynamic_precedence);
-        hasher.write_usize(self.production.steps.len());
-        hasher.write_i32(i32::from(self.has_preceding_inherited_fields));
-        self.precedence().hash(hasher);
-        self.associativity().hash(hasher);
-
-        // The already-matched children don't play any role in the parse state for
-        // this item, unless any of the following are true:
-        //   * the children have fields
-        //   * the children have aliases
-        //   * the children are hidden and
-        // See the docs for `has_preceding_inherited_fields`.
-        for step in &self.production.steps[0..self.step_index as usize] {
-            step.alias.hash(hasher);
-            step.field_name.hash(hasher);
-            if self.has_preceding_inherited_fields {
-                step.symbol.hash(hasher);
-            }
-        }
-        for step in &self.production.steps[self.step_index as usize..] {
-            step.hash(hasher);
-        }
-    }
-}
-
-impl<'a> PartialEq for ParseItem<'a> {
-    fn eq(&self, other: &Self) -> bool {
-        if self.variable_index != other.variable_index
-            || self.step_index != other.step_index
-            || self.production.dynamic_precedence != other.production.dynamic_precedence
-            || self.production.steps.len() != other.production.steps.len()
-            || self.precedence() != other.precedence()
-            || self.associativity() != other.associativity()
-            || self.has_preceding_inherited_fields != other.has_preceding_inherited_fields
-        {
-            return false;
-        }
-
-        for (i, step) in self.production.steps.iter().enumerate() {
-            // See the previous comment (in the `Hash::hash` impl) regarding comparisons
-            // of parse items' already-completed steps.
-            if i < self.step_index as usize {
-                if step.alias != other.production.steps[i].alias {
-                    return false;
-                }
-                if step.field_name != other.production.steps[i].field_name {
-                    return false;
-                }
-                if self.has_preceding_inherited_fields
-                    && step.symbol != other.production.steps[i].symbol
-                {
-                    return false;
-                }
-            } else if *step != other.production.steps[i] {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
-impl<'a> Ord for ParseItem<'a> {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.step_index
-            .cmp(&other.step_index)
-            .then_with(|| self.variable_index.cmp(&other.variable_index))
-            .then_with(|| {
-                self.production
-                    .dynamic_precedence
-                    .cmp(&other.production.dynamic_precedence)
-            })
-            .then_with(|| {
-                self.production
-                    .steps
-                    .len()
-                    .cmp(&other.production.steps.len())
-            })
-            .then_with(|| self.precedence().cmp(other.precedence()))
-            .then_with(|| self.associativity().cmp(&other.associativity()))
-            .then_with(|| {
-                for (i, step) in self.production.steps.iter().enumerate() {
-                    // See the previous comment (in the `Hash::hash` impl) regarding comparisons
-                    // of parse items' already-completed steps.
-                    let o = if i < self.step_index as usize {
-                        step.alias
-                            .cmp(&other.production.steps[i].alias)
-                            .then_with(|| {
-                                step.field_name.cmp(&other.production.steps[i].field_name)
-                            })
-                    } else {
-                        step.cmp(&other.production.steps[i])
-                    };
-                    if o != Ordering::Equal {
-                        return o;
-                    }
-                }
-                Ordering::Equal
-            })
-    }
-}
-
-impl<'a> PartialOrd for ParseItem<'a> {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl<'a> Eq for ParseItem<'a> {}
-
-impl<'a> Hash for ParseItemSet<'a> {
-    fn hash<H: Hasher>(&self, hasher: &mut H) {
-        hasher.write_usize(self.entries.len());
-        for (item, lookaheads) in &self.entries {
-            item.hash(hasher);
-            lookaheads.hash(hasher);
-        }
-    }
-}
-
-impl<'a> Hash for ParseItemSetCore<'a> {
-    fn hash<H: Hasher>(&self, hasher: &mut H) {
-        hasher.write_usize(self.entries.len());
-        for item in &self.entries {
-            item.hash(hasher);
-        }
-    }
-}

cli/generate/src/build_tables/item_set_builder.rs

@@ -1,345 +0,0 @@
-use std::{
-    collections::{HashMap, HashSet},
-    fmt,
-};
-
-use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, TokenSetDisplay};
-use crate::{
-    grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
-    rules::{Symbol, SymbolType, TokenSet},
-};
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-struct TransitiveClosureAddition<'a> {
-    item: ParseItem<'a>,
-    info: FollowSetInfo,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-struct FollowSetInfo {
-    lookaheads: TokenSet,
-    propagates_lookaheads: bool,
-}
-
-pub struct ParseItemSetBuilder<'a> {
-    syntax_grammar: &'a SyntaxGrammar,
-    lexical_grammar: &'a LexicalGrammar,
-    first_sets: HashMap<Symbol, TokenSet>,
-    last_sets: HashMap<Symbol, TokenSet>,
-    inlines: &'a InlinedProductionMap,
-    transitive_closure_additions: Vec<Vec<TransitiveClosureAddition<'a>>>,
-}
-
-fn find_or_push<T: Eq>(vector: &mut Vec<T>, value: T) {
-    if !vector.contains(&value) {
-        vector.push(value);
-    }
-}
-
-impl<'a> ParseItemSetBuilder<'a> {
-    pub fn new(
-        syntax_grammar: &'a SyntaxGrammar,
-        lexical_grammar: &'a LexicalGrammar,
-        inlines: &'a InlinedProductionMap,
-    ) -> Self {
-        let mut result = Self {
-            syntax_grammar,
-            lexical_grammar,
-            first_sets: HashMap::new(),
-            last_sets: HashMap::new(),
-            inlines,
-            transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()],
-        };
-
-        // For each grammar symbol, populate the FIRST and LAST sets: the set of
-        // terminals that appear at the beginning and end that symbol's productions,
-        // respectively.
-        //
-        // For a terminal symbol, the FIRST and LAST set just consists of the
-        // terminal itself.
-        for i in 0..lexical_grammar.variables.len() {
-            let symbol = Symbol::terminal(i);
-            let mut set = TokenSet::new();
-            set.insert(symbol);
-            result.first_sets.insert(symbol, set.clone());
-            result.last_sets.insert(symbol, set);
-        }
-
-        for i in 0..syntax_grammar.external_tokens.len() {
-            let symbol = Symbol::external(i);
-            let mut set = TokenSet::new();
-            set.insert(symbol);
-            result.first_sets.insert(symbol, set.clone());
-            result.last_sets.insert(symbol, set);
-        }
-
-        // The FIRST set of a non-terminal `i` is the union of the following sets:
-        // * the set of all terminals that appear at the beginnings of i's productions
-        // * the FIRST sets of all the non-terminals that appear at the beginnings of i's
-        //   productions
-        //
-        // Rather than computing these sets using recursion, we use an explicit stack
-        // called `symbols_to_process`.
-        let mut symbols_to_process = Vec::new();
-        let mut processed_non_terminals = HashSet::new();
-        for i in 0..syntax_grammar.variables.len() {
-            let symbol = Symbol::non_terminal(i);
-
-            let first_set = result
-                .first_sets
-                .entry(symbol)
-                .or_insert_with(TokenSet::new);
-            processed_non_terminals.clear();
-            symbols_to_process.clear();
-            symbols_to_process.push(symbol);
-            while let Some(current_symbol) = symbols_to_process.pop() {
-                if current_symbol.is_terminal() || current_symbol.is_external() {
-                    first_set.insert(current_symbol);
-                } else if processed_non_terminals.insert(current_symbol) {
-                    for production in &syntax_grammar.variables[current_symbol.index].productions {
-                        if let Some(step) = production.steps.first() {
-                            symbols_to_process.push(step.symbol);
-                        }
-                    }
-                }
-            }
-
-            // The LAST set is defined in a similar way to the FIRST set.
-            let last_set = result.last_sets.entry(symbol).or_insert_with(TokenSet::new);
-            processed_non_terminals.clear();
-            symbols_to_process.clear();
-            symbols_to_process.push(symbol);
-            while let Some(current_symbol) = symbols_to_process.pop() {
-                if current_symbol.is_terminal() || current_symbol.is_external() {
-                    last_set.insert(current_symbol);
-                } else if processed_non_terminals.insert(current_symbol) {
-                    for production in &syntax_grammar.variables[current_symbol.index].productions {
-                        if let Some(step) = production.steps.last() {
-                            symbols_to_process.push(step.symbol);
-                        }
-                    }
-                }
-            }
-        }
-
-        // To compute an item set's transitive closure, we find each item in the set
-        // whose next symbol is a non-terminal, and we add new items to the set for
-        // each of that symbols' productions. These productions might themselves begin
-        // with non-terminals, so the process continues recursively. In this process,
-        // the total set of entries that get added depends only on two things:
-        //   * the set of non-terminal symbols that occur at each item's current position
-        //   * the set of terminals that occurs after each of these non-terminal symbols
-        //
-        // So we can avoid a lot of duplicated recursive work by precomputing, for each
-        // non-terminal symbol `i`, a final list of *additions* that must be made to an
-        // item set when `i` occurs as the next symbol in one if its core items. The
-        // structure of an *addition* is as follows:
-        //   * `item` - the new item that must be added as part of the expansion of `i`
-        //   * `lookaheads` - lookahead tokens that can always come after that item in the expansion
-        //     of `i`
-        //   * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the
-        //     *end* of the expansion of `i`, so that i's own current lookahead tokens can occur
-        //     after `item`.
-        //
-        // Again, rather than computing these additions recursively, we use an explicit
-        // stack called `entries_to_process`.
-        for i in 0..syntax_grammar.variables.len() {
-            let empty_lookaheads = TokenSet::new();
-            let mut entries_to_process = vec![(i, &empty_lookaheads, true)];
-
-            // First, build up a map whose keys are all of the non-terminals that can
-            // appear at the beginning of non-terminal `i`, and whose values store
-            // information about the tokens that can follow each non-terminal.
-            let mut follow_set_info_by_non_terminal = HashMap::new();
-            while let Some(entry) = entries_to_process.pop() {
-                let (variable_index, lookaheads, propagates_lookaheads) = entry;
-                let existing_info = follow_set_info_by_non_terminal
-                    .entry(variable_index)
-                    .or_insert_with(|| FollowSetInfo {
-                        lookaheads: TokenSet::new(),
-                        propagates_lookaheads: false,
-                    });
-
-                let did_add_follow_set_info;
-                if propagates_lookaheads {
-                    did_add_follow_set_info = !existing_info.propagates_lookaheads;
-                    existing_info.propagates_lookaheads = true;
-                } else {
-                    did_add_follow_set_info = existing_info.lookaheads.insert_all(lookaheads);
-                }
-
-                if did_add_follow_set_info {
-                    for production in &syntax_grammar.variables[variable_index].productions {
-                        if let Some(symbol) = production.first_symbol() {
-                            if symbol.is_non_terminal() {
-                                if production.steps.len() == 1 {
-                                    entries_to_process.push((
-                                        symbol.index,
-                                        lookaheads,
-                                        propagates_lookaheads,
-                                    ));
-                                } else {
-                                    entries_to_process.push((
-                                        symbol.index,
-                                        &result.first_sets[&production.steps[1].symbol],
-                                        false,
-                                    ));
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-
-            // Store all of those non-terminals' productions, along with their associated
-            // lookahead info, as *additions* associated with non-terminal `i`.
-            let additions_for_non_terminal = &mut result.transitive_closure_additions[i];
-            for (variable_index, follow_set_info) in follow_set_info_by_non_terminal {
-                let variable = &syntax_grammar.variables[variable_index];
-                let non_terminal = Symbol::non_terminal(variable_index);
-                let variable_index = variable_index as u32;
-                if syntax_grammar.variables_to_inline.contains(&non_terminal) {
-                    continue;
-                }
-                for production in &variable.productions {
-                    let item = ParseItem {
-                        variable_index,
-                        production,
-                        step_index: 0,
-                        has_preceding_inherited_fields: false,
-                    };
-
-                    if let Some(inlined_productions) =
-                        inlines.inlined_productions(item.production, item.step_index)
-                    {
-                        for production in inlined_productions {
-                            find_or_push(
-                                additions_for_non_terminal,
-                                TransitiveClosureAddition {
-                                    item: item.substitute_production(production),
-                                    info: follow_set_info.clone(),
-                                },
-                            );
-                        }
-                    } else {
-                        find_or_push(
-                            additions_for_non_terminal,
-                            TransitiveClosureAddition {
-                                item,
-                                info: follow_set_info.clone(),
-                            },
-                        );
-                    }
-                }
-            }
-        }
-
-        result
-    }
-
-    pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> {
-        let mut result = ParseItemSet::default();
-        for (item, lookaheads) in &item_set.entries {
-            if let Some(productions) = self
-                .inlines
-                .inlined_productions(item.production, item.step_index)
-            {
-                for production in productions {
-                    self.add_item(
-                        &mut result,
-                        item.substitute_production(production),
-                        lookaheads,
-                    );
-                }
-            } else {
-                self.add_item(&mut result, *item, lookaheads);
-            }
-        }
-        result
-    }
-
-    pub fn first_set(&self, symbol: &Symbol) -> &TokenSet {
-        &self.first_sets[symbol]
-    }
-
-    pub fn last_set(&self, symbol: &Symbol) -> &TokenSet {
-        &self.last_sets[symbol]
-    }
-
-    fn add_item(&self, set: &mut ParseItemSet<'a>, item: ParseItem<'a>, lookaheads: &TokenSet) {
-        if let Some(step) = item.step() {
-            if step.symbol.is_non_terminal() {
-                let next_step = item.successor().step();
-
-                // Determine which tokens can follow this non-terminal.
-                let following_tokens = next_step.map_or(lookaheads, |next_step| {
-                    self.first_sets.get(&next_step.symbol).unwrap()
-                });
-
-                // Use the pre-computed *additions* to expand the non-terminal.
-                for addition in &self.transitive_closure_additions[step.symbol.index] {
-                    let lookaheads = set.insert(addition.item, &addition.info.lookaheads);
-                    if addition.info.propagates_lookaheads {
-                        lookaheads.insert_all(following_tokens);
-                    }
-                }
-            }
-        }
-        set.insert(item, lookaheads);
-    }
-}
-
-impl<'a> fmt::Debug for ParseItemSetBuilder<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "ParseItemSetBuilder {{")?;
-
-        writeln!(f, "  first_sets: {{")?;
-        for (symbol, first_set) in &self.first_sets {
-            let name = match symbol.kind {
-                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
-                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
-                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
-                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
-            };
-            writeln!(
-                f,
-                "    first({name:?}): {}",
-                TokenSetDisplay(first_set, self.syntax_grammar, self.lexical_grammar)
-            )?;
-        }
-        writeln!(f, "  }}")?;
-
-        writeln!(f, "  last_sets: {{")?;
-        for (symbol, last_set) in &self.last_sets {
-            let name = match symbol.kind {
-                SymbolType::NonTerminal => &self.syntax_grammar.variables[symbol.index].name,
-                SymbolType::External => &self.syntax_grammar.external_tokens[symbol.index].name,
-                SymbolType::Terminal => &self.lexical_grammar.variables[symbol.index].name,
-                SymbolType::End | SymbolType::EndOfNonTerminalExtra => "END",
-            };
-            writeln!(
-                f,
-                "    last({name:?}): {}",
-                TokenSetDisplay(last_set, self.syntax_grammar, self.lexical_grammar)
-            )?;
-        }
-        writeln!(f, "  }}")?;
-
-        writeln!(f, "  additions: {{")?;
-        for (i, variable) in self.syntax_grammar.variables.iter().enumerate() {
-            writeln!(f, "    {}: {{", variable.name)?;
-            for addition in &self.transitive_closure_additions[i] {
-                writeln!(
-                    f,
-                    "      {}",
-                    ParseItemDisplay(&addition.item, self.syntax_grammar, self.lexical_grammar)
-                )?;
-            }
-            writeln!(f, "    }},")?;
-        }
-        write!(f, "  }},")?;
-
-        write!(f, "}}")?;
-        Ok(())
-    }
-}

cli/generate/src/build_tables/minimize_parse_table.rs

@@ -1,493 +0,0 @@
-use std::{
-    collections::{HashMap, HashSet},
-    mem,
-};
-
-use log::info;
-
-use super::token_conflicts::TokenConflictMap;
-use crate::{
-    dedup::split_state_id_groups,
-    grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
-    rules::{AliasMap, Symbol, TokenSet},
-    tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
-};
-
-pub fn minimize_parse_table(
-    parse_table: &mut ParseTable,
-    syntax_grammar: &SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-    simple_aliases: &AliasMap,
-    token_conflict_map: &TokenConflictMap,
-    keywords: &TokenSet,
-) {
-    let mut minimizer = Minimizer {
-        parse_table,
-        syntax_grammar,
-        lexical_grammar,
-        token_conflict_map,
-        keywords,
-        simple_aliases,
-    };
-    minimizer.merge_compatible_states();
-    minimizer.remove_unit_reductions();
-    minimizer.remove_unused_states();
-    minimizer.reorder_states_by_descending_size();
-}
-
-struct Minimizer<'a> {
-    parse_table: &'a mut ParseTable,
-    syntax_grammar: &'a SyntaxGrammar,
-    lexical_grammar: &'a LexicalGrammar,
-    token_conflict_map: &'a TokenConflictMap<'a>,
-    keywords: &'a TokenSet,
-    simple_aliases: &'a AliasMap,
-}
-
-impl<'a> Minimizer<'a> {
-    fn remove_unit_reductions(&mut self) {
-        let mut aliased_symbols = HashSet::new();
-        for variable in &self.syntax_grammar.variables {
-            for production in &variable.productions {
-                for step in &production.steps {
-                    if step.alias.is_some() {
-                        aliased_symbols.insert(step.symbol);
-                    }
-                }
-            }
-        }
-
-        let mut unit_reduction_symbols_by_state = HashMap::new();
-        for (i, state) in self.parse_table.states.iter().enumerate() {
-            let mut only_unit_reductions = true;
-            let mut unit_reduction_symbol = None;
-            for (_, entry) in &state.terminal_entries {
-                for action in &entry.actions {
-                    match action {
-                        ParseAction::ShiftExtra => continue,
-                        ParseAction::Reduce {
-                            child_count: 1,
-                            production_id: 0,
-                            symbol,
-                            ..
-                        } => {
-                            if !self.simple_aliases.contains_key(symbol)
-                                && !self.syntax_grammar.supertype_symbols.contains(symbol)
-                                && !aliased_symbols.contains(symbol)
-                                && self.syntax_grammar.variables[symbol.index].kind
-                                    != VariableType::Named
-                                && (unit_reduction_symbol.is_none()
-                                    || unit_reduction_symbol == Some(symbol))
-                            {
-                                unit_reduction_symbol = Some(symbol);
-                                continue;
-                            }
-                        }
-                        _ => {}
-                    }
-                    only_unit_reductions = false;
-                    break;
-                }
-
-                if !only_unit_reductions {
-                    break;
-                }
-            }
-
-            if let Some(symbol) = unit_reduction_symbol {
-                if only_unit_reductions {
-                    unit_reduction_symbols_by_state.insert(i, *symbol);
-                }
-            }
-        }
-
-        for state in &mut self.parse_table.states {
-            let mut done = false;
-            while !done {
-                done = true;
-                state.update_referenced_states(|other_state_id, state| {
-                    unit_reduction_symbols_by_state.get(&other_state_id).map_or(
-                        other_state_id,
-                        |symbol| {
-                            done = false;
-                            match state.nonterminal_entries.get(symbol) {
-                                Some(GotoAction::Goto(state_id)) => *state_id,
-                                _ => other_state_id,
-                            }
-                        },
-                    )
-                });
-            }
-        }
-    }
-
-    fn merge_compatible_states(&mut self) {
-        let core_count = 1 + self
-            .parse_table
-            .states
-            .iter()
-            .map(|state| state.core_id)
-            .max()
-            .unwrap();
-
-        // Initially group the states by their parse item set core.
-        let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
-        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
-        for (i, state) in self.parse_table.states.iter().enumerate() {
-            state_ids_by_group_id[state.core_id].push(i);
-            group_ids_by_state_id.push(state.core_id);
-        }
-
-        split_state_id_groups(
-            &self.parse_table.states,
-            &mut state_ids_by_group_id,
-            &mut group_ids_by_state_id,
-            0,
-            |left, right, groups| self.states_conflict(left, right, groups),
-        );
-
-        while split_state_id_groups(
-            &self.parse_table.states,
-            &mut state_ids_by_group_id,
-            &mut group_ids_by_state_id,
-            0,
-            |left, right, groups| self.state_successors_differ(left, right, groups),
-        ) {
-            continue;
-        }
-
-        let error_group_index = state_ids_by_group_id
-            .iter()
-            .position(|g| g.contains(&0))
-            .unwrap();
-        let start_group_index = state_ids_by_group_id
-            .iter()
-            .position(|g| g.contains(&1))
-            .unwrap();
-        state_ids_by_group_id.swap(error_group_index, 0);
-        state_ids_by_group_id.swap(start_group_index, 1);
-
-        // Create a list of new parse states: one state for each group of old states.
-        let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
-        for state_ids in &state_ids_by_group_id {
-            // Initialize the new state based on the first old state in the group.
-            let mut parse_state = ParseState::default();
-            mem::swap(&mut parse_state, &mut self.parse_table.states[state_ids[0]]);
-
-            // Extend the new state with all of the actions from the other old states
-            // in the group.
-            for state_id in &state_ids[1..] {
-                let mut other_parse_state = ParseState::default();
-                mem::swap(
-                    &mut other_parse_state,
-                    &mut self.parse_table.states[*state_id],
-                );
-
-                parse_state
-                    .terminal_entries
-                    .extend(other_parse_state.terminal_entries);
-                parse_state
-                    .nonterminal_entries
-                    .extend(other_parse_state.nonterminal_entries);
-            }
-
-            // Update the new state's outgoing references using the new grouping.
-            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
-            new_states.push(parse_state);
-        }
-
-        self.parse_table.states = new_states;
-    }
-
-    fn states_conflict(
-        &self,
-        left_state: &ParseState,
-        right_state: &ParseState,
-        group_ids_by_state_id: &[ParseStateId],
-    ) -> bool {
-        for (token, left_entry) in &left_state.terminal_entries {
-            if let Some(right_entry) = right_state.terminal_entries.get(token) {
-                if self.entries_conflict(
-                    left_state.id,
-                    right_state.id,
-                    token,
-                    left_entry,
-                    right_entry,
-                    group_ids_by_state_id,
-                ) {
-                    return true;
-                }
-            } else if self.token_conflicts(
-                left_state.id,
-                right_state.id,
-                right_state.terminal_entries.keys(),
-                *token,
-            ) {
-                return true;
-            }
-        }
-
-        for token in right_state.terminal_entries.keys() {
-            if !left_state.terminal_entries.contains_key(token)
-                && self.token_conflicts(
-                    left_state.id,
-                    right_state.id,
-                    left_state.terminal_entries.keys(),
-                    *token,
-                )
-            {
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn state_successors_differ(
-        &self,
-        state1: &ParseState,
-        state2: &ParseState,
-        group_ids_by_state_id: &[ParseStateId],
-    ) -> bool {
-        for (token, entry1) in &state1.terminal_entries {
-            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap() {
-                if let Some(entry2) = state2.terminal_entries.get(token) {
-                    if let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap() {
-                        let group1 = group_ids_by_state_id[*s1];
-                        let group2 = group_ids_by_state_id[*s2];
-                        if group1 != group2 {
-                            info!(
-                                "split states {} {} - successors for {} are split: {s1} {s2}",
-                                state1.id,
-                                state2.id,
-                                self.symbol_name(token),
-                            );
-                            return true;
-                        }
-                    }
-                }
-            }
-        }
-
-        for (symbol, s1) in &state1.nonterminal_entries {
-            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
-                match (s1, s2) {
-                    (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => continue,
-                    (GotoAction::Goto(s1), GotoAction::Goto(s2)) => {
-                        let group1 = group_ids_by_state_id[*s1];
-                        let group2 = group_ids_by_state_id[*s2];
-                        if group1 != group2 {
-                            info!(
-                                "split states {} {} - successors for {} are split: {s1} {s2}",
-                                state1.id,
-                                state2.id,
-                                self.symbol_name(symbol),
-                            );
-                            return true;
-                        }
-                    }
-                    _ => return true,
-                }
-            }
-        }
-
-        false
-    }
-
-    fn entries_conflict(
-        &self,
-        state_id1: ParseStateId,
-        state_id2: ParseStateId,
-        token: &Symbol,
-        entry1: &ParseTableEntry,
-        entry2: &ParseTableEntry,
-        group_ids_by_state_id: &[ParseStateId],
-    ) -> bool {
-        // To be compatible, entries need to have the same actions.
-        let actions1 = &entry1.actions;
-        let actions2 = &entry2.actions;
-        if actions1.len() != actions2.len() {
-            info!(
-                "split states {state_id1} {state_id2} - differing action counts for token {}",
-                self.symbol_name(token)
-            );
-            return true;
-        }
-
-        for (i, action1) in actions1.iter().enumerate() {
-            let action2 = &actions2[i];
-
-            // Two shift actions are equivalent if their destinations are in the same group.
-            if let (
-                ParseAction::Shift {
-                    state: s1,
-                    is_repetition: is_repetition1,
-                },
-                ParseAction::Shift {
-                    state: s2,
-                    is_repetition: is_repetition2,
-                },
-            ) = (action1, action2)
-            {
-                let group1 = group_ids_by_state_id[*s1];
-                let group2 = group_ids_by_state_id[*s2];
-                if group1 == group2 && is_repetition1 == is_repetition2 {
-                    continue;
-                }
-                info!(
-                    "split states {state_id1} {state_id2} - successors for {} are split: {s1} {s2}",
-                    self.symbol_name(token),
-                );
-                return true;
-            } else if action1 != action2 {
-                info!(
-                    "split states {state_id1} {state_id2} - unequal actions for {}",
-                    self.symbol_name(token),
-                );
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn token_conflicts<'b>(
-        &self,
-        left_id: ParseStateId,
-        right_id: ParseStateId,
-        existing_tokens: impl Iterator<Item = &'b Symbol>,
-        new_token: Symbol,
-    ) -> bool {
-        if new_token == Symbol::end_of_nonterminal_extra() {
-            info!("split states {left_id} {right_id} - end of non-terminal extra",);
-            return true;
-        }
-
-        // Do not add external tokens; they could conflict lexically with any of the state's
-        // existing lookahead tokens.
-        if new_token.is_external() {
-            info!(
-                "split states {left_id} {right_id} - external token {}",
-                self.symbol_name(&new_token),
-            );
-            return true;
-        }
-
-        // Do not add tokens which are both internal and external. Their validity could
-        // influence the behavior of the external scanner.
-        if self
-            .syntax_grammar
-            .external_tokens
-            .iter()
-            .any(|external| external.corresponding_internal_token == Some(new_token))
-        {
-            info!(
-                "split states {left_id} {right_id} - internal/external token {}",
-                self.symbol_name(&new_token),
-            );
-            return true;
-        }
-
-        // Do not add a token if it conflicts with an existing token.
-        for token in existing_tokens {
-            if token.is_terminal()
-                && !(self.syntax_grammar.word_token == Some(*token)
-                    && self.keywords.contains(&new_token))
-                && !(self.syntax_grammar.word_token == Some(new_token)
-                    && self.keywords.contains(token))
-                && (self
-                    .token_conflict_map
-                    .does_conflict(new_token.index, token.index)
-                    || self
-                        .token_conflict_map
-                        .does_match_same_string(new_token.index, token.index))
-            {
-                info!(
-                    "split states {left_id} {right_id} - token {} conflicts with {}",
-                    self.symbol_name(&new_token),
-                    self.symbol_name(token),
-                );
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn symbol_name(&self, symbol: &Symbol) -> &String {
-        if symbol.is_non_terminal() {
-            &self.syntax_grammar.variables[symbol.index].name
-        } else if symbol.is_external() {
-            &self.syntax_grammar.external_tokens[symbol.index].name
-        } else {
-            &self.lexical_grammar.variables[symbol.index].name
-        }
-    }
-
-    fn remove_unused_states(&mut self) {
-        let mut state_usage_map = vec![false; self.parse_table.states.len()];
-
-        state_usage_map[0] = true;
-        state_usage_map[1] = true;
-
-        for state in &self.parse_table.states {
-            for referenced_state in state.referenced_states() {
-                state_usage_map[referenced_state] = true;
-            }
-        }
-        let mut removed_predecessor_count = 0;
-        let mut state_replacement_map = vec![0; self.parse_table.states.len()];
-        for state_id in 0..self.parse_table.states.len() {
-            state_replacement_map[state_id] = state_id - removed_predecessor_count;
-            if !state_usage_map[state_id] {
-                removed_predecessor_count += 1;
-            }
-        }
-        let mut state_id = 0;
-        let mut original_state_id = 0;
-        while state_id < self.parse_table.states.len() {
-            if state_usage_map[original_state_id] {
-                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
-                    state_replacement_map[other_state_id]
-                });
-                state_id += 1;
-            } else {
-                self.parse_table.states.remove(state_id);
-            }
-            original_state_id += 1;
-        }
-    }
-
-    fn reorder_states_by_descending_size(&mut self) {
-        // Get a mapping of old state index -> new_state_index
-        let mut old_ids_by_new_id = (0..self.parse_table.states.len()).collect::<Vec<_>>();
-        old_ids_by_new_id.sort_unstable_by_key(|i| {
-            // Don't changes states 0 (the error state) or 1 (the start state).
-            if *i <= 1 {
-                return *i as i64 - 1_000_000;
-            }
-
-            // Reorder all the other states by descending symbol count.
-            let state = &self.parse_table.states[*i];
-            -((state.terminal_entries.len() + state.nonterminal_entries.len()) as i64)
-        });
-
-        // Get the inverse mapping
-        let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
-        for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
-            new_ids_by_old_id[*old_id] = id;
-        }
-
-        // Reorder the parse states and update their references to reflect
-        // the new ordering.
-        self.parse_table.states = old_ids_by_new_id
-            .iter()
-            .map(|old_id| {
-                let mut state = ParseState::default();
-                mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
-                state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
-                state
-            })
-            .collect();
-    }
-}

cli/generate/src/build_tables/token_conflicts.rs

@@ -1,529 +0,0 @@
-use std::{cmp::Ordering, collections::HashSet, fmt};
-
-use crate::{
-    build_tables::item::TokenSetDisplay,
-    grammars::{LexicalGrammar, SyntaxGrammar},
-    nfa::{CharacterSet, NfaCursor, NfaTransition},
-    rules::TokenSet,
-};
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-struct TokenConflictStatus {
-    matches_prefix: bool,
-    does_match_continuation: bool,
-    does_match_valid_continuation: bool,
-    does_match_separators: bool,
-    matches_same_string: bool,
-    matches_different_string: bool,
-}
-
-pub struct TokenConflictMap<'a> {
-    n: usize,
-    status_matrix: Vec<TokenConflictStatus>,
-    following_tokens: Vec<TokenSet>,
-    starting_chars_by_index: Vec<CharacterSet>,
-    following_chars_by_index: Vec<CharacterSet>,
-    grammar: &'a LexicalGrammar,
-}
-
-impl<'a> TokenConflictMap<'a> {
-    /// Create a token conflict map based on a lexical grammar, which describes the structure
-    /// each token, and a `following_token` map, which indicates which tokens may be appear
-    /// immediately after each other token.
-    ///
-    /// This analyzes the possible kinds of overlap between each pair of tokens and stores
-    /// them in a matrix.
-    pub fn new(grammar: &'a LexicalGrammar, following_tokens: Vec<TokenSet>) -> Self {
-        let mut cursor = NfaCursor::new(&grammar.nfa, Vec::new());
-        let starting_chars = get_starting_chars(&mut cursor, grammar);
-        let following_chars = get_following_chars(&starting_chars, &following_tokens);
-
-        let n = grammar.variables.len();
-        let mut status_matrix = vec![TokenConflictStatus::default(); n * n];
-        for i in 0..grammar.variables.len() {
-            for j in 0..i {
-                let status = compute_conflict_status(&mut cursor, grammar, &following_chars, i, j);
-                status_matrix[matrix_index(n, i, j)] = status.0;
-                status_matrix[matrix_index(n, j, i)] = status.1;
-            }
-        }
-
-        TokenConflictMap {
-            n,
-            status_matrix,
-            following_tokens,
-            starting_chars_by_index: starting_chars,
-            following_chars_by_index: following_chars,
-            grammar,
-        }
-    }
-
-    /// Does token `i` match any strings that token `j` also matches, such that token `i`
-    /// is preferred over token `j`?
-    pub fn has_same_conflict_status(&self, a: usize, b: usize, other: usize) -> bool {
-        let left = &self.status_matrix[matrix_index(self.n, a, other)];
-        let right = &self.status_matrix[matrix_index(self.n, b, other)];
-        left == right
-    }
-
-    /// Does token `i` match any strings that token `j` does *not* match?
-    pub fn does_match_different_string(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_different_string
-    }
-
-    /// Does token `i` match any strings that token `j` also matches, where
-    /// token `i` is preferred over token `j`?
-    pub fn does_match_same_string(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_same_string
-    }
-
-    pub fn does_conflict(&self, i: usize, j: usize) -> bool {
-        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
-        entry.does_match_valid_continuation
-            || entry.does_match_separators
-            || entry.matches_same_string
-    }
-
-    /// Does token `i` match any strings that are *prefixes* of strings matched by `j`?
-    pub fn does_match_prefix(&self, i: usize, j: usize) -> bool {
-        self.status_matrix[matrix_index(self.n, i, j)].matches_prefix
-    }
-
-    pub fn does_match_shorter_or_longer(&self, i: usize, j: usize) -> bool {
-        let entry = &self.status_matrix[matrix_index(self.n, i, j)];
-        let reverse_entry = &self.status_matrix[matrix_index(self.n, j, i)];
-        (entry.does_match_valid_continuation || entry.does_match_separators)
-            && !reverse_entry.does_match_separators
-    }
-
-    pub fn does_overlap(&self, i: usize, j: usize) -> bool {
-        let status = &self.status_matrix[matrix_index(self.n, i, j)];
-        status.does_match_separators
-            || status.matches_prefix
-            || status.matches_same_string
-            || status.does_match_continuation
-    }
-
-    pub fn prefer_token(grammar: &LexicalGrammar, left: (i32, usize), right: (i32, usize)) -> bool {
-        match left.0.cmp(&right.0) {
-            Ordering::Less => false,
-            Ordering::Greater => true,
-            Ordering::Equal => match grammar.variables[left.1]
-                .implicit_precedence
-                .cmp(&grammar.variables[right.1].implicit_precedence)
-            {
-                Ordering::Less => false,
-                Ordering::Greater => true,
-                Ordering::Equal => left.1 < right.1,
-            },
-        }
-    }
-
-    pub fn prefer_transition(
-        grammar: &LexicalGrammar,
-        t: &NfaTransition,
-        completed_id: usize,
-        completed_precedence: i32,
-        has_separator_transitions: bool,
-    ) -> bool {
-        if t.precedence < completed_precedence {
-            return false;
-        }
-        if t.precedence == completed_precedence {
-            if t.is_separator {
-                return false;
-            }
-            if has_separator_transitions
-                && !grammar
-                    .variable_indices_for_nfa_states(&t.states)
-                    .any(|i| i == completed_id)
-            {
-                return false;
-            }
-        }
-        true
-    }
-}
-
-impl<'a> fmt::Debug for TokenConflictMap<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "TokenConflictMap {{")?;
-
-        let syntax_grammar = SyntaxGrammar::default();
-
-        writeln!(f, "  following_tokens: {{")?;
-        for (i, following_tokens) in self.following_tokens.iter().enumerate() {
-            writeln!(
-                f,
-                "    follow({:?}): {},",
-                self.grammar.variables[i].name,
-                TokenSetDisplay(following_tokens, &syntax_grammar, self.grammar)
-            )?;
-        }
-        writeln!(f, "  }},")?;
-
-        writeln!(f, "  starting_characters: {{")?;
-        for i in 0..self.n {
-            writeln!(
-                f,
-                "    {:?}: {:?},",
-                self.grammar.variables[i].name, self.starting_chars_by_index[i]
-            )?;
-        }
-        writeln!(f, "  }},")?;
-
-        writeln!(f, "  following_characters: {{")?;
-        for i in 0..self.n {
-            writeln!(
-                f,
-                "    {:?}: {:?},",
-                self.grammar.variables[i].name, self.following_chars_by_index[i]
-            )?;
-        }
-        writeln!(f, "  }},")?;
-
-        writeln!(f, "  status_matrix: {{")?;
-        for i in 0..self.n {
-            writeln!(f, "    {:?}: {{", self.grammar.variables[i].name)?;
-            for j in 0..self.n {
-                writeln!(
-                    f,
-                    "      {:?}: {:?},",
-                    self.grammar.variables[j].name,
-                    self.status_matrix[matrix_index(self.n, i, j)]
-                )?;
-            }
-            writeln!(f, "    }},")?;
-        }
-        write!(f, "  }},")?;
-        write!(f, "}}")?;
-        Ok(())
-    }
-}
-
-const fn matrix_index(variable_count: usize, i: usize, j: usize) -> usize {
-    variable_count * i + j
-}
-
-fn get_starting_chars(cursor: &mut NfaCursor, grammar: &LexicalGrammar) -> Vec<CharacterSet> {
-    let mut result = Vec::with_capacity(grammar.variables.len());
-    for variable in &grammar.variables {
-        cursor.reset(vec![variable.start_state]);
-        let mut all_chars = CharacterSet::empty();
-        for (chars, _) in cursor.transition_chars() {
-            all_chars = all_chars.add(chars);
-        }
-        result.push(all_chars);
-    }
-    result
-}
-
-fn get_following_chars(
-    starting_chars: &[CharacterSet],
-    following_tokens: &[TokenSet],
-) -> Vec<CharacterSet> {
-    following_tokens
-        .iter()
-        .map(|following_tokens| {
-            let mut chars = CharacterSet::empty();
-            for token in following_tokens.iter() {
-                if token.is_terminal() {
-                    chars = chars.add(&starting_chars[token.index]);
-                }
-            }
-            chars
-        })
-        .collect()
-}
-
-fn compute_conflict_status(
-    cursor: &mut NfaCursor,
-    grammar: &LexicalGrammar,
-    following_chars: &[CharacterSet],
-    i: usize,
-    j: usize,
-) -> (TokenConflictStatus, TokenConflictStatus) {
-    let mut visited_state_sets = HashSet::new();
-    let mut state_set_queue = vec![vec![
-        grammar.variables[i].start_state,
-        grammar.variables[j].start_state,
-    ]];
-    let mut result = (
-        TokenConflictStatus::default(),
-        TokenConflictStatus::default(),
-    );
-
-    while let Some(state_set) = state_set_queue.pop() {
-        let mut live_variable_indices = grammar.variable_indices_for_nfa_states(&state_set);
-
-        // If only one of the two tokens could possibly match from this state, then
-        // there is no reason to analyze any of its successors. Just record the fact
-        // that the token matches a string that the other token does not match.
-        let first_live_variable_index = live_variable_indices.next().unwrap();
-        if live_variable_indices.count() == 0 {
-            if first_live_variable_index == i {
-                result.0.matches_different_string = true;
-            } else {
-                result.1.matches_different_string = true;
-            }
-            continue;
-        }
-
-        // Don't pursue states where there's no potential for conflict.
-        cursor.reset(state_set);
-        let within_separator = cursor.transition_chars().any(|(_, sep)| sep);
-
-        // Examine each possible completed token in this state.
-        let mut completion = None;
-        for (id, precedence) in cursor.completions() {
-            if within_separator {
-                if id == i {
-                    result.0.does_match_separators = true;
-                } else {
-                    result.1.does_match_separators = true;
-                }
-            }
-
-            // If the other token has already completed, then this is
-            // a same-string conflict.
-            if let Some((prev_id, prev_precedence)) = completion {
-                if id == prev_id {
-                    continue;
-                }
-
-                // Determine which of the two tokens is preferred.
-                let preferred_id;
-                if TokenConflictMap::prefer_token(
-                    grammar,
-                    (prev_precedence, prev_id),
-                    (precedence, id),
-                ) {
-                    preferred_id = prev_id;
-                } else {
-                    preferred_id = id;
-                    completion = Some((id, precedence));
-                }
-
-                if preferred_id == i {
-                    result.0.matches_same_string = true;
-                } else {
-                    result.1.matches_same_string = true;
-                }
-            } else {
-                completion = Some((id, precedence));
-            }
-        }
-
-        // Examine each possible transition from this state to detect substring conflicts.
-        for transition in cursor.transitions() {
-            let mut can_advance = true;
-
-            // If there is already a completed token in this state, then determine
-            // if the next state can also match the completed token. If so, then
-            // this is *not* a conflict.
-            if let Some((completed_id, completed_precedence)) = completion {
-                let mut advanced_id = None;
-                let mut successor_contains_completed_id = false;
-                for variable_id in grammar.variable_indices_for_nfa_states(&transition.states) {
-                    if variable_id == completed_id {
-                        successor_contains_completed_id = true;
-                        break;
-                    }
-                    advanced_id = Some(variable_id);
-                }
-
-                // Determine which action is preferred: matching the already complete
-                // token, or continuing on to try and match the other longer token.
-                if let (Some(advanced_id), false) = (advanced_id, successor_contains_completed_id) {
-                    if TokenConflictMap::prefer_transition(
-                        grammar,
-                        &transition,
-                        completed_id,
-                        completed_precedence,
-                        within_separator,
-                    ) {
-                        can_advance = true;
-                        if advanced_id == i {
-                            result.0.does_match_continuation = true;
-                            if transition.characters.does_intersect(&following_chars[j]) {
-                                result.0.does_match_valid_continuation = true;
-                            }
-                        } else {
-                            result.1.does_match_continuation = true;
-                            if transition.characters.does_intersect(&following_chars[i]) {
-                                result.1.does_match_valid_continuation = true;
-                            }
-                        }
-                    } else if completed_id == i {
-                        result.0.matches_prefix = true;
-                    } else {
-                        result.1.matches_prefix = true;
-                    }
-                }
-            }
-
-            if can_advance && visited_state_sets.insert(transition.states.clone()) {
-                state_set_queue.push(transition.states);
-            }
-        }
-    }
-    result
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::{Variable, VariableType},
-        prepare_grammar::{expand_tokens, ExtractedLexicalGrammar},
-        rules::{Precedence, Rule, Symbol},
-    };
-
-    #[test]
-    fn test_starting_characters() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: Vec::new(),
-            variables: vec![
-                Variable {
-                    name: "token_0".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("[a-f]1|0x\\d", ""),
-                },
-                Variable {
-                    name: "token_1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("d*ef", ""),
-                },
-            ],
-        })
-        .unwrap();
-
-        let token_map = TokenConflictMap::new(&grammar, Vec::new());
-
-        assert_eq!(
-            token_map.starting_chars_by_index[0],
-            CharacterSet::empty().add_range('a', 'f').add_char('0')
-        );
-        assert_eq!(
-            token_map.starting_chars_by_index[1],
-            CharacterSet::empty().add_range('d', 'e')
-        );
-    }
-
-    #[test]
-    fn test_token_conflicts() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: Vec::new(),
-            variables: vec![
-                Variable {
-                    name: "in".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("in"),
-                },
-                Variable {
-                    name: "identifier".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::pattern("\\w+", ""),
-                },
-                Variable {
-                    name: "instanceof".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("instanceof"),
-                },
-            ],
-        })
-        .unwrap();
-
-        let var = |name| index_of_var(&grammar, name);
-
-        let token_map = TokenConflictMap::new(
-            &grammar,
-            vec![
-                std::iter::once(&Symbol::terminal(var("identifier")))
-                    .copied()
-                    .collect(),
-                std::iter::once(&Symbol::terminal(var("in")))
-                    .copied()
-                    .collect(),
-                std::iter::once(&Symbol::terminal(var("identifier")))
-                    .copied()
-                    .collect(),
-            ],
-        );
-
-        // Given the string "in", the `in` token is preferred over the `identifier` token
-        assert!(token_map.does_match_same_string(var("in"), var("identifier")));
-        assert!(!token_map.does_match_same_string(var("identifier"), var("in")));
-
-        // Depending on what character follows, the string "in" may be treated as part of an
-        // `identifier` token.
-        assert!(token_map.does_conflict(var("identifier"), var("in")));
-
-        // Depending on what character follows, the string "instanceof" may be treated as part of
-        // an `identifier` token.
-        assert!(token_map.does_conflict(var("identifier"), var("instanceof")));
-        assert!(token_map.does_conflict(var("instanceof"), var("in")));
-    }
-
-    #[test]
-    fn test_token_conflicts_with_separators() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: vec![Rule::pattern("\\s", "")],
-            variables: vec![
-                Variable {
-                    name: "x".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("x"),
-                },
-                Variable {
-                    name: "newline".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("\n"),
-                },
-            ],
-        })
-        .unwrap();
-
-        let var = |name| index_of_var(&grammar, name);
-
-        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
-
-        assert!(token_map.does_conflict(var("newline"), var("x")));
-        assert!(!token_map.does_conflict(var("x"), var("newline")));
-    }
-
-    #[test]
-    fn test_token_conflicts_with_open_ended_tokens() {
-        let grammar = expand_tokens(ExtractedLexicalGrammar {
-            separators: vec![Rule::pattern("\\s", "")],
-            variables: vec![
-                Variable {
-                    name: "x".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::string("x"),
-                },
-                Variable {
-                    name: "anything".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*", "")),
-                },
-            ],
-        })
-        .unwrap();
-
-        let var = |name| index_of_var(&grammar, name);
-
-        let token_map = TokenConflictMap::new(&grammar, vec![TokenSet::new(); 4]);
-
-        assert!(token_map.does_match_shorter_or_longer(var("anything"), var("x")));
-        assert!(!token_map.does_match_shorter_or_longer(var("x"), var("anything")));
-    }
-
-    fn index_of_var(grammar: &LexicalGrammar, name: &str) -> usize {
-        grammar
-            .variables
-            .iter()
-            .position(|v| v.name == name)
-            .unwrap()
-    }
-}

cli/generate/src/dedup.rs

@@ -1,63 +0,0 @@
-pub fn split_state_id_groups<S>(
-    states: &[S],
-    state_ids_by_group_id: &mut Vec<Vec<usize>>,
-    group_ids_by_state_id: &mut [usize],
-    start_group_id: usize,
-    mut f: impl FnMut(&S, &S, &[usize]) -> bool,
-) -> bool {
-    let mut result = false;
-
-    let mut group_id = start_group_id;
-    while group_id < state_ids_by_group_id.len() {
-        let state_ids = &state_ids_by_group_id[group_id];
-        let mut split_state_ids = Vec::new();
-
-        let mut i = 0;
-        while i < state_ids.len() {
-            let left_state_id = state_ids[i];
-            if split_state_ids.contains(&left_state_id) {
-                i += 1;
-                continue;
-            }
-
-            let left_state = &states[left_state_id];
-
-            // Identify all of the other states in the group that are incompatible with
-            // this state.
-            let mut j = i + 1;
-            while j < state_ids.len() {
-                let right_state_id = state_ids[j];
-                if split_state_ids.contains(&right_state_id) {
-                    j += 1;
-                    continue;
-                }
-                let right_state = &states[right_state_id];
-
-                if f(left_state, right_state, group_ids_by_state_id) {
-                    split_state_ids.push(right_state_id);
-                }
-
-                j += 1;
-            }
-
-            i += 1;
-        }
-
-        // If any states were removed from the group, add them all as a new group.
-        if !split_state_ids.is_empty() {
-            result = true;
-            state_ids_by_group_id[group_id].retain(|i| !split_state_ids.contains(i));
-
-            let new_group_id = state_ids_by_group_id.len();
-            for id in &split_state_ids {
-                group_ids_by_state_id[*id] = new_group_id;
-            }
-
-            state_ids_by_group_id.push(split_state_ids);
-        }
-
-        group_id += 1;
-    }
-
-    result
-}

cli/generate/src/grammar_files.rs

@@ -1 +0,0 @@
-

cli/generate/src/grammars.rs

@@ -1,261 +0,0 @@
-use std::{collections::HashMap, fmt};
-
-use super::{
-    nfa::Nfa,
-    rules::{Alias, Associativity, Precedence, Rule, Symbol},
-};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub enum VariableType {
-    Hidden,
-    Auxiliary,
-    Anonymous,
-    Named,
-}
-
-// Input grammar
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct Variable {
-    pub name: String,
-    pub kind: VariableType,
-    pub rule: Rule,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum PrecedenceEntry {
-    Name(String),
-    Symbol(String),
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct InputGrammar {
-    pub name: String,
-    pub variables: Vec<Variable>,
-    pub extra_symbols: Vec<Rule>,
-    pub expected_conflicts: Vec<Vec<String>>,
-    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
-    pub external_tokens: Vec<Rule>,
-    pub variables_to_inline: Vec<String>,
-    pub supertype_symbols: Vec<String>,
-    pub word_token: Option<String>,
-}
-
-// Extracted lexical grammar
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct LexicalVariable {
-    pub name: String,
-    pub kind: VariableType,
-    pub implicit_precedence: i32,
-    pub start_state: u32,
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct LexicalGrammar {
-    pub nfa: Nfa,
-    pub variables: Vec<LexicalVariable>,
-}
-
-// Extracted syntax grammar
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct ProductionStep {
-    pub symbol: Symbol,
-    pub precedence: Precedence,
-    pub associativity: Option<Associativity>,
-    pub alias: Option<Alias>,
-    pub field_name: Option<String>,
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct Production {
-    pub steps: Vec<ProductionStep>,
-    pub dynamic_precedence: i32,
-}
-
-#[derive(Default)]
-pub struct InlinedProductionMap {
-    pub productions: Vec<Production>,
-    pub production_map: HashMap<(*const Production, u32), Vec<usize>>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct SyntaxVariable {
-    pub name: String,
-    pub kind: VariableType,
-    pub productions: Vec<Production>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct ExternalToken {
-    pub name: String,
-    pub kind: VariableType,
-    pub corresponding_internal_token: Option<Symbol>,
-}
-
-#[derive(Debug, Default)]
-pub struct SyntaxGrammar {
-    pub variables: Vec<SyntaxVariable>,
-    pub extra_symbols: Vec<Symbol>,
-    pub expected_conflicts: Vec<Vec<Symbol>>,
-    pub external_tokens: Vec<ExternalToken>,
-    pub supertype_symbols: Vec<Symbol>,
-    pub variables_to_inline: Vec<Symbol>,
-    pub word_token: Option<Symbol>,
-    pub precedence_orderings: Vec<Vec<PrecedenceEntry>>,
-}
-
-#[cfg(test)]
-impl ProductionStep {
-    #[must_use]
-    pub const fn new(symbol: Symbol) -> Self {
-        Self {
-            symbol,
-            precedence: Precedence::None,
-            associativity: None,
-            alias: None,
-            field_name: None,
-        }
-    }
-
-    pub fn with_prec(self, precedence: Precedence, associativity: Option<Associativity>) -> Self {
-        Self {
-            symbol: self.symbol,
-            precedence,
-            associativity,
-            alias: self.alias,
-            field_name: self.field_name,
-        }
-    }
-
-    pub fn with_alias(self, value: &str, is_named: bool) -> Self {
-        Self {
-            symbol: self.symbol,
-            precedence: self.precedence,
-            associativity: self.associativity,
-            alias: Some(Alias {
-                value: value.to_string(),
-                is_named,
-            }),
-            field_name: self.field_name,
-        }
-    }
-    pub fn with_field_name(self, name: &str) -> Self {
-        Self {
-            symbol: self.symbol,
-            precedence: self.precedence,
-            associativity: self.associativity,
-            alias: self.alias,
-            field_name: Some(name.to_string()),
-        }
-    }
-}
-
-impl Production {
-    pub fn first_symbol(&self) -> Option<Symbol> {
-        self.steps.first().map(|s| s.symbol)
-    }
-}
-
-#[cfg(test)]
-impl Variable {
-    pub fn named(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Named,
-            rule,
-        }
-    }
-
-    pub fn auxiliary(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Auxiliary,
-            rule,
-        }
-    }
-
-    pub fn hidden(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Hidden,
-            rule,
-        }
-    }
-
-    pub fn anonymous(name: &str, rule: Rule) -> Self {
-        Self {
-            name: name.to_string(),
-            kind: VariableType::Anonymous,
-            rule,
-        }
-    }
-}
-
-impl VariableType {
-    pub fn is_visible(self) -> bool {
-        self == Self::Named || self == Self::Anonymous
-    }
-}
-
-impl LexicalGrammar {
-    pub fn variable_indices_for_nfa_states<'a>(
-        &'a self,
-        state_ids: &'a [u32],
-    ) -> impl Iterator<Item = usize> + 'a {
-        let mut prev = None;
-        state_ids.iter().filter_map(move |state_id| {
-            let variable_id = self.variable_index_for_nfa_state(*state_id);
-            if prev == Some(variable_id) {
-                None
-            } else {
-                prev = Some(variable_id);
-                prev
-            }
-        })
-    }
-
-    pub fn variable_index_for_nfa_state(&self, state_id: u32) -> usize {
-        self.variables
-            .iter()
-            .position(|v| v.start_state >= state_id)
-            .unwrap()
-    }
-}
-
-impl SyntaxVariable {
-    pub fn is_auxiliary(&self) -> bool {
-        self.kind == VariableType::Auxiliary
-    }
-
-    pub fn is_hidden(&self) -> bool {
-        self.kind == VariableType::Hidden || self.kind == VariableType::Auxiliary
-    }
-}
-
-impl InlinedProductionMap {
-    pub fn inlined_productions<'a>(
-        &'a self,
-        production: &Production,
-        step_index: u32,
-    ) -> Option<impl Iterator<Item = &'a Production> + 'a> {
-        self.production_map
-            .get(&(production as *const Production, step_index))
-            .map(|production_indices| {
-                production_indices
-                    .iter()
-                    .copied()
-                    .map(move |index| &self.productions[index])
-            })
-    }
-}
-
-impl fmt::Display for PrecedenceEntry {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Self::Name(n) => write!(f, "'{n}'"),
-            Self::Symbol(s) => write!(f, "$.{s}"),
-        }
-    }
-}

cli/generate/src/lib.rs

@@ -1,252 +0,0 @@
-use std::{
-    env, fs,
-    io::Write,
-    path::{Path, PathBuf},
-    process::{Command, Stdio},
-};
-
-use anyhow::{anyhow, Context, Result};
-use build_tables::build_tables;
-use grammars::InputGrammar;
-use lazy_static::lazy_static;
-use parse_grammar::parse_grammar;
-use prepare_grammar::prepare_grammar;
-use regex::{Regex, RegexBuilder};
-use render::render_c_code;
-use semver::Version;
-
-mod build_tables;
-mod dedup;
-mod grammar_files;
-mod grammars;
-mod nfa;
-mod node_types;
-pub mod parse_grammar;
-mod prepare_grammar;
-mod render;
-mod rules;
-mod tables;
-
-lazy_static! {
-    static ref JSON_COMMENT_REGEX: Regex = RegexBuilder::new("^\\s*//.*")
-        .multi_line(true)
-        .build()
-        .unwrap();
-}
-
-struct GeneratedParser {
-    c_code: String,
-    node_types_json: String,
-}
-
-pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h");
-pub const ARRAY_HEADER: &str = include_str!("templates/array.h");
-
-pub fn generate_parser_in_directory(
-    repo_path: &Path,
-    grammar_path: Option<&str>,
-    abi_version: usize,
-    report_symbol_name: Option<&str>,
-    js_runtime: Option<&str>,
-) -> Result<()> {
-    let mut repo_path = repo_path.to_owned();
-    let mut grammar_path = grammar_path;
-
-    // Populate a new empty grammar directory.
-    if let Some(path) = grammar_path {
-        let path = PathBuf::from(path);
-        if !path
-            .try_exists()
-            .with_context(|| "Some error with specified path")?
-        {
-            fs::create_dir_all(&path)?;
-            grammar_path = None;
-            repo_path = path;
-        }
-    }
-
-    let grammar_path = grammar_path
-        .map(PathBuf::from)
-        .unwrap_or_else(|| repo_path.join("grammar.js"));
-
-    // Read the grammar file.
-    let grammar_json = load_grammar_file(&grammar_path, js_runtime)?;
-
-    let src_path = repo_path.join("src");
-    let header_path = src_path.join("tree_sitter");
-
-    // Ensure that the output directories exist.
-    fs::create_dir_all(&src_path)?;
-    fs::create_dir_all(&header_path)?;
-
-    if grammar_path.file_name().unwrap() != "grammar.json" {
-        fs::write(src_path.join("grammar.json"), &grammar_json)
-            .with_context(|| format!("Failed to write grammar.json to {src_path:?}"))?;
-    }
-
-    // Parse and preprocess the grammar.
-    let input_grammar = parse_grammar(&grammar_json)?;
-
-    // Generate the parser and related files.
-    let GeneratedParser {
-        c_code,
-        node_types_json,
-    } = generate_parser_for_grammar_with_opts(&input_grammar, abi_version, report_symbol_name)?;
-
-    write_file(&src_path.join("parser.c"), c_code)?;
-    write_file(&src_path.join("node-types.json"), node_types_json)?;
-    write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
-    write_file(&header_path.join("array.h"), ARRAY_HEADER)?;
-    write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
-
-    Ok(())
-}
-
-pub fn generate_parser_for_grammar(grammar_json: &str) -> Result<(String, String)> {
-    let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n");
-    let input_grammar = parse_grammar(&grammar_json)?;
-    let parser =
-        generate_parser_for_grammar_with_opts(&input_grammar, tree_sitter::LANGUAGE_VERSION, None)?;
-    Ok((input_grammar.name, parser.c_code))
-}
-
-fn generate_parser_for_grammar_with_opts(
-    input_grammar: &InputGrammar,
-    abi_version: usize,
-    report_symbol_name: Option<&str>,
-) -> Result<GeneratedParser> {
-    let (syntax_grammar, lexical_grammar, inlines, simple_aliases) =
-        prepare_grammar(input_grammar)?;
-    let variable_info =
-        node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?;
-    let node_types_json = node_types::generate_node_types_json(
-        &syntax_grammar,
-        &lexical_grammar,
-        &simple_aliases,
-        &variable_info,
-    );
-    let tables = build_tables(
-        &syntax_grammar,
-        &lexical_grammar,
-        &simple_aliases,
-        &variable_info,
-        &inlines,
-        report_symbol_name,
-    )?;
-    let c_code = render_c_code(
-        &input_grammar.name,
-        tables,
-        syntax_grammar,
-        lexical_grammar,
-        simple_aliases,
-        abi_version,
-    );
-    Ok(GeneratedParser {
-        c_code,
-        node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(),
-    })
-}
-
-pub fn load_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result<String> {
-    if grammar_path.is_dir() {
-        return Err(anyhow!(
-            "Path to a grammar file with `.js` or `.json` extension is required"
-        ));
-    }
-    match grammar_path.extension().and_then(|e| e.to_str()) {
-        Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)
-            .with_context(|| "Failed to load grammar.js")?),
-        Some("json") => {
-            Ok(fs::read_to_string(grammar_path).with_context(|| "Failed to load grammar.json")?)
-        }
-        _ => Err(anyhow!("Unknown grammar file extension: {grammar_path:?}",)),
-    }
-}
-
-fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result<String> {
-    let grammar_path = fs::canonicalize(grammar_path)?;
-
-    #[cfg(windows)]
-    let grammar_path = url::Url::from_file_path(grammar_path)
-        .expect("Failed to convert path to URL")
-        .to_string();
-
-    let js_runtime = js_runtime.unwrap_or("node");
-
-    let mut js_command = Command::new(js_runtime);
-    match js_runtime {
-        "node" => {
-            js_command.args(["--input-type=module", "-"]);
-        }
-        "bun" => {
-            js_command.arg("-");
-        }
-        "deno" => {
-            js_command.args(["run", "--allow-all", "-"]);
-        }
-        _ => {}
-    }
-
-    let mut js_process = js_command
-        .env("TREE_SITTER_GRAMMAR_PATH", grammar_path)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()
-        .with_context(|| format!("Failed to run `{js_runtime}`"))?;
-
-    let mut js_stdin = js_process
-        .stdin
-        .take()
-        .with_context(|| format!("Failed to open stdin for {js_runtime}"))?;
-    let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))
-        .with_context(|| "Could not parse this package's version as semver.")?;
-    write!(
-        js_stdin,
-        "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {};
-         globalThis.TREE_SITTER_CLI_VERSION_MINOR = {};
-         globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
-        cli_version.major, cli_version.minor, cli_version.patch,
-    )
-    .with_context(|| format!("Failed to write tree-sitter version to {js_runtime}'s stdin"))?;
-    js_stdin
-        .write(include_bytes!("./dsl.js"))
-        .with_context(|| format!("Failed to write grammar dsl to {js_runtime}'s stdin"))?;
-    drop(js_stdin);
-
-    let output = js_process
-        .wait_with_output()
-        .with_context(|| format!("Failed to read output from {js_runtime}"))?;
-    match output.status.code() {
-        None => panic!("{js_runtime} process was killed"),
-        Some(0) => {
-            let stdout = String::from_utf8(output.stdout)
-                .with_context(|| format!("Got invalid UTF8 from {js_runtime}"))?;
-
-            let mut grammar_json = &stdout[..];
-
-            if let Some(pos) = stdout.rfind('\n') {
-                // If there's a newline, split the last line from the rest of the output
-                let node_output = &stdout[..pos];
-                grammar_json = &stdout[pos + 1..];
-
-                let mut stdout = std::io::stdout().lock();
-                stdout.write_all(node_output.as_bytes())?;
-                stdout.write_all(b"\n")?;
-                stdout.flush()?;
-            }
-
-            Ok(serde_json::to_string_pretty(
-                &serde_json::from_str::<serde_json::Value>(grammar_json)
-                    .with_context(|| "Failed to parse grammar JSON")?,
-            )
-            .with_context(|| "Failed to serialize grammar JSON")?
-                + "\n")
-        }
-        Some(code) => Err(anyhow!("{js_runtime} process exited with status {code}")),
-    }
-}
-
-pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> Result<()> {
-    fs::write(path, body)
-        .with_context(|| format!("Failed to write {:?}", path.file_name().unwrap()))
-}

cli/generate/src/parse_grammar.rs

@@ -1,343 +0,0 @@
-use std::collections::HashSet;
-
-use anyhow::{anyhow, Result};
-use serde::Deserialize;
-use serde_json::{Map, Value};
-
-use super::{
-    grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType},
-    rules::{Precedence, Rule},
-};
-
-#[derive(Deserialize)]
-#[serde(tag = "type")]
-#[allow(non_camel_case_types)]
-#[allow(clippy::upper_case_acronyms)]
-enum RuleJSON {
-    ALIAS {
-        content: Box<RuleJSON>,
-        named: bool,
-        value: String,
-    },
-    BLANK,
-    STRING {
-        value: String,
-    },
-    PATTERN {
-        value: String,
-        flags: Option<String>,
-    },
-    SYMBOL {
-        name: String,
-    },
-    CHOICE {
-        members: Vec<RuleJSON>,
-    },
-    FIELD {
-        name: String,
-        content: Box<RuleJSON>,
-    },
-    SEQ {
-        members: Vec<RuleJSON>,
-    },
-    REPEAT {
-        content: Box<RuleJSON>,
-    },
-    REPEAT1 {
-        content: Box<RuleJSON>,
-    },
-    PREC_DYNAMIC {
-        value: i32,
-        content: Box<RuleJSON>,
-    },
-    PREC_LEFT {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    PREC_RIGHT {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    PREC {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    TOKEN {
-        content: Box<RuleJSON>,
-    },
-    IMMEDIATE_TOKEN {
-        content: Box<RuleJSON>,
-    },
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-enum PrecedenceValueJSON {
-    Integer(i32),
-    Name(String),
-}
-
-#[derive(Deserialize)]
-pub struct GrammarJSON {
-    pub name: String,
-    rules: Map<String, Value>,
-    #[serde(default)]
-    precedences: Vec<Vec<RuleJSON>>,
-    #[serde(default)]
-    conflicts: Vec<Vec<String>>,
-    #[serde(default)]
-    externals: Vec<RuleJSON>,
-    #[serde(default)]
-    extras: Vec<RuleJSON>,
-    #[serde(default)]
-    inline: Vec<String>,
-    #[serde(default)]
-    supertypes: Vec<String>,
-    word: Option<String>,
-}
-
-fn rule_is_referenced(rule: &Rule, target: &str) -> bool {
-    match rule {
-        Rule::NamedSymbol(name) => name == target,
-        Rule::Choice(rules) | Rule::Seq(rules) => {
-            rules.iter().any(|r| rule_is_referenced(r, target))
-        }
-        Rule::Metadata { rule, .. } => rule_is_referenced(rule, target),
-        Rule::Repeat(inner) => rule_is_referenced(inner, target),
-        Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false,
-    }
-}
-
-fn variable_is_used(
-    grammar_rules: &[(String, Rule)],
-    other_rules: (&[Rule], &[Rule]),
-    target_name: &str,
-    in_progress: &mut HashSet<String>,
-) -> bool {
-    let root = &grammar_rules.first().unwrap().0;
-    if target_name == root {
-        return true;
-    }
-
-    if other_rules
-        .0
-        .iter()
-        .chain(other_rules.1.iter())
-        .any(|rule| rule_is_referenced(rule, target_name))
-    {
-        return true;
-    }
-
-    in_progress.insert(target_name.to_string());
-    let result = grammar_rules
-        .iter()
-        .filter(|(key, _)| *key != target_name)
-        .any(|(name, rule)| {
-            if !rule_is_referenced(rule, target_name) || in_progress.contains(name) {
-                return false;
-            }
-            variable_is_used(grammar_rules, other_rules, name, in_progress)
-        });
-    in_progress.remove(target_name);
-
-    result
-}
-
-pub(crate) fn parse_grammar(input: &str) -> Result<InputGrammar> {
-    let mut grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
-
-    let mut extra_symbols =
-        grammar_json
-            .extras
-            .into_iter()
-            .try_fold(Vec::new(), |mut acc, item| {
-                let rule = parse_rule(item);
-                if let Rule::String(ref value) = rule {
-                    if value.is_empty() {
-                        return Err(anyhow!(
-                            "Rules in the `extras` array must not contain empty strings"
-                        ));
-                    }
-                }
-                acc.push(rule);
-                Ok(acc)
-            })?;
-
-    let mut external_tokens = grammar_json
-        .externals
-        .into_iter()
-        .map(parse_rule)
-        .collect::<Vec<_>>();
-
-    let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len());
-    for list in grammar_json.precedences {
-        let mut ordering = Vec::with_capacity(list.len());
-        for entry in list {
-            ordering.push(match entry {
-                RuleJSON::STRING { value } => PrecedenceEntry::Name(value),
-                RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name),
-                _ => {
-                    return Err(anyhow!(
-                        "Invalid rule in precedences array. Only strings and symbols are allowed"
-                    ))
-                }
-            });
-        }
-        precedence_orderings.push(ordering);
-    }
-
-    let mut variables = Vec::with_capacity(grammar_json.rules.len());
-
-    let rules = grammar_json
-        .rules
-        .into_iter()
-        .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?))))
-        .collect::<Result<Vec<_>>>()?;
-
-    let mut in_progress = HashSet::new();
-
-    for (name, rule) in &rules {
-        if !variable_is_used(
-            &rules,
-            (&extra_symbols, &external_tokens),
-            name,
-            &mut in_progress,
-        ) && grammar_json.word.as_ref().map_or(true, |w| w != name)
-        {
-            grammar_json.conflicts.retain(|r| !r.contains(name));
-            grammar_json.supertypes.retain(|r| r != name);
-            grammar_json.inline.retain(|r| r != name);
-            extra_symbols.retain(|r| !rule_is_referenced(r, name));
-            external_tokens.retain(|r| !rule_is_referenced(r, name));
-            precedence_orderings.retain(|r| {
-                !r.iter().any(|e| {
-                    let PrecedenceEntry::Symbol(s) = e else {
-                        return false;
-                    };
-                    s == name
-                })
-            });
-            continue;
-        }
-        variables.push(Variable {
-            name: name.clone(),
-            kind: VariableType::Named,
-            rule: rule.clone(),
-        });
-    }
-
-    Ok(InputGrammar {
-        name: grammar_json.name,
-        word_token: grammar_json.word,
-        expected_conflicts: grammar_json.conflicts,
-        supertype_symbols: grammar_json.supertypes,
-        variables_to_inline: grammar_json.inline,
-        precedence_orderings,
-        variables,
-        extra_symbols,
-        external_tokens,
-    })
-}
-
-fn parse_rule(json: RuleJSON) -> Rule {
-    match json {
-        RuleJSON::ALIAS {
-            content,
-            value,
-            named,
-        } => Rule::alias(parse_rule(*content), value, named),
-        RuleJSON::BLANK => Rule::Blank,
-        RuleJSON::STRING { value } => Rule::String(value),
-        RuleJSON::PATTERN { value, flags } => Rule::Pattern(
-            value,
-            flags.map_or(String::new(), |f| {
-                f.matches(|c| {
-                    if c == 'i' {
-                        true
-                    } else {
-                        // silently ignore unicode flags
-                        if c != 'u' && c != 'v' {
-                            eprintln!("Warning: unsupported flag {c}");
-                        }
-                        false
-                    }
-                })
-                .collect()
-            }),
-        ),
-        RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name),
-        RuleJSON::CHOICE { members } => Rule::choice(members.into_iter().map(parse_rule).collect()),
-        RuleJSON::FIELD { content, name } => Rule::field(name, parse_rule(*content)),
-        RuleJSON::SEQ { members } => Rule::seq(members.into_iter().map(parse_rule).collect()),
-        RuleJSON::REPEAT1 { content } => Rule::repeat(parse_rule(*content)),
-        RuleJSON::REPEAT { content } => {
-            Rule::choice(vec![Rule::repeat(parse_rule(*content)), Rule::Blank])
-        }
-        RuleJSON::PREC { value, content } => Rule::prec(value.into(), parse_rule(*content)),
-        RuleJSON::PREC_LEFT { value, content } => {
-            Rule::prec_left(value.into(), parse_rule(*content))
-        }
-        RuleJSON::PREC_RIGHT { value, content } => {
-            Rule::prec_right(value.into(), parse_rule(*content))
-        }
-        RuleJSON::PREC_DYNAMIC { value, content } => {
-            Rule::prec_dynamic(value, parse_rule(*content))
-        }
-        RuleJSON::TOKEN { content } => Rule::token(parse_rule(*content)),
-        RuleJSON::IMMEDIATE_TOKEN { content } => Rule::immediate_token(parse_rule(*content)),
-    }
-}
-
-impl From<PrecedenceValueJSON> for Precedence {
-    fn from(val: PrecedenceValueJSON) -> Self {
-        match val {
-            PrecedenceValueJSON::Integer(i) => Self::Integer(i),
-            PrecedenceValueJSON::Name(i) => Self::Name(i),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_grammar() {
-        let grammar = parse_grammar(
-            r#"{
-            "name": "my_lang",
-            "rules": {
-                "file": {
-                    "type": "REPEAT1",
-                    "content": {
-                        "type": "SYMBOL",
-                        "name": "statement"
-                    }
-                },
-                "statement": {
-                    "type": "STRING",
-                    "value": "foo"
-                }
-            }
-        }"#,
-        )
-        .unwrap();
-
-        assert_eq!(grammar.name, "my_lang");
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable {
-                    name: "file".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::NamedSymbol("statement".to_string()))
-                },
-                Variable {
-                    name: "statement".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::String("foo".to_string())
-                },
-            ]
-        );
-    }
-}

cli/generate/src/prepare_grammar/expand_repeats.rs

@@ -1,289 +0,0 @@
-use std::{collections::HashMap, mem};
-
-use super::ExtractedSyntaxGrammar;
-use crate::{
-    grammars::{Variable, VariableType},
-    rules::{Rule, Symbol},
-};
-
-struct Expander {
-    variable_name: String,
-    repeat_count_in_variable: usize,
-    preceding_symbol_count: usize,
-    auxiliary_variables: Vec<Variable>,
-    existing_repeats: HashMap<Rule, Symbol>,
-}
-
-impl Expander {
-    fn expand_variable(&mut self, index: usize, variable: &mut Variable) -> bool {
-        self.variable_name.clear();
-        self.variable_name.push_str(&variable.name);
-        self.repeat_count_in_variable = 0;
-        let mut rule = Rule::Blank;
-        mem::swap(&mut rule, &mut variable.rule);
-
-        // In the special case of a hidden variable with a repetition at its top level,
-        // convert that rule itself into a binary tree structure instead of introducing
-        // another auxiliary rule.
-        if let (VariableType::Hidden, Rule::Repeat(repeated_content)) = (variable.kind, &rule) {
-            let inner_rule = self.expand_rule(repeated_content);
-            variable.rule = self.wrap_rule_in_binary_tree(Symbol::non_terminal(index), inner_rule);
-            variable.kind = VariableType::Auxiliary;
-            return true;
-        }
-
-        variable.rule = self.expand_rule(&rule);
-        false
-    }
-
-    fn expand_rule(&mut self, rule: &Rule) -> Rule {
-        match rule {
-            // For choices, sequences, and metadata, descend into the child rules,
-            // replacing any nested repetitions.
-            Rule::Choice(elements) => Rule::Choice(
-                elements
-                    .iter()
-                    .map(|element| self.expand_rule(element))
-                    .collect(),
-            ),
-
-            Rule::Seq(elements) => Rule::Seq(
-                elements
-                    .iter()
-                    .map(|element| self.expand_rule(element))
-                    .collect(),
-            ),
-
-            Rule::Metadata { rule, params } => Rule::Metadata {
-                rule: Box::new(self.expand_rule(rule)),
-                params: params.clone(),
-            },
-
-            // For repetitions, introduce an auxiliary rule that contains the
-            // repeated content, but can also contain a recursive binary tree structure.
-            Rule::Repeat(content) => {
-                let inner_rule = self.expand_rule(content);
-
-                if let Some(existing_symbol) = self.existing_repeats.get(&inner_rule) {
-                    return Rule::Symbol(*existing_symbol);
-                }
-
-                self.repeat_count_in_variable += 1;
-                let rule_name = format!(
-                    "{}_repeat{}",
-                    self.variable_name, self.repeat_count_in_variable
-                );
-                let repeat_symbol = Symbol::non_terminal(
-                    self.preceding_symbol_count + self.auxiliary_variables.len(),
-                );
-                self.existing_repeats
-                    .insert(inner_rule.clone(), repeat_symbol);
-                self.auxiliary_variables.push(Variable {
-                    name: rule_name,
-                    kind: VariableType::Auxiliary,
-                    rule: self.wrap_rule_in_binary_tree(repeat_symbol, inner_rule),
-                });
-
-                Rule::Symbol(repeat_symbol)
-            }
-
-            // For primitive rules, don't change anything.
-            _ => rule.clone(),
-        }
-    }
-
-    fn wrap_rule_in_binary_tree(&self, symbol: Symbol, rule: Rule) -> Rule {
-        Rule::choice(vec![
-            Rule::Seq(vec![Rule::Symbol(symbol), Rule::Symbol(symbol)]),
-            rule,
-        ])
-    }
-}
-
-pub(super) fn expand_repeats(mut grammar: ExtractedSyntaxGrammar) -> ExtractedSyntaxGrammar {
-    let mut expander = Expander {
-        variable_name: String::new(),
-        repeat_count_in_variable: 0,
-        preceding_symbol_count: grammar.variables.len(),
-        auxiliary_variables: Vec::new(),
-        existing_repeats: HashMap::new(),
-    };
-
-    for (i, variable) in grammar.variables.iter_mut().enumerate() {
-        let expanded_top_level_repetition = expander.expand_variable(i, variable);
-
-        // If a hidden variable had a top-level repetition and it was converted to
-        // a recursive rule, then it can't be inlined.
-        if expanded_top_level_repetition {
-            grammar
-                .variables_to_inline
-                .retain(|symbol| *symbol != Symbol::non_terminal(i));
-        }
-    }
-
-    grammar.variables.extend(expander.auxiliary_variables);
-    grammar
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_basic_repeat_expansion() {
-        // Repeats nested inside of sequences and choices are expanded.
-        let grammar = expand_repeats(build_grammar(vec![Variable::named(
-            "rule0",
-            Rule::seq(vec![
-                Rule::terminal(10),
-                Rule::choice(vec![
-                    Rule::repeat(Rule::terminal(11)),
-                    Rule::repeat(Rule::terminal(12)),
-                ]),
-                Rule::terminal(13),
-            ]),
-        )]));
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "rule0",
-                    Rule::seq(vec![
-                        Rule::terminal(10),
-                        Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),]),
-                        Rule::terminal(13),
-                    ])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
-                        Rule::terminal(11),
-                    ])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat2",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
-                        Rule::terminal(12),
-                    ])
-                ),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_repeat_deduplication() {
-        // Terminal 4 appears inside of a repeat in three different places.
-        let grammar = expand_repeats(build_grammar(vec![
-            Variable::named(
-                "rule0",
-                Rule::choice(vec![
-                    Rule::seq(vec![Rule::terminal(1), Rule::repeat(Rule::terminal(4))]),
-                    Rule::seq(vec![Rule::terminal(2), Rule::repeat(Rule::terminal(4))]),
-                ]),
-            ),
-            Variable::named(
-                "rule1",
-                Rule::seq(vec![Rule::terminal(3), Rule::repeat(Rule::terminal(4))]),
-            ),
-        ]));
-
-        // Only one auxiliary rule is created for repeating terminal 4.
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "rule0",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::terminal(1), Rule::non_terminal(2)]),
-                        Rule::seq(vec![Rule::terminal(2), Rule::non_terminal(2)]),
-                    ])
-                ),
-                Variable::named(
-                    "rule1",
-                    Rule::seq(vec![Rule::terminal(3), Rule::non_terminal(2),])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
-                        Rule::terminal(4),
-                    ])
-                )
-            ]
-        );
-    }
-
-    #[test]
-    fn test_expansion_of_nested_repeats() {
-        let grammar = expand_repeats(build_grammar(vec![Variable::named(
-            "rule0",
-            Rule::seq(vec![
-                Rule::terminal(10),
-                Rule::repeat(Rule::seq(vec![
-                    Rule::terminal(11),
-                    Rule::repeat(Rule::terminal(12)),
-                ])),
-            ]),
-        )]));
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "rule0",
-                    Rule::seq(vec![Rule::terminal(10), Rule::non_terminal(2),])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1),]),
-                        Rule::terminal(12),
-                    ])
-                ),
-                Variable::auxiliary(
-                    "rule0_repeat2",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2),]),
-                        Rule::seq(vec![Rule::terminal(11), Rule::non_terminal(1),]),
-                    ])
-                ),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_expansion_of_repeats_at_top_of_hidden_rules() {
-        let grammar = expand_repeats(build_grammar(vec![
-            Variable::named("rule0", Rule::non_terminal(1)),
-            Variable::hidden(
-                "_rule1",
-                Rule::repeat(Rule::choice(vec![Rule::terminal(11), Rule::terminal(12)])),
-            ),
-        ]));
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named("rule0", Rule::non_terminal(1),),
-                Variable::auxiliary(
-                    "_rule1",
-                    Rule::choice(vec![
-                        Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(1)]),
-                        Rule::terminal(11),
-                        Rule::terminal(12),
-                    ]),
-                ),
-            ]
-        );
-    }
-
-    fn build_grammar(variables: Vec<Variable>) -> ExtractedSyntaxGrammar {
-        ExtractedSyntaxGrammar {
-            variables,
-            ..Default::default()
-        }
-    }
-}

cli/generate/src/prepare_grammar/expand_tokens.rs

@@ -1,940 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::{anyhow, Context, Result};
-use lazy_static::lazy_static;
-use regex_syntax::ast::{
-    parse, Ast, ClassPerlKind, ClassSet, ClassSetBinaryOpKind, ClassSetItem, ClassUnicodeKind,
-    RepetitionKind, RepetitionRange,
-};
-
-use super::ExtractedLexicalGrammar;
-use crate::{
-    grammars::{LexicalGrammar, LexicalVariable},
-    nfa::{CharacterSet, Nfa, NfaState},
-    rules::{Precedence, Rule},
-};
-
-lazy_static! {
-    static ref UNICODE_CATEGORIES: HashMap<&'static str, Vec<u32>> =
-        serde_json::from_str(UNICODE_CATEGORIES_JSON).unwrap();
-    static ref UNICODE_PROPERTIES: HashMap<&'static str, Vec<u32>> =
-        serde_json::from_str(UNICODE_PROPERTIES_JSON).unwrap();
-    static ref UNICODE_CATEGORY_ALIASES: HashMap<&'static str, String> =
-        serde_json::from_str(UNICODE_CATEGORY_ALIASES_JSON).unwrap();
-    static ref UNICODE_PROPERTY_ALIASES: HashMap<&'static str, String> =
-        serde_json::from_str(UNICODE_PROPERTY_ALIASES_JSON).unwrap();
-}
-
-const UNICODE_CATEGORIES_JSON: &str = include_str!("./unicode-categories.json");
-const UNICODE_PROPERTIES_JSON: &str = include_str!("./unicode-properties.json");
-const UNICODE_CATEGORY_ALIASES_JSON: &str = include_str!("./unicode-category-aliases.json");
-const UNICODE_PROPERTY_ALIASES_JSON: &str = include_str!("./unicode-property-aliases.json");
-
-struct NfaBuilder {
-    nfa: Nfa,
-    is_sep: bool,
-    precedence_stack: Vec<i32>,
-}
-
-fn get_implicit_precedence(rule: &Rule) -> i32 {
-    match rule {
-        Rule::String(_) => 2,
-        Rule::Metadata { rule, params } => {
-            if params.is_main_token {
-                get_implicit_precedence(rule) + 1
-            } else {
-                get_implicit_precedence(rule)
-            }
-        }
-        _ => 0,
-    }
-}
-
-const fn get_completion_precedence(rule: &Rule) -> i32 {
-    if let Rule::Metadata { params, .. } = rule {
-        if let Precedence::Integer(p) = params.precedence {
-            return p;
-        }
-    }
-    0
-}
-
-pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> Result<LexicalGrammar> {
-    let mut builder = NfaBuilder {
-        nfa: Nfa::new(),
-        is_sep: true,
-        precedence_stack: vec![0],
-    };
-
-    let separator_rule = if grammar.separators.is_empty() {
-        Rule::Blank
-    } else {
-        grammar.separators.push(Rule::Blank);
-        Rule::repeat(Rule::choice(grammar.separators))
-    };
-
-    let mut variables = Vec::new();
-    for (i, variable) in grammar.variables.into_iter().enumerate() {
-        let is_immediate_token = match &variable.rule {
-            Rule::Metadata { params, .. } => params.is_main_token,
-            _ => false,
-        };
-
-        builder.is_sep = false;
-        builder.nfa.states.push(NfaState::Accept {
-            variable_index: i,
-            precedence: get_completion_precedence(&variable.rule),
-        });
-        let last_state_id = builder.nfa.last_state_id();
-        builder
-            .expand_rule(&variable.rule, last_state_id)
-            .with_context(|| format!("Error processing rule {}", variable.name))?;
-
-        if !is_immediate_token {
-            builder.is_sep = true;
-            let last_state_id = builder.nfa.last_state_id();
-            builder.expand_rule(&separator_rule, last_state_id)?;
-        }
-
-        variables.push(LexicalVariable {
-            name: variable.name,
-            kind: variable.kind,
-            implicit_precedence: get_implicit_precedence(&variable.rule),
-            start_state: builder.nfa.last_state_id(),
-        });
-    }
-
-    Ok(LexicalGrammar {
-        nfa: builder.nfa,
-        variables,
-    })
-}
-
-impl NfaBuilder {
-    fn expand_rule(&mut self, rule: &Rule, mut next_state_id: u32) -> Result<bool> {
-        match rule {
-            Rule::Pattern(s, f) => {
-                let ast = parse::Parser::new().parse(s)?;
-                self.expand_regex(&ast, next_state_id, f.contains('i'))
-            }
-            Rule::String(s) => {
-                for c in s.chars().rev() {
-                    self.push_advance(CharacterSet::empty().add_char(c), next_state_id);
-                    next_state_id = self.nfa.last_state_id();
-                }
-                Ok(!s.is_empty())
-            }
-            Rule::Choice(elements) => {
-                let mut alternative_state_ids = Vec::new();
-                for element in elements {
-                    if self.expand_rule(element, next_state_id)? {
-                        alternative_state_ids.push(self.nfa.last_state_id());
-                    } else {
-                        alternative_state_ids.push(next_state_id);
-                    }
-                }
-                alternative_state_ids.sort_unstable();
-                alternative_state_ids.dedup();
-                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
-                for alternative_state_id in alternative_state_ids {
-                    self.push_split(alternative_state_id);
-                }
-                Ok(true)
-            }
-            Rule::Seq(elements) => {
-                let mut result = false;
-                for element in elements.iter().rev() {
-                    if self.expand_rule(element, next_state_id)? {
-                        result = true;
-                    }
-                    next_state_id = self.nfa.last_state_id();
-                }
-                Ok(result)
-            }
-            Rule::Repeat(rule) => {
-                self.nfa.states.push(NfaState::Accept {
-                    variable_index: 0,
-                    precedence: 0,
-                }); // Placeholder for split
-                let split_state_id = self.nfa.last_state_id();
-                if self.expand_rule(rule, split_state_id)? {
-                    self.nfa.states[split_state_id as usize] =
-                        NfaState::Split(self.nfa.last_state_id(), next_state_id);
-                    Ok(true)
-                } else {
-                    Ok(false)
-                }
-            }
-            Rule::Metadata { rule, params } => {
-                let has_precedence = if let Precedence::Integer(precedence) = &params.precedence {
-                    self.precedence_stack.push(*precedence);
-                    true
-                } else {
-                    false
-                };
-                let result = self.expand_rule(rule, next_state_id);
-                if has_precedence {
-                    self.precedence_stack.pop();
-                }
-                result
-            }
-            Rule::Blank => Ok(false),
-            _ => Err(anyhow!("Grammar error: Unexpected rule {rule:?}")),
-        }
-    }
-
-    fn expand_regex(
-        &mut self,
-        ast: &Ast,
-        mut next_state_id: u32,
-        case_insensitive: bool,
-    ) -> Result<bool> {
-        const fn inverse_char(c: char) -> char {
-            match c {
-                'a'..='z' => (c as u8 - b'a' + b'A') as char,
-                'A'..='Z' => (c as u8 - b'A' + b'a') as char,
-                c => c,
-            }
-        }
-
-        fn with_inverse_char(mut chars: CharacterSet) -> CharacterSet {
-            for char in chars.clone().chars() {
-                let inverted = inverse_char(char);
-                if char != inverted {
-                    chars = chars.add_char(inverted);
-                }
-            }
-            chars
-        }
-
-        match ast {
-            Ast::Empty(_) => Ok(false),
-            Ast::Flags(_) => Err(anyhow!("Regex error: Flags are not supported")),
-            Ast::Literal(literal) => {
-                let mut char_set = CharacterSet::from_char(literal.c);
-                if case_insensitive {
-                    let inverted = inverse_char(literal.c);
-                    if literal.c != inverted {
-                        char_set = char_set.add_char(inverted);
-                    }
-                }
-                self.push_advance(char_set, next_state_id);
-                Ok(true)
-            }
-            Ast::Dot(_) => {
-                self.push_advance(CharacterSet::from_char('\n').negate(), next_state_id);
-                Ok(true)
-            }
-            Ast::Assertion(_) => Err(anyhow!("Regex error: Assertions are not supported")),
-            Ast::ClassUnicode(class) => {
-                let mut chars = self.expand_unicode_character_class(&class.kind)?;
-                if class.negated {
-                    chars = chars.negate();
-                }
-                if case_insensitive {
-                    chars = with_inverse_char(chars);
-                }
-                self.push_advance(chars, next_state_id);
-                Ok(true)
-            }
-            Ast::ClassPerl(class) => {
-                let mut chars = self.expand_perl_character_class(&class.kind);
-                if class.negated {
-                    chars = chars.negate();
-                }
-                if case_insensitive {
-                    chars = with_inverse_char(chars);
-                }
-                self.push_advance(chars, next_state_id);
-                Ok(true)
-            }
-            Ast::ClassBracketed(class) => {
-                let mut chars = self.translate_class_set(&class.kind)?;
-                if class.negated {
-                    chars = chars.negate();
-                }
-                if case_insensitive {
-                    chars = with_inverse_char(chars);
-                }
-                self.push_advance(chars, next_state_id);
-                Ok(true)
-            }
-            Ast::Repetition(repetition) => match repetition.op.kind {
-                RepetitionKind::ZeroOrOne => {
-                    self.expand_zero_or_one(&repetition.ast, next_state_id, case_insensitive)
-                }
-                RepetitionKind::OneOrMore => {
-                    self.expand_one_or_more(&repetition.ast, next_state_id, case_insensitive)
-                }
-                RepetitionKind::ZeroOrMore => {
-                    self.expand_zero_or_more(&repetition.ast, next_state_id, case_insensitive)
-                }
-                RepetitionKind::Range(RepetitionRange::Exactly(count)) => {
-                    self.expand_count(&repetition.ast, count, next_state_id, case_insensitive)
-                }
-                RepetitionKind::Range(RepetitionRange::AtLeast(min)) => {
-                    if self.expand_zero_or_more(&repetition.ast, next_state_id, case_insensitive)? {
-                        self.expand_count(&repetition.ast, min, next_state_id, case_insensitive)
-                    } else {
-                        Ok(false)
-                    }
-                }
-                RepetitionKind::Range(RepetitionRange::Bounded(min, max)) => {
-                    let mut result =
-                        self.expand_count(&repetition.ast, min, next_state_id, case_insensitive)?;
-                    for _ in min..max {
-                        if result {
-                            next_state_id = self.nfa.last_state_id();
-                        }
-                        if self.expand_zero_or_one(
-                            &repetition.ast,
-                            next_state_id,
-                            case_insensitive,
-                        )? {
-                            result = true;
-                        }
-                    }
-                    Ok(result)
-                }
-            },
-            Ast::Group(group) => self.expand_regex(&group.ast, next_state_id, case_insensitive),
-            Ast::Alternation(alternation) => {
-                let mut alternative_state_ids = Vec::new();
-                for ast in &alternation.asts {
-                    if self.expand_regex(ast, next_state_id, case_insensitive)? {
-                        alternative_state_ids.push(self.nfa.last_state_id());
-                    } else {
-                        alternative_state_ids.push(next_state_id);
-                    }
-                }
-                alternative_state_ids.sort_unstable();
-                alternative_state_ids.dedup();
-                alternative_state_ids.retain(|i| *i != self.nfa.last_state_id());
-
-                for alternative_state_id in alternative_state_ids {
-                    self.push_split(alternative_state_id);
-                }
-                Ok(true)
-            }
-            Ast::Concat(concat) => {
-                let mut result = false;
-                for ast in concat.asts.iter().rev() {
-                    if self.expand_regex(ast, next_state_id, case_insensitive)? {
-                        result = true;
-                        next_state_id = self.nfa.last_state_id();
-                    }
-                }
-                Ok(result)
-            }
-        }
-    }
-
-    fn translate_class_set(&self, class_set: &ClassSet) -> Result<CharacterSet> {
-        match &class_set {
-            ClassSet::Item(item) => self.expand_character_class(item),
-            ClassSet::BinaryOp(binary_op) => {
-                let mut lhs_char_class = self.translate_class_set(&binary_op.lhs)?;
-                let mut rhs_char_class = self.translate_class_set(&binary_op.rhs)?;
-                match binary_op.kind {
-                    ClassSetBinaryOpKind::Intersection => {
-                        Ok(lhs_char_class.remove_intersection(&mut rhs_char_class))
-                    }
-                    ClassSetBinaryOpKind::Difference => {
-                        Ok(lhs_char_class.difference(rhs_char_class))
-                    }
-                    ClassSetBinaryOpKind::SymmetricDifference => {
-                        Ok(lhs_char_class.symmetric_difference(rhs_char_class))
-                    }
-                }
-            }
-        }
-    }
-
-    fn expand_one_or_more(
-        &mut self,
-        ast: &Ast,
-        next_state_id: u32,
-        case_insensitive: bool,
-    ) -> Result<bool> {
-        self.nfa.states.push(NfaState::Accept {
-            variable_index: 0,
-            precedence: 0,
-        }); // Placeholder for split
-        let split_state_id = self.nfa.last_state_id();
-        if self.expand_regex(ast, split_state_id, case_insensitive)? {
-            self.nfa.states[split_state_id as usize] =
-                NfaState::Split(self.nfa.last_state_id(), next_state_id);
-            Ok(true)
-        } else {
-            self.nfa.states.pop();
-            Ok(false)
-        }
-    }
-
-    fn expand_zero_or_one(
-        &mut self,
-        ast: &Ast,
-        next_state_id: u32,
-        case_insensitive: bool,
-    ) -> Result<bool> {
-        if self.expand_regex(ast, next_state_id, case_insensitive)? {
-            self.push_split(next_state_id);
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    fn expand_zero_or_more(
-        &mut self,
-        ast: &Ast,
-        next_state_id: u32,
-        case_insensitive: bool,
-    ) -> Result<bool> {
-        if self.expand_one_or_more(ast, next_state_id, case_insensitive)? {
-            self.push_split(next_state_id);
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    fn expand_count(
-        &mut self,
-        ast: &Ast,
-        count: u32,
-        mut next_state_id: u32,
-        case_insensitive: bool,
-    ) -> Result<bool> {
-        let mut result = false;
-        for _ in 0..count {
-            if self.expand_regex(ast, next_state_id, case_insensitive)? {
-                result = true;
-                next_state_id = self.nfa.last_state_id();
-            }
-        }
-        Ok(result)
-    }
-
-    fn expand_character_class(&self, item: &ClassSetItem) -> Result<CharacterSet> {
-        match item {
-            ClassSetItem::Empty(_) => Ok(CharacterSet::empty()),
-            ClassSetItem::Literal(literal) => Ok(CharacterSet::from_char(literal.c)),
-            ClassSetItem::Range(range) => Ok(CharacterSet::from_range(range.start.c, range.end.c)),
-            ClassSetItem::Union(union) => {
-                let mut result = CharacterSet::empty();
-                for item in &union.items {
-                    result = result.add(&self.expand_character_class(item)?);
-                }
-                Ok(result)
-            }
-            ClassSetItem::Perl(class) => Ok(self.expand_perl_character_class(&class.kind)),
-            ClassSetItem::Unicode(class) => {
-                let mut set = self.expand_unicode_character_class(&class.kind)?;
-                if class.negated {
-                    set = set.negate();
-                }
-                Ok(set)
-            }
-            ClassSetItem::Bracketed(class) => {
-                let mut set = self.translate_class_set(&class.kind)?;
-                if class.negated {
-                    set = set.negate();
-                }
-                Ok(set)
-            }
-            ClassSetItem::Ascii(_) => Err(anyhow!(
-                "Regex error: Unsupported character class syntax {item:?}",
-            )),
-        }
-    }
-
-    fn expand_unicode_character_class(&self, class: &ClassUnicodeKind) -> Result<CharacterSet> {
-        let mut chars = CharacterSet::empty();
-
-        let category_letter;
-        match class {
-            ClassUnicodeKind::OneLetter(le) => {
-                category_letter = le.to_string();
-            }
-            ClassUnicodeKind::Named(class_name) => {
-                let actual_class_name = UNICODE_CATEGORY_ALIASES
-                    .get(class_name.as_str())
-                    .or_else(|| UNICODE_PROPERTY_ALIASES.get(class_name.as_str()))
-                    .unwrap_or(class_name);
-                if actual_class_name.len() == 1 {
-                    category_letter = actual_class_name.clone();
-                } else {
-                    let code_points =
-                        UNICODE_CATEGORIES
-                            .get(actual_class_name.as_str())
-                            .or_else(|| UNICODE_PROPERTIES.get(actual_class_name.as_str()))
-                            .ok_or_else(|| {
-                                anyhow!(
-                                    "Regex error: Unsupported unicode character class {class_name}",
-                                )
-                            })?;
-                    for c in code_points {
-                        if let Some(c) = char::from_u32(*c) {
-                            chars = chars.add_char(c);
-                        }
-                    }
-
-                    return Ok(chars);
-                }
-            }
-            ClassUnicodeKind::NamedValue { .. } => {
-                return Err(anyhow!(
-                    "Regex error: Key-value unicode properties are not supported"
-                ))
-            }
-        }
-
-        for (category, code_points) in UNICODE_CATEGORIES.iter() {
-            if category.starts_with(&category_letter) {
-                for c in code_points {
-                    if let Some(c) = char::from_u32(*c) {
-                        chars = chars.add_char(c);
-                    }
-                }
-            }
-        }
-
-        Ok(chars)
-    }
-
-    fn expand_perl_character_class(&self, item: &ClassPerlKind) -> CharacterSet {
-        match item {
-            ClassPerlKind::Digit => CharacterSet::from_range('0', '9'),
-            ClassPerlKind::Space => CharacterSet::empty()
-                .add_char(' ')
-                .add_char('\t')
-                .add_char('\r')
-                .add_char('\n')
-                .add_char('\x0B')
-                .add_char('\x0C'),
-            ClassPerlKind::Word => CharacterSet::empty()
-                .add_char('_')
-                .add_range('A', 'Z')
-                .add_range('a', 'z')
-                .add_range('0', '9'),
-        }
-    }
-
-    fn push_advance(&mut self, chars: CharacterSet, state_id: u32) {
-        let precedence = *self.precedence_stack.last().unwrap();
-        self.nfa.states.push(NfaState::Advance {
-            chars,
-            state_id,
-            precedence,
-            is_sep: self.is_sep,
-        });
-    }
-
-    fn push_split(&mut self, state_id: u32) {
-        let last_state_id = self.nfa.last_state_id();
-        self.nfa
-            .states
-            .push(NfaState::Split(state_id, last_state_id));
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::Variable,
-        nfa::{NfaCursor, NfaTransition},
-    };
-
-    fn simulate_nfa<'a>(grammar: &'a LexicalGrammar, s: &'a str) -> Option<(usize, &'a str)> {
-        let start_states = grammar.variables.iter().map(|v| v.start_state).collect();
-        let mut cursor = NfaCursor::new(&grammar.nfa, start_states);
-
-        let mut result = None;
-        let mut result_precedence = i32::MIN;
-        let mut start_char = 0;
-        let mut end_char = 0;
-        for c in s.chars() {
-            for (id, precedence) in cursor.completions() {
-                if result.is_none() || result_precedence <= precedence {
-                    result = Some((id, &s[start_char..end_char]));
-                    result_precedence = precedence;
-                }
-            }
-            if let Some(NfaTransition {
-                states,
-                is_separator,
-                ..
-            }) = cursor
-                .transitions()
-                .into_iter()
-                .find(|t| t.characters.contains(c) && t.precedence >= result_precedence)
-            {
-                cursor.reset(states);
-                end_char += c.len_utf8();
-                if is_separator {
-                    start_char = end_char;
-                }
-            } else {
-                break;
-            }
-        }
-
-        for (id, precedence) in cursor.completions() {
-            if result.is_none() || result_precedence <= precedence {
-                result = Some((id, &s[start_char..end_char]));
-                result_precedence = precedence;
-            }
-        }
-
-        result
-    }
-
-    #[test]
-    fn test_rule_expansion() {
-        struct Row {
-            rules: Vec<Rule>,
-            separators: Vec<Rule>,
-            examples: Vec<(&'static str, Option<(usize, &'static str)>)>,
-        }
-
-        let table = [
-            // regex with sequences and alternatives
-            Row {
-                rules: vec![Rule::pattern("(a|b|c)d(e|f|g)h?", "")],
-                separators: vec![],
-                examples: vec![
-                    ("ade1", Some((0, "ade"))),
-                    ("bdf1", Some((0, "bdf"))),
-                    ("bdfh1", Some((0, "bdfh"))),
-                    ("ad1", None),
-                ],
-            },
-            // regex with repeats
-            Row {
-                rules: vec![Rule::pattern("a*", "")],
-                separators: vec![],
-                examples: vec![("aaa1", Some((0, "aaa"))), ("b", Some((0, "")))],
-            },
-            // regex with repeats in sequences
-            Row {
-                rules: vec![Rule::pattern("a((bc)+|(de)*)f", "")],
-                separators: vec![],
-                examples: vec![
-                    ("af1", Some((0, "af"))),
-                    ("adedef1", Some((0, "adedef"))),
-                    ("abcbcbcf1", Some((0, "abcbcbcf"))),
-                    ("a", None),
-                ],
-            },
-            // regex with character ranges
-            Row {
-                rules: vec![Rule::pattern("[a-fA-F0-9]+", "")],
-                separators: vec![],
-                examples: vec![("A1ff0.", Some((0, "A1ff0")))],
-            },
-            // regex with perl character classes
-            Row {
-                rules: vec![Rule::pattern("\\w\\d\\s", "")],
-                separators: vec![],
-                examples: vec![("_0  ", Some((0, "_0 ")))],
-            },
-            // string
-            Row {
-                rules: vec![Rule::string("abc")],
-                separators: vec![],
-                examples: vec![("abcd", Some((0, "abc"))), ("ab", None)],
-            },
-            // complex rule containing strings and regexes
-            Row {
-                rules: vec![Rule::repeat(Rule::seq(vec![
-                    Rule::string("{"),
-                    Rule::pattern("[a-f]+", ""),
-                    Rule::string("}"),
-                ]))],
-                separators: vec![],
-                examples: vec![
-                    ("{a}{", Some((0, "{a}"))),
-                    ("{a}{d", Some((0, "{a}"))),
-                    ("ab", None),
-                ],
-            },
-            // longest match rule
-            Row {
-                rules: vec![
-                    Rule::pattern("a|bc", ""),
-                    Rule::pattern("aa", ""),
-                    Rule::pattern("bcd", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("a.", Some((0, "a"))),
-                    ("bc.", Some((0, "bc"))),
-                    ("aa.", Some((1, "aa"))),
-                    ("bcd?", Some((2, "bcd"))),
-                    ("b.", None),
-                    ("c.", None),
-                ],
-            },
-            // regex with an alternative including the empty string
-            Row {
-                rules: vec![Rule::pattern("a(b|)+c", "")],
-                separators: vec![],
-                examples: vec![
-                    ("ac.", Some((0, "ac"))),
-                    ("abc.", Some((0, "abc"))),
-                    ("abbc.", Some((0, "abbc"))),
-                ],
-            },
-            // separators
-            Row {
-                rules: vec![Rule::pattern("[a-f]+", "")],
-                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
-                examples: vec![
-                    ("  a", Some((0, "a"))),
-                    ("  \nb", Some((0, "b"))),
-                    ("  \\a", None),
-                    ("  \\\na", Some((0, "a"))),
-                ],
-            },
-            // shorter tokens with higher precedence
-            Row {
-                rules: vec![
-                    Rule::prec(Precedence::Integer(2), Rule::pattern("abc", "")),
-                    Rule::prec(Precedence::Integer(1), Rule::pattern("ab[cd]e", "")),
-                    Rule::pattern("[a-e]+", ""),
-                ],
-                separators: vec![Rule::string("\\\n"), Rule::pattern("\\s", "")],
-                examples: vec![
-                    ("abceef", Some((0, "abc"))),
-                    ("abdeef", Some((1, "abde"))),
-                    ("aeeeef", Some((2, "aeeee"))),
-                ],
-            },
-            // immediate tokens with higher precedence
-            Row {
-                rules: vec![
-                    Rule::prec(Precedence::Integer(1), Rule::pattern("[^a]+", "")),
-                    Rule::immediate_token(Rule::prec(
-                        Precedence::Integer(2),
-                        Rule::pattern("[^ab]+", ""),
-                    )),
-                ],
-                separators: vec![Rule::pattern("\\s", "")],
-                examples: vec![("cccb", Some((1, "ccc")))],
-            },
-            Row {
-                rules: vec![Rule::seq(vec![
-                    Rule::string("a"),
-                    Rule::choice(vec![Rule::string("b"), Rule::string("c")]),
-                    Rule::string("d"),
-                ])],
-                separators: vec![],
-                examples: vec![
-                    ("abd", Some((0, "abd"))),
-                    ("acd", Some((0, "acd"))),
-                    ("abc", None),
-                    ("ad", None),
-                    ("d", None),
-                    ("a", None),
-                ],
-            },
-            // nested choices within sequences
-            Row {
-                rules: vec![Rule::seq(vec![
-                    Rule::pattern("[0-9]+", ""),
-                    Rule::choice(vec![
-                        Rule::Blank,
-                        Rule::choice(vec![Rule::seq(vec![
-                            Rule::choice(vec![Rule::string("e"), Rule::string("E")]),
-                            Rule::choice(vec![
-                                Rule::Blank,
-                                Rule::choice(vec![Rule::string("+"), Rule::string("-")]),
-                            ]),
-                            Rule::pattern("[0-9]+", ""),
-                        ])]),
-                    ]),
-                ])],
-                separators: vec![],
-                examples: vec![
-                    ("12", Some((0, "12"))),
-                    ("12e", Some((0, "12"))),
-                    ("12g", Some((0, "12"))),
-                    ("12e3", Some((0, "12e3"))),
-                    ("12e+", Some((0, "12"))),
-                    ("12E+34 +", Some((0, "12E+34"))),
-                    ("12e34", Some((0, "12e34"))),
-                ],
-            },
-            // nested groups
-            Row {
-                rules: vec![Rule::seq(vec![Rule::pattern(r"([^x\\]|\\(.|\n))+", "")])],
-                separators: vec![],
-                examples: vec![("abcx", Some((0, "abc"))), ("abc\\0x", Some((0, "abc\\0")))],
-            },
-            // allowing unrecognized escape sequences
-            Row {
-                rules: vec![
-                    // Escaped forward slash (used in JS because '/' is the regex delimiter)
-                    Rule::pattern(r"\/", ""),
-                    // Escaped quotes
-                    Rule::pattern(r#"\"\'"#, ""),
-                    // Quote preceded by a literal backslash
-                    Rule::pattern(r"[\\']+", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("/", Some((0, "/"))),
-                    ("\"\'", Some((1, "\"\'"))),
-                    (r"'\'a", Some((2, r"'\'"))),
-                ],
-            },
-            // unicode property escapes
-            Row {
-                rules: vec![
-                    Rule::pattern(r"\p{L}+\P{L}+", ""),
-                    Rule::pattern(r"\p{White_Space}+\P{White_Space}+[\p{White_Space}]*", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("  123   abc", Some((1, "  123   "))),
-                    ("ბΨƁ___ƀƔ", Some((0, "ბΨƁ___"))),
-                ],
-            },
-            // unicode property escapes in bracketed sets
-            Row {
-                rules: vec![Rule::pattern(r"[\p{L}\p{Nd}]+", "")],
-                separators: vec![],
-                examples: vec![("abΨ12٣٣, ok", Some((0, "abΨ12٣٣")))],
-            },
-            // unicode character escapes
-            Row {
-                rules: vec![
-                    Rule::pattern(r"\u{00dc}", ""),
-                    Rule::pattern(r"\U{000000dd}", ""),
-                    Rule::pattern(r"\u00de", ""),
-                    Rule::pattern(r"\U000000df", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("\u{00dc}", Some((0, "\u{00dc}"))),
-                    ("\u{00dd}", Some((1, "\u{00dd}"))),
-                    ("\u{00de}", Some((2, "\u{00de}"))),
-                    ("\u{00df}", Some((3, "\u{00df}"))),
-                ],
-            },
-            Row {
-                rules: vec![
-                    Rule::pattern(r"u\{[0-9a-fA-F]+\}", ""),
-                    // Already-escaped curly braces
-                    Rule::pattern(r"\{[ab]{3}\}", ""),
-                    // Unicode codepoints
-                    Rule::pattern(r"\u{1000A}", ""),
-                    // Unicode codepoints (lowercase)
-                    Rule::pattern(r"\u{1000b}", ""),
-                ],
-                separators: vec![],
-                examples: vec![
-                    ("u{1234} ok", Some((0, "u{1234}"))),
-                    ("{aba}}", Some((1, "{aba}"))),
-                    ("\u{1000A}", Some((2, "\u{1000A}"))),
-                    ("\u{1000b}", Some((3, "\u{1000b}"))),
-                ],
-            },
-            // Emojis
-            Row {
-                rules: vec![Rule::pattern(r"\p{Emoji}+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("🐎", Some((0, "🐎"))),
-                    ("🐴🐴", Some((0, "🐴🐴"))),
-                    ("#0", Some((0, "#0"))), // These chars are technically emojis!
-                    ("⻢", None),
-                    ("♞", None),
-                    ("horse", None),
-                ],
-            },
-            // Intersection
-            Row {
-                rules: vec![Rule::pattern(r"[[0-7]&&[4-9]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("456", Some((0, "456"))),
-                    ("64", Some((0, "64"))),
-                    ("452", Some((0, "45"))),
-                    ("91", None),
-                    ("8", None),
-                    ("3", None),
-                ],
-            },
-            // Difference
-            Row {
-                rules: vec![Rule::pattern(r"[[0-9]--[4-7]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("123", Some((0, "123"))),
-                    ("83", Some((0, "83"))),
-                    ("9", Some((0, "9"))),
-                    ("124", Some((0, "12"))),
-                    ("67", None),
-                    ("4", None),
-                ],
-            },
-            // Symmetric difference
-            Row {
-                rules: vec![Rule::pattern(r"[[0-7]~~[4-9]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("123", Some((0, "123"))),
-                    ("83", Some((0, "83"))),
-                    ("9", Some((0, "9"))),
-                    ("124", Some((0, "12"))),
-                    ("67", None),
-                    ("4", None),
-                ],
-            },
-            // Nested set operations
-            Row {
-                //               0 1 2 3 4 5 6 7 8 9
-                // [0-5]:        y y y y y y
-                // [2-4]:            y y y
-                // [0-5]--[2-4]: y y       y
-                // [3-9]:              y y y y y y y
-                // [6-7]:                    y y
-                // [3-9]--[5-7]:       y y y     y y
-                // final regex:  y y   y y       y y
-                rules: vec![Rule::pattern(r"[[[0-5]--[2-4]]~~[[3-9]--[6-7]]]+", "")],
-                separators: vec![],
-                examples: vec![
-                    ("01", Some((0, "01"))),
-                    ("432", Some((0, "43"))),
-                    ("8", Some((0, "8"))),
-                    ("9", Some((0, "9"))),
-                    ("2", None),
-                    ("567", None),
-                ],
-            },
-        ];
-
-        for Row {
-            rules,
-            separators,
-            examples,
-        } in &table
-        {
-            let grammar = expand_tokens(ExtractedLexicalGrammar {
-                separators: separators.clone(),
-                variables: rules
-                    .iter()
-                    .map(|rule| Variable::named("", rule.clone()))
-                    .collect(),
-            })
-            .unwrap();
-
-            for (haystack, needle) in examples {
-                assert_eq!(simulate_nfa(&grammar, haystack), *needle);
-            }
-        }
-    }
-}

cli/generate/src/prepare_grammar/extract_default_aliases.rs

@@ -1,304 +0,0 @@
-use crate::{
-    grammars::{LexicalGrammar, SyntaxGrammar},
-    rules::{Alias, AliasMap, Symbol, SymbolType},
-};
-
-#[derive(Clone, Default)]
-struct SymbolStatus {
-    aliases: Vec<(Alias, usize)>,
-    appears_unaliased: bool,
-}
-
-// Update the grammar by finding symbols that always are aliased, and for each such symbol,
-// promoting one of its aliases to a "default alias", which is applied globally instead
-// of in a context-specific way.
-//
-// This has two benefits:
-// * It reduces the overhead of storing production-specific alias info in the parse table.
-// * Within an `ERROR` node, no context-specific aliases will be applied. This transformation
-//   ensures that the children of an `ERROR` node have symbols that are consistent with the way that
-//   they would appear in a valid syntax tree.
-pub(super) fn extract_default_aliases(
-    syntax_grammar: &mut SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-) -> AliasMap {
-    let mut terminal_status_list = vec![SymbolStatus::default(); lexical_grammar.variables.len()];
-    let mut non_terminal_status_list =
-        vec![SymbolStatus::default(); syntax_grammar.variables.len()];
-    let mut external_status_list =
-        vec![SymbolStatus::default(); syntax_grammar.external_tokens.len()];
-
-    // For each grammar symbol, find all of the aliases under which the symbol appears,
-    // and determine whether or not the symbol ever appears *unaliased*.
-    for variable in &syntax_grammar.variables {
-        for production in &variable.productions {
-            for step in &production.steps {
-                let status = match step.symbol.kind {
-                    SymbolType::External => &mut external_status_list[step.symbol.index],
-                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
-                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
-                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                        panic!("Unexpected end token")
-                    }
-                };
-
-                // Default aliases don't work for inlined variables.
-                if syntax_grammar.variables_to_inline.contains(&step.symbol) {
-                    continue;
-                }
-
-                if let Some(alias) = &step.alias {
-                    if let Some(count_for_alias) = status
-                        .aliases
-                        .iter_mut()
-                        .find_map(|(a, count)| if a == alias { Some(count) } else { None })
-                    {
-                        *count_for_alias += 1;
-                    } else {
-                        status.aliases.push((alias.clone(), 1));
-                    }
-                } else {
-                    status.appears_unaliased = true;
-                }
-            }
-        }
-    }
-
-    for symbol in &syntax_grammar.extra_symbols {
-        let status = match symbol.kind {
-            SymbolType::External => &mut external_status_list[symbol.index],
-            SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
-            SymbolType::Terminal => &mut terminal_status_list[symbol.index],
-            SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                panic!("Unexpected end token")
-            }
-        };
-        status.appears_unaliased = true;
-    }
-
-    let symbols_with_statuses = (terminal_status_list
-        .iter_mut()
-        .enumerate()
-        .map(|(i, status)| (Symbol::terminal(i), status)))
-    .chain(
-        non_terminal_status_list
-            .iter_mut()
-            .enumerate()
-            .map(|(i, status)| (Symbol::non_terminal(i), status)),
-    )
-    .chain(
-        external_status_list
-            .iter_mut()
-            .enumerate()
-            .map(|(i, status)| (Symbol::external(i), status)),
-    );
-
-    // For each symbol that always appears aliased, find the alias the occurs most often,
-    // and designate that alias as the symbol's "default alias". Store all of these
-    // default aliases in a map that will be returned.
-    let mut result = AliasMap::new();
-    for (symbol, status) in symbols_with_statuses {
-        if status.appears_unaliased {
-            status.aliases.clear();
-        } else if let Some(default_entry) = status
-            .aliases
-            .iter()
-            .enumerate()
-            .max_by_key(|(i, (_, count))| (count, -(*i as i64)))
-            .map(|(_, entry)| entry.clone())
-        {
-            status.aliases.clear();
-            status.aliases.push(default_entry.clone());
-            result.insert(symbol, default_entry.0);
-        }
-    }
-
-    // Wherever a symbol is aliased as its default alias, remove the usage of the alias,
-    // because it will now be redundant.
-    let mut alias_positions_to_clear = Vec::new();
-    for variable in &mut syntax_grammar.variables {
-        alias_positions_to_clear.clear();
-
-        for (i, production) in variable.productions.iter().enumerate() {
-            for (j, step) in production.steps.iter().enumerate() {
-                let status = match step.symbol.kind {
-                    SymbolType::External => &mut external_status_list[step.symbol.index],
-                    SymbolType::NonTerminal => &mut non_terminal_status_list[step.symbol.index],
-                    SymbolType::Terminal => &mut terminal_status_list[step.symbol.index],
-                    SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
-                        panic!("Unexpected end token")
-                    }
-                };
-
-                // If this step is aliased as the symbol's default alias, then remove that alias.
-                if step.alias.is_some()
-                    && step.alias.as_ref() == status.aliases.first().map(|t| &t.0)
-                {
-                    let mut other_productions_must_use_this_alias_at_this_index = false;
-                    for (other_i, other_production) in variable.productions.iter().enumerate() {
-                        if other_i != i
-                            && other_production.steps.len() > j
-                            && other_production.steps[j].alias == step.alias
-                            && result.get(&other_production.steps[j].symbol) != step.alias.as_ref()
-                        {
-                            other_productions_must_use_this_alias_at_this_index = true;
-                            break;
-                        }
-                    }
-
-                    if !other_productions_must_use_this_alias_at_this_index {
-                        alias_positions_to_clear.push((i, j));
-                    }
-                }
-            }
-        }
-
-        for (production_index, step_index) in &alias_positions_to_clear {
-            variable.productions[*production_index].steps[*step_index].alias = None;
-        }
-    }
-
-    result
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType},
-        nfa::Nfa,
-    };
-
-    #[test]
-    fn test_extract_simple_aliases() {
-        let mut syntax_grammar = SyntaxGrammar {
-            variables: vec![
-                SyntaxVariable {
-                    name: "v1".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
-                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
-                            ProductionStep::new(Symbol::terminal(2)).with_alias("a3", true),
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "v2".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            // Token 0 is always aliased as "a1".
-                            ProductionStep::new(Symbol::terminal(0)).with_alias("a1", true),
-                            // Token 1 is aliased within rule `v1` above, but not here.
-                            ProductionStep::new(Symbol::terminal(1)),
-                            // Token 2 is aliased differently here than in `v1`. The alias from
-                            // `v1` should be promoted to the default alias, because `v1` appears
-                            // first in the grammar.
-                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
-                            // Token 3 is also aliased differently here than in `v1`. In this case,
-                            // this alias should be promoted to the default alias, because it is
-                            // used a greater number of times (twice).
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a6", true),
-                        ],
-                    }],
-                },
-            ],
-            ..Default::default()
-        };
-
-        let lexical_grammar = LexicalGrammar {
-            nfa: Nfa::new(),
-            variables: vec![
-                LexicalVariable {
-                    name: "t0".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-                LexicalVariable {
-                    name: "t1".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-                LexicalVariable {
-                    name: "t2".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-                LexicalVariable {
-                    name: "t3".to_string(),
-                    kind: VariableType::Anonymous,
-                    implicit_precedence: 0,
-                    start_state: 0,
-                },
-            ],
-        };
-
-        let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
-        assert_eq!(default_aliases.len(), 3);
-
-        assert_eq!(
-            default_aliases.get(&Symbol::terminal(0)),
-            Some(&Alias {
-                value: "a1".to_string(),
-                is_named: true,
-            })
-        );
-        assert_eq!(
-            default_aliases.get(&Symbol::terminal(2)),
-            Some(&Alias {
-                value: "a3".to_string(),
-                is_named: true,
-            })
-        );
-        assert_eq!(
-            default_aliases.get(&Symbol::terminal(3)),
-            Some(&Alias {
-                value: "a6".to_string(),
-                is_named: true,
-            })
-        );
-        assert_eq!(default_aliases.get(&Symbol::terminal(1)), None);
-
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![
-                SyntaxVariable {
-                    name: "v1".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(0)),
-                            ProductionStep::new(Symbol::terminal(1)).with_alias("a2", true),
-                            ProductionStep::new(Symbol::terminal(2)),
-                            ProductionStep::new(Symbol::terminal(3)).with_alias("a4", true),
-                        ],
-                    },],
-                },
-                SyntaxVariable {
-                    name: "v2".to_owned(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(0)),
-                            ProductionStep::new(Symbol::terminal(1)),
-                            ProductionStep::new(Symbol::terminal(2)).with_alias("a5", true),
-                            ProductionStep::new(Symbol::terminal(3)),
-                            ProductionStep::new(Symbol::terminal(3)),
-                        ],
-                    },],
-                },
-            ]
-        );
-    }
-}

cli/generate/src/prepare_grammar/extract_tokens.rs

@@ -1,554 +0,0 @@
-use std::{collections::HashMap, mem};
-
-use anyhow::{anyhow, Result};
-
-use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar};
-use crate::{
-    grammars::{ExternalToken, Variable, VariableType},
-    rules::{MetadataParams, Rule, Symbol, SymbolType},
-};
-
-pub(super) fn extract_tokens(
-    mut grammar: InternedGrammar,
-) -> Result<(ExtractedSyntaxGrammar, ExtractedLexicalGrammar)> {
-    let mut extractor = TokenExtractor {
-        current_variable_name: String::new(),
-        current_variable_token_count: 0,
-        is_first_rule: false,
-        extracted_variables: Vec::new(),
-        extracted_usage_counts: Vec::new(),
-    };
-
-    for (i, variable) in &mut grammar.variables.iter_mut().enumerate() {
-        extractor.extract_tokens_in_variable(i == 0, variable)?;
-    }
-
-    for variable in &mut grammar.external_tokens {
-        extractor.extract_tokens_in_variable(false, variable)?;
-    }
-
-    let mut lexical_variables = Vec::with_capacity(extractor.extracted_variables.len());
-    for variable in extractor.extracted_variables {
-        lexical_variables.push(variable);
-    }
-
-    // If a variable's entire rule was extracted as a token and that token didn't
-    // appear within any other rule, then remove that variable from the syntax
-    // grammar, giving its name to the token in the lexical grammar. Any symbols
-    // that pointed to that variable will need to be updated to point to the
-    // variable in the lexical grammar. Symbols that pointed to later variables
-    // will need to have their indices decremented.
-    let mut variables = Vec::new();
-    let mut symbol_replacer = SymbolReplacer {
-        replacements: HashMap::new(),
-    };
-    for (i, variable) in grammar.variables.into_iter().enumerate() {
-        if let Rule::Symbol(Symbol {
-            kind: SymbolType::Terminal,
-            index,
-        }) = variable.rule
-        {
-            if i > 0 && extractor.extracted_usage_counts[index] == 1 {
-                let lexical_variable = &mut lexical_variables[index];
-                if lexical_variable.kind == VariableType::Auxiliary
-                    || variable.kind != VariableType::Hidden
-                {
-                    lexical_variable.kind = variable.kind;
-                    lexical_variable.name = variable.name;
-                    symbol_replacer.replacements.insert(i, index);
-                    continue;
-                }
-            }
-        }
-        variables.push(variable);
-    }
-
-    for variable in &mut variables {
-        variable.rule = symbol_replacer.replace_symbols_in_rule(&variable.rule);
-    }
-
-    let expected_conflicts = grammar
-        .expected_conflicts
-        .into_iter()
-        .map(|conflict| {
-            let mut result = conflict
-                .iter()
-                .map(|symbol| symbol_replacer.replace_symbol(*symbol))
-                .collect::<Vec<_>>();
-            result.sort_unstable();
-            result.dedup();
-            result
-        })
-        .collect();
-
-    let supertype_symbols = grammar
-        .supertype_symbols
-        .into_iter()
-        .map(|symbol| symbol_replacer.replace_symbol(symbol))
-        .collect();
-
-    let variables_to_inline = grammar
-        .variables_to_inline
-        .into_iter()
-        .map(|symbol| symbol_replacer.replace_symbol(symbol))
-        .collect();
-
-    let mut separators = Vec::new();
-    let mut extra_symbols = Vec::new();
-    for rule in grammar.extra_symbols {
-        if let Rule::Symbol(symbol) = rule {
-            extra_symbols.push(symbol_replacer.replace_symbol(symbol));
-        } else if let Some(index) = lexical_variables.iter().position(|v| v.rule == rule) {
-            extra_symbols.push(Symbol::terminal(index));
-        } else {
-            separators.push(rule);
-        }
-    }
-
-    let mut external_tokens = Vec::new();
-    for external_token in grammar.external_tokens {
-        let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule);
-        if let Rule::Symbol(symbol) = rule {
-            if symbol.is_non_terminal() {
-                return Err(anyhow!(
-                    "Rule '{}' cannot be used as both an external token and a non-terminal rule",
-                    &variables[symbol.index].name,
-                ));
-            }
-
-            if symbol.is_external() {
-                external_tokens.push(ExternalToken {
-                    name: external_token.name,
-                    kind: external_token.kind,
-                    corresponding_internal_token: None,
-                });
-            } else {
-                external_tokens.push(ExternalToken {
-                    name: lexical_variables[symbol.index].name.clone(),
-                    kind: external_token.kind,
-                    corresponding_internal_token: Some(symbol),
-                });
-            }
-        } else {
-            return Err(anyhow!(
-                "Non-symbol rules cannot be used as external tokens"
-            ));
-        }
-    }
-
-    let mut word_token = None;
-    if let Some(token) = grammar.word_token {
-        let token = symbol_replacer.replace_symbol(token);
-        if token.is_non_terminal() {
-            return Err(anyhow!(
-                "Non-terminal symbol '{}' cannot be used as the word token",
-                &variables[token.index].name
-            ));
-        }
-        word_token = Some(token);
-    }
-
-    Ok((
-        ExtractedSyntaxGrammar {
-            variables,
-            expected_conflicts,
-            extra_symbols,
-            variables_to_inline,
-            supertype_symbols,
-            external_tokens,
-            word_token,
-            precedence_orderings: grammar.precedence_orderings,
-        },
-        ExtractedLexicalGrammar {
-            variables: lexical_variables,
-            separators,
-        },
-    ))
-}
-
-struct TokenExtractor {
-    current_variable_name: String,
-    current_variable_token_count: usize,
-    is_first_rule: bool,
-    extracted_variables: Vec<Variable>,
-    extracted_usage_counts: Vec<usize>,
-}
-
-struct SymbolReplacer {
-    replacements: HashMap<usize, usize>,
-}
-
-impl TokenExtractor {
-    fn extract_tokens_in_variable(
-        &mut self,
-        is_first: bool,
-        variable: &mut Variable,
-    ) -> Result<()> {
-        self.current_variable_name.clear();
-        self.current_variable_name.push_str(&variable.name);
-        self.current_variable_token_count = 0;
-        self.is_first_rule = is_first;
-        let mut rule = Rule::Blank;
-        mem::swap(&mut rule, &mut variable.rule);
-        variable.rule = self.extract_tokens_in_rule(&rule)?;
-        Ok(())
-    }
-
-    fn extract_tokens_in_rule(&mut self, input: &Rule) -> Result<Rule> {
-        match input {
-            Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()),
-            Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()),
-            Rule::Metadata { params, rule } => {
-                if params.is_token {
-                    let mut params = params.clone();
-                    params.is_token = false;
-
-                    let mut string_value = None;
-                    if let Rule::String(value) = rule.as_ref() {
-                        string_value = Some(value);
-                    }
-
-                    let rule_to_extract = if params == MetadataParams::default() {
-                        rule.as_ref()
-                    } else {
-                        input
-                    };
-
-                    Ok(self.extract_token(rule_to_extract, string_value)?.into())
-                } else {
-                    Ok(Rule::Metadata {
-                        params: params.clone(),
-                        rule: Box::new(self.extract_tokens_in_rule(rule)?),
-                    })
-                }
-            }
-            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(
-                self.extract_tokens_in_rule(content)?,
-            ))),
-            Rule::Seq(elements) => Ok(Rule::Seq(
-                elements
-                    .iter()
-                    .map(|e| self.extract_tokens_in_rule(e))
-                    .collect::<Result<Vec<_>>>()?,
-            )),
-            Rule::Choice(elements) => Ok(Rule::Choice(
-                elements
-                    .iter()
-                    .map(|e| self.extract_tokens_in_rule(e))
-                    .collect::<Result<Vec<_>>>()?,
-            )),
-            _ => Ok(input.clone()),
-        }
-    }
-
-    fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Result<Symbol> {
-        for (i, variable) in self.extracted_variables.iter_mut().enumerate() {
-            if variable.rule == *rule {
-                self.extracted_usage_counts[i] += 1;
-                return Ok(Symbol::terminal(i));
-            }
-        }
-
-        let index = self.extracted_variables.len();
-        let variable = if let Some(string_value) = string_value {
-            if string_value.is_empty() && !self.is_first_rule {
-                return Err(anyhow!(
-                    "The rule `{}` contains an empty string.
-
-Tree-sitter does not support syntactic rules that contain an empty string
-unless they are used only as the grammar's start rule.
-",
-                    self.current_variable_name
-                ));
-            }
-            Variable {
-                name: string_value.clone(),
-                kind: VariableType::Anonymous,
-                rule: rule.clone(),
-            }
-        } else {
-            self.current_variable_token_count += 1;
-            Variable {
-                name: format!(
-                    "{}_token{}",
-                    &self.current_variable_name, self.current_variable_token_count
-                ),
-                kind: VariableType::Auxiliary,
-                rule: rule.clone(),
-            }
-        };
-
-        self.extracted_variables.push(variable);
-        self.extracted_usage_counts.push(1);
-        Ok(Symbol::terminal(index))
-    }
-}
-
-impl SymbolReplacer {
-    fn replace_symbols_in_rule(&mut self, rule: &Rule) -> Rule {
-        match rule {
-            Rule::Symbol(symbol) => self.replace_symbol(*symbol).into(),
-            Rule::Choice(elements) => Rule::Choice(
-                elements
-                    .iter()
-                    .map(|e| self.replace_symbols_in_rule(e))
-                    .collect(),
-            ),
-            Rule::Seq(elements) => Rule::Seq(
-                elements
-                    .iter()
-                    .map(|e| self.replace_symbols_in_rule(e))
-                    .collect(),
-            ),
-            Rule::Repeat(content) => Rule::Repeat(Box::new(self.replace_symbols_in_rule(content))),
-            Rule::Metadata { rule, params } => Rule::Metadata {
-                params: params.clone(),
-                rule: Box::new(self.replace_symbols_in_rule(rule)),
-            },
-            _ => rule.clone(),
-        }
-    }
-
-    fn replace_symbol(&self, symbol: Symbol) -> Symbol {
-        if !symbol.is_non_terminal() {
-            return symbol;
-        }
-
-        if let Some(replacement) = self.replacements.get(&symbol.index) {
-            return Symbol::terminal(*replacement);
-        }
-
-        let mut adjusted_index = symbol.index;
-        for replaced_index in self.replacements.keys() {
-            if *replaced_index < symbol.index {
-                adjusted_index -= 1;
-            }
-        }
-
-        Symbol::non_terminal(adjusted_index)
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_extraction() {
-        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
-            Variable::named(
-                "rule_0",
-                Rule::repeat(Rule::seq(vec![
-                    Rule::string("a"),
-                    Rule::pattern("b", ""),
-                    Rule::choice(vec![
-                        Rule::non_terminal(1),
-                        Rule::non_terminal(2),
-                        Rule::token(Rule::repeat(Rule::choice(vec![
-                            Rule::string("c"),
-                            Rule::string("d"),
-                        ]))),
-                    ]),
-                ])),
-            ),
-            Variable::named("rule_1", Rule::pattern("e", "")),
-            Variable::named("rule_2", Rule::pattern("b", "")),
-            Variable::named(
-                "rule_3",
-                Rule::seq(vec![Rule::non_terminal(2), Rule::Blank]),
-            ),
-        ]))
-        .unwrap();
-
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![
-                Variable::named(
-                    "rule_0",
-                    Rule::repeat(Rule::seq(vec![
-                        // The string "a" was replaced by a symbol referencing the lexical grammar
-                        Rule::terminal(0),
-                        // The pattern "b" was replaced by a symbol referencing the lexical grammar
-                        Rule::terminal(1),
-                        Rule::choice(vec![
-                            // The symbol referencing `rule_1` was replaced by a symbol referencing
-                            // the lexical grammar.
-                            Rule::terminal(3),
-                            // The symbol referencing `rule_2` had its index decremented because
-                            // `rule_1` was moved to the lexical grammar.
-                            Rule::non_terminal(1),
-                            // The rule wrapped in `token` was replaced by a symbol referencing
-                            // the lexical grammar.
-                            Rule::terminal(2),
-                        ])
-                    ]))
-                ),
-                // The pattern "e" was only used in once place: as the definition of `rule_1`,
-                // so that rule was moved to the lexical grammar. The pattern "b" appeared in
-                // two places, so it was not moved into the lexical grammar.
-                Variable::named("rule_2", Rule::terminal(1)),
-                Variable::named(
-                    "rule_3",
-                    Rule::seq(vec![Rule::non_terminal(1), Rule::Blank,])
-                ),
-            ]
-        );
-
-        assert_eq!(
-            lexical_grammar.variables,
-            vec![
-                Variable::anonymous("a", Rule::string("a")),
-                Variable::auxiliary("rule_0_token1", Rule::pattern("b", "")),
-                Variable::auxiliary(
-                    "rule_0_token2",
-                    Rule::repeat(Rule::choice(vec![Rule::string("c"), Rule::string("d"),]))
-                ),
-                Variable::named("rule_1", Rule::pattern("e", "")),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_start_rule_is_token() {
-        let (syntax_grammar, lexical_grammar) =
-            extract_tokens(build_grammar(vec![Variable::named(
-                "rule_0",
-                Rule::string("hello"),
-            )]))
-            .unwrap();
-
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![Variable::named("rule_0", Rule::terminal(0)),]
-        );
-        assert_eq!(
-            lexical_grammar.variables,
-            vec![Variable::anonymous("hello", Rule::string("hello")),]
-        );
-    }
-
-    #[test]
-    fn test_extracting_extra_symbols() {
-        let mut grammar = build_grammar(vec![
-            Variable::named("rule_0", Rule::string("x")),
-            Variable::named("comment", Rule::pattern("//.*", "")),
-        ]);
-        grammar.extra_symbols = vec![Rule::string(" "), Rule::non_terminal(1)];
-
-        let (syntax_grammar, lexical_grammar) = extract_tokens(grammar).unwrap();
-        assert_eq!(syntax_grammar.extra_symbols, vec![Symbol::terminal(1),]);
-        assert_eq!(lexical_grammar.separators, vec![Rule::string(" "),]);
-    }
-
-    #[test]
-    fn test_extract_externals() {
-        let mut grammar = build_grammar(vec![
-            Variable::named(
-                "rule_0",
-                Rule::seq(vec![
-                    Rule::external(0),
-                    Rule::string("a"),
-                    Rule::non_terminal(1),
-                    Rule::non_terminal(2),
-                ]),
-            ),
-            Variable::named("rule_1", Rule::string("b")),
-            Variable::named("rule_2", Rule::string("c")),
-        ]);
-        grammar.external_tokens = vec![
-            Variable::named("external_0", Rule::external(0)),
-            Variable::anonymous("a", Rule::string("a")),
-            Variable::named("rule_2", Rule::non_terminal(2)),
-        ];
-
-        let (syntax_grammar, _) = extract_tokens(grammar).unwrap();
-
-        assert_eq!(
-            syntax_grammar.external_tokens,
-            vec![
-                ExternalToken {
-                    name: "external_0".to_string(),
-                    kind: VariableType::Named,
-                    corresponding_internal_token: None,
-                },
-                ExternalToken {
-                    name: "a".to_string(),
-                    kind: VariableType::Anonymous,
-                    corresponding_internal_token: Some(Symbol::terminal(0)),
-                },
-                ExternalToken {
-                    name: "rule_2".to_string(),
-                    kind: VariableType::Named,
-                    corresponding_internal_token: Some(Symbol::terminal(2)),
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_error_on_external_with_same_name_as_non_terminal() {
-        let mut grammar = build_grammar(vec![
-            Variable::named(
-                "rule_0",
-                Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
-            ),
-            Variable::named(
-                "rule_1",
-                Rule::seq(vec![Rule::non_terminal(2), Rule::non_terminal(2)]),
-            ),
-            Variable::named("rule_2", Rule::string("a")),
-        ]);
-        grammar.external_tokens = vec![Variable::named("rule_1", Rule::non_terminal(1))];
-
-        match extract_tokens(grammar) {
-            Err(e) => {
-                assert_eq!(e.to_string(), "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule");
-            }
-            _ => {
-                panic!("Expected an error but got no error");
-            }
-        }
-    }
-
-    #[test]
-    fn test_extraction_on_hidden_terminal() {
-        let (syntax_grammar, lexical_grammar) = extract_tokens(build_grammar(vec![
-            Variable::named("rule_0", Rule::non_terminal(1)),
-            Variable::hidden("_rule_1", Rule::string("a")),
-        ]))
-        .unwrap();
-
-        // The rule `_rule_1` should not "absorb" the
-        // terminal "a", since it is hidden,
-        // so we expect two variables still
-        assert_eq!(
-            syntax_grammar.variables,
-            vec![
-                Variable::named("rule_0", Rule::non_terminal(1)),
-                Variable::hidden("_rule_1", Rule::terminal(0)),
-            ]
-        );
-
-        // We should not have a hidden rule in our lexical grammar, only the terminal "a"
-        assert_eq!(
-            lexical_grammar.variables,
-            vec![Variable::anonymous("a", Rule::string("a"))]
-        );
-    }
-
-    #[test]
-    fn test_extraction_with_empty_string() {
-        assert!(extract_tokens(build_grammar(vec![
-            Variable::named("rule_0", Rule::non_terminal(1)),
-            Variable::hidden("_rule_1", Rule::string("")),
-        ]))
-        .is_err());
-    }
-
-    fn build_grammar(variables: Vec<Variable>) -> InternedGrammar {
-        InternedGrammar {
-            variables,
-            ..Default::default()
-        }
-    }
-}

cli/generate/src/prepare_grammar/flatten_grammar.rs

@@ -1,453 +0,0 @@
-use anyhow::{anyhow, Result};
-
-use super::ExtractedSyntaxGrammar;
-use crate::{
-    grammars::{Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable},
-    rules::{Alias, Associativity, Precedence, Rule, Symbol},
-};
-
-struct RuleFlattener {
-    production: Production,
-    precedence_stack: Vec<Precedence>,
-    associativity_stack: Vec<Associativity>,
-    alias_stack: Vec<Alias>,
-    field_name_stack: Vec<String>,
-}
-
-impl RuleFlattener {
-    const fn new() -> Self {
-        Self {
-            production: Production {
-                steps: Vec::new(),
-                dynamic_precedence: 0,
-            },
-            precedence_stack: Vec::new(),
-            associativity_stack: Vec::new(),
-            alias_stack: Vec::new(),
-            field_name_stack: Vec::new(),
-        }
-    }
-
-    fn flatten(mut self, rule: Rule) -> Production {
-        self.apply(rule, true);
-        self.production
-    }
-
-    fn apply(&mut self, rule: Rule, at_end: bool) -> bool {
-        match rule {
-            Rule::Seq(members) => {
-                let mut result = false;
-                let last_index = members.len() - 1;
-                for (i, member) in members.into_iter().enumerate() {
-                    result |= self.apply(member, i == last_index && at_end);
-                }
-                result
-            }
-            Rule::Metadata { rule, params } => {
-                let mut has_precedence = false;
-                if !params.precedence.is_none() {
-                    has_precedence = true;
-                    self.precedence_stack.push(params.precedence);
-                }
-
-                let mut has_associativity = false;
-                if let Some(associativity) = params.associativity {
-                    has_associativity = true;
-                    self.associativity_stack.push(associativity);
-                }
-
-                let mut has_alias = false;
-                if let Some(alias) = params.alias {
-                    has_alias = true;
-                    self.alias_stack.push(alias);
-                }
-
-                let mut has_field_name = false;
-                if let Some(field_name) = params.field_name {
-                    has_field_name = true;
-                    self.field_name_stack.push(field_name);
-                }
-
-                if params.dynamic_precedence.abs() > self.production.dynamic_precedence.abs() {
-                    self.production.dynamic_precedence = params.dynamic_precedence;
-                }
-
-                let did_push = self.apply(*rule, at_end);
-
-                if has_precedence {
-                    self.precedence_stack.pop();
-                    if did_push && !at_end {
-                        self.production.steps.last_mut().unwrap().precedence = self
-                            .precedence_stack
-                            .last()
-                            .cloned()
-                            .unwrap_or(Precedence::None);
-                    }
-                }
-
-                if has_associativity {
-                    self.associativity_stack.pop();
-                    if did_push && !at_end {
-                        self.production.steps.last_mut().unwrap().associativity =
-                            self.associativity_stack.last().copied();
-                    }
-                }
-
-                if has_alias {
-                    self.alias_stack.pop();
-                }
-
-                if has_field_name {
-                    self.field_name_stack.pop();
-                }
-
-                did_push
-            }
-            Rule::Symbol(symbol) => {
-                self.production.steps.push(ProductionStep {
-                    symbol,
-                    precedence: self
-                        .precedence_stack
-                        .last()
-                        .cloned()
-                        .unwrap_or(Precedence::None),
-                    associativity: self.associativity_stack.last().copied(),
-                    alias: self.alias_stack.last().cloned(),
-                    field_name: self.field_name_stack.last().cloned(),
-                });
-                true
-            }
-            _ => false,
-        }
-    }
-}
-
-fn extract_choices(rule: Rule) -> Vec<Rule> {
-    match rule {
-        Rule::Seq(elements) => {
-            let mut result = vec![Rule::Blank];
-            for element in elements {
-                let extraction = extract_choices(element);
-                let mut next_result = Vec::new();
-                for entry in result {
-                    for extraction_entry in &extraction {
-                        next_result.push(Rule::Seq(vec![entry.clone(), extraction_entry.clone()]));
-                    }
-                }
-                result = next_result;
-            }
-            result
-        }
-        Rule::Choice(elements) => {
-            let mut result = Vec::new();
-            for element in elements {
-                for rule in extract_choices(element) {
-                    result.push(rule);
-                }
-            }
-            result
-        }
-        Rule::Metadata { rule, params } => extract_choices(*rule)
-            .into_iter()
-            .map(|rule| Rule::Metadata {
-                rule: Box::new(rule),
-                params: params.clone(),
-            })
-            .collect(),
-        _ => vec![rule],
-    }
-}
-
-fn flatten_variable(variable: Variable) -> SyntaxVariable {
-    let mut productions = Vec::new();
-    for rule in extract_choices(variable.rule) {
-        let production = RuleFlattener::new().flatten(rule);
-        if !productions.contains(&production) {
-            productions.push(production);
-        }
-    }
-    SyntaxVariable {
-        name: variable.name,
-        kind: variable.kind,
-        productions,
-    }
-}
-
-fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool {
-    for variable in variables {
-        for production in &variable.productions {
-            for step in &production.steps {
-                if step.symbol == symbol {
-                    return true;
-                }
-            }
-        }
-    }
-    false
-}
-
-pub(super) fn flatten_grammar(grammar: ExtractedSyntaxGrammar) -> Result<SyntaxGrammar> {
-    let mut variables = Vec::new();
-    for variable in grammar.variables {
-        variables.push(flatten_variable(variable));
-    }
-    for (i, variable) in variables.iter().enumerate() {
-        let symbol = Symbol::non_terminal(i);
-
-        for production in &variable.productions {
-            if production.steps.is_empty() && symbol_is_used(&variables, symbol) {
-                return Err(anyhow!(
-                    "The rule `{}` matches the empty string.
-
-Tree-sitter does not support syntactic rules that match the empty string
-unless they are used only as the grammar's start rule.
-",
-                    variable.name
-                ));
-            }
-
-            if grammar.variables_to_inline.contains(&symbol)
-                && production.steps.iter().any(|step| step.symbol == symbol)
-            {
-                return Err(anyhow!(
-                    "Rule `{}` cannot be inlined because it contains a reference to itself.",
-                    variable.name,
-                ));
-            }
-        }
-    }
-    Ok(SyntaxGrammar {
-        extra_symbols: grammar.extra_symbols,
-        expected_conflicts: grammar.expected_conflicts,
-        variables_to_inline: grammar.variables_to_inline,
-        precedence_orderings: grammar.precedence_orderings,
-        external_tokens: grammar.external_tokens,
-        supertype_symbols: grammar.supertype_symbols,
-        word_token: grammar.word_token,
-        variables,
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::grammars::VariableType;
-
-    #[test]
-    fn test_flatten_grammar() {
-        let result = flatten_variable(Variable {
-            name: "test".to_string(),
-            kind: VariableType::Named,
-            rule: Rule::seq(vec![
-                Rule::non_terminal(1),
-                Rule::prec_left(
-                    Precedence::Integer(101),
-                    Rule::seq(vec![
-                        Rule::non_terminal(2),
-                        Rule::choice(vec![
-                            Rule::prec_right(
-                                Precedence::Integer(102),
-                                Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
-                            ),
-                            Rule::non_terminal(5),
-                        ]),
-                        Rule::non_terminal(6),
-                    ]),
-                ),
-                Rule::non_terminal(7),
-            ]),
-        });
-
-        assert_eq!(
-            result.productions,
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(3))
-                            .with_prec(Precedence::Integer(102), Some(Associativity::Right)),
-                        ProductionStep::new(Symbol::non_terminal(4))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ]
-                },
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(5))
-                            .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ]
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_maximum_dynamic_precedence() {
-        let result = flatten_variable(Variable {
-            name: "test".to_string(),
-            kind: VariableType::Named,
-            rule: Rule::seq(vec![
-                Rule::non_terminal(1),
-                Rule::prec_dynamic(
-                    101,
-                    Rule::seq(vec![
-                        Rule::non_terminal(2),
-                        Rule::choice(vec![
-                            Rule::prec_dynamic(
-                                102,
-                                Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
-                            ),
-                            Rule::non_terminal(5),
-                        ]),
-                        Rule::non_terminal(6),
-                    ]),
-                ),
-                Rule::non_terminal(7),
-            ]),
-        });
-
-        assert_eq!(
-            result.productions,
-            vec![
-                Production {
-                    dynamic_precedence: 102,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::non_terminal(3)),
-                        ProductionStep::new(Symbol::non_terminal(4)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ],
-                },
-                Production {
-                    dynamic_precedence: 101,
-                    steps: vec![
-                        ProductionStep::new(Symbol::non_terminal(1)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::non_terminal(5)),
-                        ProductionStep::new(Symbol::non_terminal(6)),
-                        ProductionStep::new(Symbol::non_terminal(7)),
-                    ],
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_final_precedence() {
-        let result = flatten_variable(Variable {
-            name: "test".to_string(),
-            kind: VariableType::Named,
-            rule: Rule::prec_left(
-                Precedence::Integer(101),
-                Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
-            ),
-        });
-
-        assert_eq!(
-            result.productions,
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    ProductionStep::new(Symbol::non_terminal(1))
-                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                    ProductionStep::new(Symbol::non_terminal(2))
-                        .with_prec(Precedence::Integer(101), Some(Associativity::Left)),
-                ]
-            }]
-        );
-
-        let result = flatten_variable(Variable {
-            name: "test".to_string(),
-            kind: VariableType::Named,
-            rule: Rule::prec_left(
-                Precedence::Integer(101),
-                Rule::seq(vec![Rule::non_terminal(1)]),
-            ),
-        });
-
-        assert_eq!(
-            result.productions,
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![ProductionStep::new(Symbol::non_terminal(1))
-                    .with_prec(Precedence::Integer(101), Some(Associativity::Left)),]
-            }]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_field_names() {
-        let result = flatten_variable(Variable {
-            name: "test".to_string(),
-            kind: VariableType::Named,
-            rule: Rule::seq(vec![
-                Rule::field("first-thing".to_string(), Rule::terminal(1)),
-                Rule::terminal(2),
-                Rule::choice(vec![
-                    Rule::Blank,
-                    Rule::field("second-thing".to_string(), Rule::terminal(3)),
-                ]),
-            ]),
-        });
-
-        assert_eq!(
-            result.productions,
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
-                        ProductionStep::new(Symbol::terminal(2))
-                    ]
-                },
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(1)).with_field_name("first-thing"),
-                        ProductionStep::new(Symbol::terminal(2)),
-                        ProductionStep::new(Symbol::terminal(3)).with_field_name("second-thing"),
-                    ]
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_flatten_grammar_with_recursive_inline_variable() {
-        let result = flatten_grammar(ExtractedSyntaxGrammar {
-            extra_symbols: Vec::new(),
-            expected_conflicts: Vec::new(),
-            variables_to_inline: vec![Symbol::non_terminal(0)],
-            precedence_orderings: Vec::new(),
-            external_tokens: Vec::new(),
-            supertype_symbols: Vec::new(),
-            word_token: None,
-            variables: vec![Variable {
-                name: "test".to_string(),
-                kind: VariableType::Named,
-                rule: Rule::seq(vec![
-                    Rule::non_terminal(0),
-                    Rule::non_terminal(1),
-                    Rule::non_terminal(2),
-                ]),
-            }],
-        });
-
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Rule `test` cannot be inlined because it contains a reference to itself.",
-        );
-    }
-}

cli/generate/src/prepare_grammar/intern_symbols.rs

@@ -1,256 +0,0 @@
-use anyhow::{anyhow, Result};
-
-use super::InternedGrammar;
-use crate::{
-    grammars::{InputGrammar, Variable, VariableType},
-    rules::{Rule, Symbol},
-};
-
-pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result<InternedGrammar> {
-    let interner = Interner { grammar };
-
-    if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden {
-        return Err(anyhow!("A grammar's start rule must be visible."));
-    }
-
-    let mut variables = Vec::with_capacity(grammar.variables.len());
-    for variable in &grammar.variables {
-        variables.push(Variable {
-            name: variable.name.clone(),
-            kind: variable_type_for_name(&variable.name),
-            rule: interner.intern_rule(&variable.rule, Some(&variable.name))?,
-        });
-    }
-
-    let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len());
-    for external_token in &grammar.external_tokens {
-        let rule = interner.intern_rule(external_token, None)?;
-        let (name, kind) = if let Rule::NamedSymbol(name) = external_token {
-            (name.clone(), variable_type_for_name(name))
-        } else {
-            (String::new(), VariableType::Anonymous)
-        };
-        external_tokens.push(Variable { name, kind, rule });
-    }
-
-    let mut extra_symbols = Vec::with_capacity(grammar.extra_symbols.len());
-    for extra_token in &grammar.extra_symbols {
-        extra_symbols.push(interner.intern_rule(extra_token, None)?);
-    }
-
-    let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len());
-    for supertype_symbol_name in &grammar.supertype_symbols {
-        supertype_symbols.push(interner.intern_name(supertype_symbol_name).ok_or_else(|| {
-            anyhow!("Undefined symbol `{supertype_symbol_name}` in grammar's supertypes array")
-        })?);
-    }
-
-    let mut expected_conflicts = Vec::new();
-    for conflict in &grammar.expected_conflicts {
-        let mut interned_conflict = Vec::with_capacity(conflict.len());
-        for name in conflict {
-            interned_conflict.push(interner.intern_name(name).ok_or_else(|| {
-                anyhow!("Undefined symbol `{name}` in grammar's conflicts array")
-            })?);
-        }
-        expected_conflicts.push(interned_conflict);
-    }
-
-    let mut variables_to_inline = Vec::new();
-    for name in &grammar.variables_to_inline {
-        if let Some(symbol) = interner.intern_name(name) {
-            variables_to_inline.push(symbol);
-        }
-    }
-
-    let mut word_token = None;
-    if let Some(name) = grammar.word_token.as_ref() {
-        word_token = Some(
-            interner
-                .intern_name(name)
-                .ok_or_else(|| anyhow!("Undefined symbol `{name}` as grammar's word token"))?,
-        );
-    }
-
-    for (i, variable) in variables.iter_mut().enumerate() {
-        if supertype_symbols.contains(&Symbol::non_terminal(i)) {
-            variable.kind = VariableType::Hidden;
-        }
-    }
-
-    Ok(InternedGrammar {
-        variables,
-        external_tokens,
-        extra_symbols,
-        expected_conflicts,
-        variables_to_inline,
-        supertype_symbols,
-        word_token,
-        precedence_orderings: grammar.precedence_orderings.clone(),
-    })
-}
-
-struct Interner<'a> {
-    grammar: &'a InputGrammar,
-}
-
-impl<'a> Interner<'a> {
-    fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> Result<Rule> {
-        match rule {
-            Rule::Choice(elements) => {
-                self.check_single(elements, name);
-                let mut result = Vec::with_capacity(elements.len());
-                for element in elements {
-                    result.push(self.intern_rule(element, name)?);
-                }
-                Ok(Rule::Choice(result))
-            }
-            Rule::Seq(elements) => {
-                self.check_single(elements, name);
-                let mut result = Vec::with_capacity(elements.len());
-                for element in elements {
-                    result.push(self.intern_rule(element, name)?);
-                }
-                Ok(Rule::Seq(result))
-            }
-            Rule::Repeat(content) => Ok(Rule::Repeat(Box::new(self.intern_rule(content, name)?))),
-            Rule::Metadata { rule, params } => Ok(Rule::Metadata {
-                rule: Box::new(self.intern_rule(rule, name)?),
-                params: params.clone(),
-            }),
-            Rule::NamedSymbol(name) => self.intern_name(name).map_or_else(
-                || Err(anyhow!("Undefined symbol `{name}`")),
-                |symbol| Ok(Rule::Symbol(symbol)),
-            ),
-            _ => Ok(rule.clone()),
-        }
-    }
-
-    fn intern_name(&self, symbol: &str) -> Option<Symbol> {
-        for (i, variable) in self.grammar.variables.iter().enumerate() {
-            if variable.name == symbol {
-                return Some(Symbol::non_terminal(i));
-            }
-        }
-
-        for (i, external_token) in self.grammar.external_tokens.iter().enumerate() {
-            if let Rule::NamedSymbol(name) = external_token {
-                if name == symbol {
-                    return Some(Symbol::external(i));
-                }
-            }
-        }
-
-        None
-    }
-
-    // In the case of a seq or choice rule of 1 element in a hidden rule, weird
-    // inconsistent behavior with queries can occur. So we should warn the user about it.
-    fn check_single(&self, elements: &[Rule], name: Option<&str>) {
-        if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) {
-            eprintln!(
-                "Warning: rule {} is just a `seq` or `choice` rule with a single element. This is unnecessary.",
-                name.unwrap_or_default()
-            );
-        }
-    }
-}
-
-fn variable_type_for_name(name: &str) -> VariableType {
-    if name.starts_with('_') {
-        VariableType::Hidden
-    } else {
-        VariableType::Named
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_basic_repeat_expansion() {
-        let grammar = intern_symbols(&build_grammar(vec![
-            Variable::named("x", Rule::choice(vec![Rule::named("y"), Rule::named("_z")])),
-            Variable::named("y", Rule::named("_z")),
-            Variable::named("_z", Rule::string("a")),
-        ]))
-        .unwrap();
-
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "x",
-                    Rule::choice(vec![Rule::non_terminal(1), Rule::non_terminal(2),])
-                ),
-                Variable::named("y", Rule::non_terminal(2)),
-                Variable::hidden("_z", Rule::string("a")),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_interning_external_token_names() {
-        // Variable `y` is both an internal and an external token.
-        // Variable `z` is just an external token.
-        let mut input_grammar = build_grammar(vec![
-            Variable::named(
-                "w",
-                Rule::choice(vec![Rule::named("x"), Rule::named("y"), Rule::named("z")]),
-            ),
-            Variable::named("x", Rule::string("a")),
-            Variable::named("y", Rule::string("b")),
-        ]);
-        input_grammar
-            .external_tokens
-            .extend(vec![Rule::named("y"), Rule::named("z")]);
-
-        let grammar = intern_symbols(&input_grammar).unwrap();
-
-        // Variable `y` is referred to by its internal index.
-        // Variable `z` is referred to by its external index.
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable::named(
-                    "w",
-                    Rule::choice(vec![
-                        Rule::non_terminal(1),
-                        Rule::non_terminal(2),
-                        Rule::external(1),
-                    ])
-                ),
-                Variable::named("x", Rule::string("a")),
-                Variable::named("y", Rule::string("b")),
-            ]
-        );
-
-        // The external token for `y` refers back to its internal index.
-        assert_eq!(
-            grammar.external_tokens,
-            vec![
-                Variable::named("y", Rule::non_terminal(2)),
-                Variable::named("z", Rule::external(1)),
-            ]
-        );
-    }
-
-    #[test]
-    fn test_grammar_with_undefined_symbols() {
-        let result = intern_symbols(&build_grammar(vec![Variable::named("x", Rule::named("y"))]));
-
-        match result {
-            Err(e) => assert_eq!(e.to_string(), "Undefined symbol `y`"),
-            _ => panic!("Expected an error but got none"),
-        }
-    }
-
-    fn build_grammar(variables: Vec<Variable>) -> InputGrammar {
-        InputGrammar {
-            variables,
-            name: "the_language".to_string(),
-            ..Default::default()
-        }
-    }
-}

cli/generate/src/prepare_grammar/mod.rs

@@ -1,251 +0,0 @@
-mod expand_repeats;
-mod expand_tokens;
-mod extract_default_aliases;
-mod extract_tokens;
-mod flatten_grammar;
-mod intern_symbols;
-mod process_inlines;
-
-use std::{
-    cmp::Ordering,
-    collections::{hash_map, HashMap, HashSet},
-    mem,
-};
-
-use anyhow::{anyhow, Result};
-
-pub use self::expand_tokens::expand_tokens;
-use self::{
-    expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
-    extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
-    intern_symbols::intern_symbols, process_inlines::process_inlines,
-};
-use super::{
-    grammars::{
-        ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
-        SyntaxGrammar, Variable,
-    },
-    rules::{AliasMap, Precedence, Rule, Symbol},
-};
-
-pub struct IntermediateGrammar<T, U> {
-    variables: Vec<Variable>,
-    extra_symbols: Vec<T>,
-    expected_conflicts: Vec<Vec<Symbol>>,
-    precedence_orderings: Vec<Vec<PrecedenceEntry>>,
-    external_tokens: Vec<U>,
-    variables_to_inline: Vec<Symbol>,
-    supertype_symbols: Vec<Symbol>,
-    word_token: Option<Symbol>,
-}
-
-pub type InternedGrammar = IntermediateGrammar<Rule, Variable>;
-
-pub type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct ExtractedLexicalGrammar {
-    pub variables: Vec<Variable>,
-    pub separators: Vec<Rule>,
-}
-
-impl<T, U> Default for IntermediateGrammar<T, U> {
-    fn default() -> Self {
-        Self {
-            variables: Vec::default(),
-            extra_symbols: Vec::default(),
-            expected_conflicts: Vec::default(),
-            precedence_orderings: Vec::default(),
-            external_tokens: Vec::default(),
-            variables_to_inline: Vec::default(),
-            supertype_symbols: Vec::default(),
-            word_token: Option::default(),
-        }
-    }
-}
-
-/// Transform an input grammar into separate components that are ready
-/// for parse table construction.
-pub fn prepare_grammar(
-    input_grammar: &InputGrammar,
-) -> Result<(
-    SyntaxGrammar,
-    LexicalGrammar,
-    InlinedProductionMap,
-    AliasMap,
-)> {
-    validate_precedences(input_grammar)?;
-
-    let interned_grammar = intern_symbols(input_grammar)?;
-    let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
-    let syntax_grammar = expand_repeats(syntax_grammar);
-    let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
-    let lexical_grammar = expand_tokens(lexical_grammar)?;
-    let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
-    let inlines = process_inlines(&syntax_grammar, &lexical_grammar)?;
-    Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
-}
-
-/// Check that all of the named precedences used in the grammar are declared
-/// within the `precedences` lists, and also that there are no conflicting
-/// precedence orderings declared in those lists.
-fn validate_precedences(grammar: &InputGrammar) -> Result<()> {
-    // Check that no rule contains a named precedence that is not present in
-    // any of the `precedences` lists.
-    fn validate(rule_name: &str, rule: &Rule, names: &HashSet<&String>) -> Result<()> {
-        match rule {
-            Rule::Repeat(rule) => validate(rule_name, rule, names),
-            Rule::Seq(elements) | Rule::Choice(elements) => elements
-                .iter()
-                .try_for_each(|e| validate(rule_name, e, names)),
-            Rule::Metadata { rule, params } => {
-                if let Precedence::Name(n) = &params.precedence {
-                    if !names.contains(n) {
-                        return Err(anyhow!("Undeclared precedence '{n}' in rule '{rule_name}'"));
-                    }
-                }
-                validate(rule_name, rule, names)?;
-                Ok(())
-            }
-            _ => Ok(()),
-        }
-    }
-
-    // For any two precedence names `a` and `b`, if `a` comes before `b`
-    // in some list, then it cannot come *after* `b` in any list.
-    let mut pairs = HashMap::new();
-    for list in &grammar.precedence_orderings {
-        for (i, mut entry1) in list.iter().enumerate() {
-            for mut entry2 in list.iter().skip(i + 1) {
-                if entry2 == entry1 {
-                    continue;
-                }
-                let mut ordering = Ordering::Greater;
-                if entry1 > entry2 {
-                    ordering = Ordering::Less;
-                    mem::swap(&mut entry1, &mut entry2);
-                }
-                match pairs.entry((entry1, entry2)) {
-                    hash_map::Entry::Vacant(e) => {
-                        e.insert(ordering);
-                    }
-                    hash_map::Entry::Occupied(e) => {
-                        if e.get() != &ordering {
-                            return Err(anyhow!(
-                                "Conflicting orderings for precedences {entry1} and {entry2}",
-                            ));
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    let precedence_names = grammar
-        .precedence_orderings
-        .iter()
-        .flat_map(|l| l.iter())
-        .filter_map(|p| {
-            if let PrecedenceEntry::Name(n) = p {
-                Some(n)
-            } else {
-                None
-            }
-        })
-        .collect::<HashSet<&String>>();
-    for variable in &grammar.variables {
-        validate(&variable.name, &variable.rule, &precedence_names)?;
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::grammars::VariableType;
-
-    #[test]
-    fn test_validate_precedences_with_undeclared_precedence() {
-        let grammar = InputGrammar {
-            precedence_orderings: vec![
-                vec![
-                    PrecedenceEntry::Name("a".to_string()),
-                    PrecedenceEntry::Name("b".to_string()),
-                ],
-                vec![
-                    PrecedenceEntry::Name("b".to_string()),
-                    PrecedenceEntry::Name("c".to_string()),
-                    PrecedenceEntry::Name("d".to_string()),
-                ],
-            ],
-            variables: vec![
-                Variable {
-                    name: "v1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::Seq(vec![
-                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
-                    ]),
-                },
-                Variable {
-                    name: "v2".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::Choice(vec![
-                        Rule::prec_left(Precedence::Name("omg".to_string()), Rule::string("y")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
-                    ])),
-                },
-            ],
-            ..Default::default()
-        };
-
-        let result = validate_precedences(&grammar);
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Undeclared precedence 'omg' in rule 'v2'",
-        );
-    }
-
-    #[test]
-    fn test_validate_precedences_with_conflicting_order() {
-        let grammar = InputGrammar {
-            precedence_orderings: vec![
-                vec![
-                    PrecedenceEntry::Name("a".to_string()),
-                    PrecedenceEntry::Name("b".to_string()),
-                ],
-                vec![
-                    PrecedenceEntry::Name("b".to_string()),
-                    PrecedenceEntry::Name("c".to_string()),
-                    PrecedenceEntry::Name("a".to_string()),
-                ],
-            ],
-            variables: vec![
-                Variable {
-                    name: "v1".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::Seq(vec![
-                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
-                    ]),
-                },
-                Variable {
-                    name: "v2".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::Choice(vec![
-                        Rule::prec_left(Precedence::Name("a".to_string()), Rule::string("y")),
-                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
-                    ])),
-                },
-            ],
-            ..Default::default()
-        };
-
-        let result = validate_precedences(&grammar);
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Conflicting orderings for precedences 'a' and 'b'",
-        );
-    }
-}

cli/generate/src/prepare_grammar/process_inlines.rs

@@ -1,547 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::{anyhow, Result};
-
-use crate::{
-    grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
-    rules::SymbolType,
-};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-struct ProductionStepId {
-    // A `None` value here means that the production itself was produced via inlining,
-    // and is stored in the builder's `productions` vector, as opposed to being
-    // stored in one of the grammar's variables.
-    variable_index: Option<usize>,
-    production_index: usize,
-    step_index: usize,
-}
-
-struct InlinedProductionMapBuilder {
-    production_indices_by_step_id: HashMap<ProductionStepId, Vec<usize>>,
-    productions: Vec<Production>,
-}
-
-impl InlinedProductionMapBuilder {
-    fn build(mut self, grammar: &SyntaxGrammar) -> InlinedProductionMap {
-        let mut step_ids_to_process = Vec::new();
-        for (variable_index, variable) in grammar.variables.iter().enumerate() {
-            for production_index in 0..variable.productions.len() {
-                step_ids_to_process.push(ProductionStepId {
-                    variable_index: Some(variable_index),
-                    production_index,
-                    step_index: 0,
-                });
-                while !step_ids_to_process.is_empty() {
-                    let mut i = 0;
-                    while i < step_ids_to_process.len() {
-                        let step_id = step_ids_to_process[i];
-                        if let Some(step) = self.production_step_for_id(step_id, grammar) {
-                            if grammar.variables_to_inline.contains(&step.symbol) {
-                                let inlined_step_ids = self
-                                    .inline_production_at_step(step_id, grammar)
-                                    .iter()
-                                    .copied()
-                                    .map(|production_index| ProductionStepId {
-                                        variable_index: None,
-                                        production_index,
-                                        step_index: step_id.step_index,
-                                    });
-                                step_ids_to_process.splice(i..=i, inlined_step_ids);
-                            } else {
-                                step_ids_to_process[i] = ProductionStepId {
-                                    variable_index: step_id.variable_index,
-                                    production_index: step_id.production_index,
-                                    step_index: step_id.step_index + 1,
-                                };
-                                i += 1;
-                            }
-                        } else {
-                            step_ids_to_process.remove(i);
-                        }
-                    }
-                }
-            }
-        }
-
-        let productions = self.productions;
-        let production_indices_by_step_id = self.production_indices_by_step_id;
-        let production_map = production_indices_by_step_id
-            .into_iter()
-            .map(|(step_id, production_indices)| {
-                let production = step_id.variable_index.map_or_else(
-                    || &productions[step_id.production_index],
-                    |variable_index| {
-                        &grammar.variables[variable_index].productions[step_id.production_index]
-                    },
-                ) as *const Production;
-                ((production, step_id.step_index as u32), production_indices)
-            })
-            .collect();
-
-        InlinedProductionMap {
-            productions,
-            production_map,
-        }
-    }
-
-    fn inline_production_at_step<'a>(
-        &'a mut self,
-        step_id: ProductionStepId,
-        grammar: &'a SyntaxGrammar,
-    ) -> &'a [usize] {
-        // Build a list of productions produced by inlining rules.
-        let mut i = 0;
-        let step_index = step_id.step_index;
-        let mut productions_to_add = vec![self.production_for_id(step_id, grammar).clone()];
-        while i < productions_to_add.len() {
-            if let Some(step) = productions_to_add[i].steps.get(step_index) {
-                let symbol = step.symbol;
-                if grammar.variables_to_inline.contains(&symbol) {
-                    // Remove the production from the vector, replacing it with a placeholder.
-                    let production = productions_to_add
-                        .splice(i..=i, std::iter::once(&Production::default()).cloned())
-                        .next()
-                        .unwrap();
-
-                    // Replace the placeholder with the inlined productions.
-                    productions_to_add.splice(
-                        i..=i,
-                        grammar.variables[symbol.index].productions.iter().map(|p| {
-                            let mut production = production.clone();
-                            let removed_step = production
-                                .steps
-                                .splice(step_index..=step_index, p.steps.iter().cloned())
-                                .next()
-                                .unwrap();
-                            let inserted_steps =
-                                &mut production.steps[step_index..(step_index + p.steps.len())];
-                            if let Some(alias) = removed_step.alias {
-                                for inserted_step in inserted_steps.iter_mut() {
-                                    inserted_step.alias = Some(alias.clone());
-                                }
-                            }
-                            if let Some(field_name) = removed_step.field_name {
-                                for inserted_step in inserted_steps.iter_mut() {
-                                    inserted_step.field_name = Some(field_name.clone());
-                                }
-                            }
-                            if let Some(last_inserted_step) = inserted_steps.last_mut() {
-                                if last_inserted_step.precedence.is_none() {
-                                    last_inserted_step.precedence = removed_step.precedence;
-                                }
-                                if last_inserted_step.associativity.is_none() {
-                                    last_inserted_step.associativity = removed_step.associativity;
-                                }
-                            }
-                            if p.dynamic_precedence.abs() > production.dynamic_precedence.abs() {
-                                production.dynamic_precedence = p.dynamic_precedence;
-                            }
-                            production
-                        }),
-                    );
-
-                    continue;
-                }
-            }
-            i += 1;
-        }
-
-        // Store all the computed productions.
-        let result = productions_to_add
-            .into_iter()
-            .map(|production| {
-                self.productions
-                    .iter()
-                    .position(|p| *p == production)
-                    .unwrap_or_else(|| {
-                        self.productions.push(production);
-                        self.productions.len() - 1
-                    })
-            })
-            .collect();
-
-        // Cache these productions based on the original production step.
-        self.production_indices_by_step_id
-            .entry(step_id)
-            .or_insert(result)
-    }
-
-    fn production_for_id<'a>(
-        &'a self,
-        id: ProductionStepId,
-        grammar: &'a SyntaxGrammar,
-    ) -> &'a Production {
-        id.variable_index.map_or_else(
-            || &self.productions[id.production_index],
-            |variable_index| &grammar.variables[variable_index].productions[id.production_index],
-        )
-    }
-
-    fn production_step_for_id<'a>(
-        &'a self,
-        id: ProductionStepId,
-        grammar: &'a SyntaxGrammar,
-    ) -> Option<&'a ProductionStep> {
-        self.production_for_id(id, grammar).steps.get(id.step_index)
-    }
-}
-
-pub(super) fn process_inlines(
-    grammar: &SyntaxGrammar,
-    lexical_grammar: &LexicalGrammar,
-) -> Result<InlinedProductionMap> {
-    for symbol in &grammar.variables_to_inline {
-        match symbol.kind {
-            SymbolType::External => {
-                return Err(anyhow!(
-                    "External token `{}` cannot be inlined",
-                    grammar.external_tokens[symbol.index].name
-                ))
-            }
-            SymbolType::Terminal => {
-                return Err(anyhow!(
-                    "Token `{}` cannot be inlined",
-                    lexical_grammar.variables[symbol.index].name,
-                ))
-            }
-            SymbolType::NonTerminal if symbol.index == 0 => {
-                return Err(anyhow!(
-                    "Rule `{}` cannot be inlined because it is the first rule",
-                    grammar.variables[symbol.index].name,
-                ))
-            }
-            _ => {}
-        }
-    }
-
-    Ok(InlinedProductionMapBuilder {
-        productions: Vec::new(),
-        production_indices_by_step_id: HashMap::new(),
-    }
-    .build(grammar))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        grammars::{LexicalVariable, SyntaxVariable, VariableType},
-        rules::{Associativity, Precedence, Symbol},
-    };
-
-    #[test]
-    fn test_basic_inlining() {
-        let grammar = SyntaxGrammar {
-            variables_to_inline: vec![Symbol::non_terminal(1)],
-            variables: vec![
-                SyntaxVariable {
-                    name: "non-terminal-0".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(10)),
-                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
-                            ProductionStep::new(Symbol::terminal(11)),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-1".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![
-                        Production {
-                            dynamic_precedence: 0,
-                            steps: vec![
-                                ProductionStep::new(Symbol::terminal(12)),
-                                ProductionStep::new(Symbol::terminal(13)),
-                            ],
-                        },
-                        Production {
-                            dynamic_precedence: -2,
-                            steps: vec![ProductionStep::new(Symbol::terminal(14))],
-                        },
-                    ],
-                },
-            ],
-            ..Default::default()
-        };
-
-        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
-
-        // Nothing to inline at step 0.
-        assert!(inline_map
-            .inlined_productions(&grammar.variables[0].productions[0], 0)
-            .is_none());
-
-        // Inlining variable 1 yields two productions.
-        assert_eq!(
-            inline_map
-                .inlined_productions(&grammar.variables[0].productions[0], 1)
-                .unwrap()
-                .cloned()
-                .collect::<Vec<_>>(),
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(12)),
-                        ProductionStep::new(Symbol::terminal(13)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                    ],
-                },
-                Production {
-                    dynamic_precedence: -2,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(14)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                    ],
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn test_nested_inlining() {
-        let grammar = SyntaxGrammar {
-            variables: vec![
-                SyntaxVariable {
-                    name: "non-terminal-0".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(10)),
-                            ProductionStep::new(Symbol::non_terminal(1)), // inlined
-                            ProductionStep::new(Symbol::terminal(11)),
-                            ProductionStep::new(Symbol::non_terminal(2)), // inlined
-                            ProductionStep::new(Symbol::terminal(12)),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-1".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![
-                        Production {
-                            dynamic_precedence: 0,
-                            steps: vec![ProductionStep::new(Symbol::terminal(13))],
-                        },
-                        Production {
-                            dynamic_precedence: 0,
-                            steps: vec![
-                                ProductionStep::new(Symbol::non_terminal(3)), // inlined
-                                ProductionStep::new(Symbol::terminal(14)),
-                            ],
-                        },
-                    ],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-2".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![ProductionStep::new(Symbol::terminal(15))],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-3".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![ProductionStep::new(Symbol::terminal(16))],
-                    }],
-                },
-            ],
-            variables_to_inline: vec![
-                Symbol::non_terminal(1),
-                Symbol::non_terminal(2),
-                Symbol::non_terminal(3),
-            ],
-            ..Default::default()
-        };
-
-        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
-
-        let productions = inline_map
-            .inlined_productions(&grammar.variables[0].productions[0], 1)
-            .unwrap()
-            .collect::<Vec<_>>();
-
-        assert_eq!(
-            productions.iter().copied().cloned().collect::<Vec<_>>(),
-            vec![
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(13)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::terminal(12)),
-                    ],
-                },
-                Production {
-                    dynamic_precedence: 0,
-                    steps: vec![
-                        ProductionStep::new(Symbol::terminal(10)),
-                        ProductionStep::new(Symbol::terminal(16)),
-                        ProductionStep::new(Symbol::terminal(14)),
-                        ProductionStep::new(Symbol::terminal(11)),
-                        ProductionStep::new(Symbol::non_terminal(2)),
-                        ProductionStep::new(Symbol::terminal(12)),
-                    ],
-                },
-            ]
-        );
-
-        assert_eq!(
-            inline_map
-                .inlined_productions(productions[0], 3)
-                .unwrap()
-                .cloned()
-                .collect::<Vec<_>>(),
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    ProductionStep::new(Symbol::terminal(10)),
-                    ProductionStep::new(Symbol::terminal(13)),
-                    ProductionStep::new(Symbol::terminal(11)),
-                    ProductionStep::new(Symbol::terminal(15)),
-                    ProductionStep::new(Symbol::terminal(12)),
-                ],
-            },]
-        );
-    }
-
-    #[test]
-    fn test_inlining_with_precedence_and_alias() {
-        let grammar = SyntaxGrammar {
-            variables_to_inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
-            variables: vec![
-                SyntaxVariable {
-                    name: "non-terminal-0".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            // inlined
-                            ProductionStep::new(Symbol::non_terminal(1))
-                                .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
-                            ProductionStep::new(Symbol::terminal(10)),
-                            // inlined
-                            ProductionStep::new(Symbol::non_terminal(2))
-                                .with_alias("outer_alias", true),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-1".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![
-                            ProductionStep::new(Symbol::terminal(11))
-                                .with_prec(Precedence::Integer(2), None)
-                                .with_alias("inner_alias", true),
-                            ProductionStep::new(Symbol::terminal(12)),
-                        ],
-                    }],
-                },
-                SyntaxVariable {
-                    name: "non-terminal-2".to_string(),
-                    kind: VariableType::Named,
-                    productions: vec![Production {
-                        dynamic_precedence: 0,
-                        steps: vec![ProductionStep::new(Symbol::terminal(13))],
-                    }],
-                },
-            ],
-            ..Default::default()
-        };
-
-        let inline_map = process_inlines(&grammar, &LexicalGrammar::default()).unwrap();
-
-        let productions = inline_map
-            .inlined_productions(&grammar.variables[0].productions[0], 0)
-            .unwrap()
-            .collect::<Vec<_>>();
-
-        assert_eq!(
-            productions.iter().copied().cloned().collect::<Vec<_>>(),
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    // The first step in the inlined production retains its precedence
-                    // and alias.
-                    ProductionStep::new(Symbol::terminal(11))
-                        .with_prec(Precedence::Integer(2), None)
-                        .with_alias("inner_alias", true),
-                    // The final step of the inlined production inherits the precedence of
-                    // the inlined step.
-                    ProductionStep::new(Symbol::terminal(12))
-                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
-                    ProductionStep::new(Symbol::terminal(10)),
-                    ProductionStep::new(Symbol::non_terminal(2)).with_alias("outer_alias", true),
-                ]
-            }],
-        );
-
-        assert_eq!(
-            inline_map
-                .inlined_productions(productions[0], 3)
-                .unwrap()
-                .cloned()
-                .collect::<Vec<_>>(),
-            vec![Production {
-                dynamic_precedence: 0,
-                steps: vec![
-                    ProductionStep::new(Symbol::terminal(11))
-                        .with_prec(Precedence::Integer(2), None)
-                        .with_alias("inner_alias", true),
-                    ProductionStep::new(Symbol::terminal(12))
-                        .with_prec(Precedence::Integer(1), Some(Associativity::Left)),
-                    ProductionStep::new(Symbol::terminal(10)),
-                    // All steps of the inlined production inherit their alias from the
-                    // inlined step.
-                    ProductionStep::new(Symbol::terminal(13)).with_alias("outer_alias", true),
-                ]
-            }],
-        );
-    }
-
-    #[test]
-    fn test_error_when_inlining_tokens() {
-        let lexical_grammar = LexicalGrammar {
-            variables: vec![LexicalVariable {
-                name: "something".to_string(),
-                kind: VariableType::Named,
-                implicit_precedence: 0,
-                start_state: 0,
-            }],
-            ..Default::default()
-        };
-
-        let grammar = SyntaxGrammar {
-            variables_to_inline: vec![Symbol::terminal(0)],
-            variables: vec![SyntaxVariable {
-                name: "non-terminal-0".to_string(),
-                kind: VariableType::Named,
-                productions: vec![Production {
-                    dynamic_precedence: 0,
-                    steps: vec![ProductionStep::new(Symbol::terminal(0))],
-                }],
-            }],
-            ..Default::default()
-        };
-
-        if let Err(error) = process_inlines(&grammar, &lexical_grammar) {
-            assert_eq!(error.to_string(), "Token `something` cannot be inlined");
-        } else {
-            panic!("expected an error, but got none");
-        }
-    }
-}

cli/generate/src/prepare_grammar/unicode-category-aliases.json

@@ -1 +0,0 @@
-{"Other":"C","Control":"Cc","cntrl":"Cc","Format":"Cf","Unassigned":"Cn","Private_Use":"Co","Surrogate":"Cs","Letter":"L","Cased_Letter":"LC","Lowercase_Letter":"Ll","Modifier_Letter":"Lm","Other_Letter":"Lo","Titlecase_Letter":"Lt","Uppercase_Letter":"Lu","Mark":"M","Combining_Mark":"M","Spacing_Mark":"Mc","Enclosing_Mark":"Me","Nonspacing_Mark":"Mn","Number":"N","Decimal_Number":"Nd","digit":"Nd","Letter_Number":"Nl","Other_Number":"No","Punctuation":"P","punct":"P","Connector_Punctuation":"Pc","Dash_Punctuation":"Pd","Close_Punctuation":"Pe","Final_Punctuation":"Pf","Initial_Punctuation":"Pi","Other_Punctuation":"Po","Open_Punctuation":"Ps","Symbol":"S","Currency_Symbol":"Sc","Modifier_Symbol":"Sk","Math_Symbol":"Sm","Other_Symbol":"So","Separator":"Z","Line_Separator":"Zl","Paragraph_Separator":"Zp","Space_Separator":"Zs"}

cli/generate/src/prepare_grammar/unicode-property-aliases.json

@@ -1 +0,0 @@
-{"cjkAccountingNumeric":"kAccountingNumeric","cjkOtherNumeric":"kOtherNumeric","cjkPrimaryNumeric":"kPrimaryNumeric","nv":"Numeric_Value","bmg":"Bidi_Mirroring_Glyph","bpb":"Bidi_Paired_Bracket","cf":"Case_Folding","cjkCompatibilityVariant":"kCompatibilityVariant","dm":"Decomposition_Mapping","EqUIdeo":"Equivalent_Unified_Ideograph","FC_NFKC":"FC_NFKC_Closure","lc":"Lowercase_Mapping","NFKC_CF":"NFKC_Casefold","NFKC_SCF":"NFKC_Simple_Casefold","scf":"Simple_Case_Folding","sfc":"Simple_Case_Folding","slc":"Simple_Lowercase_Mapping","stc":"Simple_Titlecase_Mapping","suc":"Simple_Uppercase_Mapping","tc":"Titlecase_Mapping","uc":"Uppercase_Mapping","cjkIICore":"kIICore","cjkIRG_GSource":"kIRG_GSource","cjkIRG_HSource":"kIRG_HSource","cjkIRG_JSource":"kIRG_JSource","cjkIRG_KPSource":"kIRG_KPSource","cjkIRG_KSource":"kIRG_KSource","cjkIRG_MSource":"kIRG_MSource","cjkIRG_SSource":"kIRG_SSource","cjkIRG_TSource":"kIRG_TSource","cjkIRG_UKSource":"kIRG_UKSource","cjkIRG_USource":"kIRG_USource","cjkIRG_VSource":"kIRG_VSource","cjkRSUnicode":"kRSUnicode","Unicode_Radical_Stroke":"kRSUnicode","URS":"kRSUnicode","isc":"ISO_Comment","JSN":"Jamo_Short_Name","na":"Name","na1":"Unicode_1_Name","Name_Alias":"Name_Alias","scx":"Script_Extensions","age":"Age","blk":"Block","sc":"Script","bc":"Bidi_Class","bpt":"Bidi_Paired_Bracket_Type","ccc":"Canonical_Combining_Class","dt":"Decomposition_Type","ea":"East_Asian_Width","gc":"General_Category","GCB":"Grapheme_Cluster_Break","hst":"Hangul_Syllable_Type","InCB":"Indic_Conjunct_Break","InPC":"Indic_Positional_Category","InSC":"Indic_Syllabic_Category","jg":"Joining_Group","jt":"Joining_Type","lb":"Line_Break","NFC_QC":"NFC_Quick_Check","NFD_QC":"NFD_Quick_Check","NFKC_QC":"NFKC_Quick_Check","NFKD_QC":"NFKD_Quick_Check","nt":"Numeric_Type","SB":"Sentence_Break","vo":"Vertical_Orientation","WB":"Word_Break","AHex":"ASCII_Hex_Digit","Alpha":"Alphabetic","Bidi_C":"Bidi_Control","Bidi_M":"Bidi_Mirrored","Cased":"Cased","CE":"Composition_Exclusion","CI":"Case_Ignorable","Comp_Ex":"Full_Composition_Exclusion","CWCF":"Changes_When_Casefolded","CWCM":"Changes_When_Casemapped","CWKCF":"Changes_When_NFKC_Casefolded","CWL":"Changes_When_Lowercased","CWT":"Changes_When_Titlecased","CWU":"Changes_When_Uppercased","Dash":"Dash","Dep":"Deprecated","DI":"Default_Ignorable_Code_Point","Dia":"Diacritic","EBase":"Emoji_Modifier_Base","EComp":"Emoji_Component","EMod":"Emoji_Modifier","Emoji":"Emoji","EPres":"Emoji_Presentation","Ext":"Extender","ExtPict":"Extended_Pictographic","Gr_Base":"Grapheme_Base","Gr_Ext":"Grapheme_Extend","Gr_Link":"Grapheme_Link","Hex":"Hex_Digit","Hyphen":"Hyphen","ID_Compat_Math_Continue":"ID_Compat_Math_Continue","ID_Compat_Math_Start":"ID_Compat_Math_Start","IDC":"ID_Continue","Ideo":"Ideographic","IDS":"ID_Start","IDSB":"IDS_Binary_Operator","IDST":"IDS_Trinary_Operator","IDSU":"IDS_Unary_Operator","Join_C":"Join_Control","LOE":"Logical_Order_Exception","Lower":"Lowercase","Math":"Math","NChar":"Noncharacter_Code_Point","OAlpha":"Other_Alphabetic","ODI":"Other_Default_Ignorable_Code_Point","OGr_Ext":"Other_Grapheme_Extend","OIDC":"Other_ID_Continue","OIDS":"Other_ID_Start","OLower":"Other_Lowercase","OMath":"Other_Math","OUpper":"Other_Uppercase","Pat_Syn":"Pattern_Syntax","Pat_WS":"Pattern_White_Space","PCM":"Prepended_Concatenation_Mark","QMark":"Quotation_Mark","Radical":"Radical","RI":"Regional_Indicator","SD":"Soft_Dotted","STerm":"Sentence_Terminal","Term":"Terminal_Punctuation","UIdeo":"Unified_Ideograph","Upper":"Uppercase","VS":"Variation_Selector","WSpace":"White_Space","space":"White_Space","XIDC":"XID_Continue","XIDS":"XID_Start","XO_NFC":"Expands_On_NFC","XO_NFD":"Expands_On_NFD","XO_NFKC":"Expands_On_NFKC","XO_NFKD":"Expands_On_NFKD"}

cli/generate/src/rules.rs

@@ -1,493 +0,0 @@
-use std::{collections::HashMap, fmt};
-
-use smallbitvec::SmallBitVec;
-
-use super::grammars::VariableType;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub enum SymbolType {
-    External,
-    End,
-    EndOfNonTerminalExtra,
-    Terminal,
-    NonTerminal,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub enum Associativity {
-    Left,
-    Right,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct Alias {
-    pub value: String,
-    pub is_named: bool,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default)]
-pub enum Precedence {
-    #[default]
-    None,
-    Integer(i32),
-    Name(String),
-}
-
-pub type AliasMap = HashMap<Symbol, Alias>;
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
-pub struct MetadataParams {
-    pub precedence: Precedence,
-    pub dynamic_precedence: i32,
-    pub associativity: Option<Associativity>,
-    pub is_token: bool,
-    pub is_string: bool,
-    pub is_active: bool,
-    pub is_main_token: bool,
-    pub alias: Option<Alias>,
-    pub field_name: Option<String>,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct Symbol {
-    pub kind: SymbolType,
-    pub index: usize,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum Rule {
-    Blank,
-    String(String),
-    Pattern(String, String),
-    NamedSymbol(String),
-    Symbol(Symbol),
-    Choice(Vec<Rule>),
-    Metadata {
-        params: MetadataParams,
-        rule: Box<Rule>,
-    },
-    Repeat(Box<Rule>),
-    Seq(Vec<Rule>),
-}
-
-// Because tokens are represented as small (~400 max) unsigned integers,
-// sets of tokens can be efficiently represented as bit vectors with each
-// index corresponding to a token, and each value representing whether or not
-// the token is present in the set.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct TokenSet {
-    terminal_bits: SmallBitVec,
-    external_bits: SmallBitVec,
-    eof: bool,
-    end_of_nonterminal_extra: bool,
-}
-
-impl Rule {
-    pub fn field(name: String, content: Self) -> Self {
-        add_metadata(content, move |params| {
-            params.field_name = Some(name);
-        })
-    }
-
-    pub fn alias(content: Self, value: String, is_named: bool) -> Self {
-        add_metadata(content, move |params| {
-            params.alias = Some(Alias { value, is_named });
-        })
-    }
-
-    pub fn token(content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.is_token = true;
-        })
-    }
-
-    pub fn immediate_token(content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.is_token = true;
-            params.is_main_token = true;
-        })
-    }
-
-    pub fn prec(value: Precedence, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.precedence = value;
-        })
-    }
-
-    pub fn prec_left(value: Precedence, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.associativity = Some(Associativity::Left);
-            params.precedence = value;
-        })
-    }
-
-    pub fn prec_right(value: Precedence, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.associativity = Some(Associativity::Right);
-            params.precedence = value;
-        })
-    }
-
-    pub fn prec_dynamic(value: i32, content: Self) -> Self {
-        add_metadata(content, |params| {
-            params.dynamic_precedence = value;
-        })
-    }
-
-    pub fn repeat(rule: Self) -> Self {
-        Self::Repeat(Box::new(rule))
-    }
-
-    pub fn choice(rules: Vec<Self>) -> Self {
-        let mut elements = Vec::with_capacity(rules.len());
-        for rule in rules {
-            choice_helper(&mut elements, rule);
-        }
-        Self::Choice(elements)
-    }
-
-    pub const fn seq(rules: Vec<Self>) -> Self {
-        Self::Seq(rules)
-    }
-}
-
-impl Alias {
-    #[must_use]
-    pub const fn kind(&self) -> VariableType {
-        if self.is_named {
-            VariableType::Named
-        } else {
-            VariableType::Anonymous
-        }
-    }
-}
-
-impl Precedence {
-    #[must_use]
-    pub const fn is_none(&self) -> bool {
-        matches!(self, Self::None)
-    }
-}
-
-#[cfg(test)]
-impl Rule {
-    #[must_use]
-    pub const fn terminal(index: usize) -> Self {
-        Self::Symbol(Symbol::terminal(index))
-    }
-
-    #[must_use]
-    pub const fn non_terminal(index: usize) -> Self {
-        Self::Symbol(Symbol::non_terminal(index))
-    }
-
-    #[must_use]
-    pub const fn external(index: usize) -> Self {
-        Self::Symbol(Symbol::external(index))
-    }
-
-    #[must_use]
-    pub fn named(name: &'static str) -> Self {
-        Self::NamedSymbol(name.to_string())
-    }
-
-    #[must_use]
-    pub fn string(value: &'static str) -> Self {
-        Self::String(value.to_string())
-    }
-
-    #[must_use]
-    pub fn pattern(value: &'static str, flags: &'static str) -> Self {
-        Self::Pattern(value.to_string(), flags.to_string())
-    }
-}
-
-impl Symbol {
-    #[must_use]
-    pub fn is_terminal(&self) -> bool {
-        self.kind == SymbolType::Terminal
-    }
-
-    #[must_use]
-    pub fn is_non_terminal(&self) -> bool {
-        self.kind == SymbolType::NonTerminal
-    }
-
-    #[must_use]
-    pub fn is_external(&self) -> bool {
-        self.kind == SymbolType::External
-    }
-
-    #[must_use]
-    pub fn is_eof(&self) -> bool {
-        self.kind == SymbolType::End
-    }
-
-    #[must_use]
-    pub const fn non_terminal(index: usize) -> Self {
-        Self {
-            kind: SymbolType::NonTerminal,
-            index,
-        }
-    }
-
-    #[must_use]
-    pub const fn terminal(index: usize) -> Self {
-        Self {
-            kind: SymbolType::Terminal,
-            index,
-        }
-    }
-
-    #[must_use]
-    pub const fn external(index: usize) -> Self {
-        Self {
-            kind: SymbolType::External,
-            index,
-        }
-    }
-
-    #[must_use]
-    pub const fn end() -> Self {
-        Self {
-            kind: SymbolType::End,
-            index: 0,
-        }
-    }
-
-    #[must_use]
-    pub const fn end_of_nonterminal_extra() -> Self {
-        Self {
-            kind: SymbolType::EndOfNonTerminalExtra,
-            index: 0,
-        }
-    }
-}
-
-impl From<Symbol> for Rule {
-    #[must_use]
-    fn from(symbol: Symbol) -> Self {
-        Self::Symbol(symbol)
-    }
-}
-
-impl TokenSet {
-    #[must_use]
-    pub const fn new() -> Self {
-        Self {
-            terminal_bits: SmallBitVec::new(),
-            external_bits: SmallBitVec::new(),
-            eof: false,
-            end_of_nonterminal_extra: false,
-        }
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = Symbol> + '_ {
-        self.terminal_bits
-            .iter()
-            .enumerate()
-            .filter_map(|(i, value)| {
-                if value {
-                    Some(Symbol::terminal(i))
-                } else {
-                    None
-                }
-            })
-            .chain(
-                self.external_bits
-                    .iter()
-                    .enumerate()
-                    .filter_map(|(i, value)| {
-                        if value {
-                            Some(Symbol::external(i))
-                        } else {
-                            None
-                        }
-                    }),
-            )
-            .chain(if self.eof { Some(Symbol::end()) } else { None })
-            .chain(if self.end_of_nonterminal_extra {
-                Some(Symbol::end_of_nonterminal_extra())
-            } else {
-                None
-            })
-    }
-
-    pub fn terminals(&self) -> impl Iterator<Item = Symbol> + '_ {
-        self.terminal_bits
-            .iter()
-            .enumerate()
-            .filter_map(|(i, value)| {
-                if value {
-                    Some(Symbol::terminal(i))
-                } else {
-                    None
-                }
-            })
-    }
-
-    pub fn contains(&self, symbol: &Symbol) -> bool {
-        match symbol.kind {
-            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
-            SymbolType::Terminal => self.terminal_bits.get(symbol.index).unwrap_or(false),
-            SymbolType::External => self.external_bits.get(symbol.index).unwrap_or(false),
-            SymbolType::End => self.eof,
-            SymbolType::EndOfNonTerminalExtra => self.end_of_nonterminal_extra,
-        }
-    }
-
-    pub fn contains_terminal(&self, index: usize) -> bool {
-        self.terminal_bits.get(index).unwrap_or(false)
-    }
-
-    pub fn insert(&mut self, other: Symbol) {
-        let vec = match other.kind {
-            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
-            SymbolType::Terminal => &mut self.terminal_bits,
-            SymbolType::External => &mut self.external_bits,
-            SymbolType::End => {
-                self.eof = true;
-                return;
-            }
-            SymbolType::EndOfNonTerminalExtra => {
-                self.end_of_nonterminal_extra = true;
-                return;
-            }
-        };
-        if other.index >= vec.len() {
-            vec.resize(other.index + 1, false);
-        }
-        vec.set(other.index, true);
-    }
-
-    pub fn remove(&mut self, other: &Symbol) -> bool {
-        let vec = match other.kind {
-            SymbolType::NonTerminal => panic!("Cannot store non-terminals in a TokenSet"),
-            SymbolType::Terminal => &mut self.terminal_bits,
-            SymbolType::External => &mut self.external_bits,
-            SymbolType::End => {
-                return if self.eof {
-                    self.eof = false;
-                    true
-                } else {
-                    false
-                }
-            }
-            SymbolType::EndOfNonTerminalExtra => {
-                return if self.end_of_nonterminal_extra {
-                    self.end_of_nonterminal_extra = false;
-                    true
-                } else {
-                    false
-                };
-            }
-        };
-        if other.index < vec.len() && vec[other.index] {
-            vec.set(other.index, false);
-            return true;
-        }
-        false
-    }
-
-    pub fn is_empty(&self) -> bool {
-        !self.eof
-            && !self.end_of_nonterminal_extra
-            && !self.terminal_bits.iter().any(|a| a)
-            && !self.external_bits.iter().any(|a| a)
-    }
-
-    pub fn insert_all_terminals(&mut self, other: &Self) -> bool {
-        let mut result = false;
-        if other.terminal_bits.len() > self.terminal_bits.len() {
-            self.terminal_bits.resize(other.terminal_bits.len(), false);
-        }
-        for (i, element) in other.terminal_bits.iter().enumerate() {
-            if element {
-                result |= !self.terminal_bits[i];
-                self.terminal_bits.set(i, element);
-            }
-        }
-        result
-    }
-
-    fn insert_all_externals(&mut self, other: &Self) -> bool {
-        let mut result = false;
-        if other.external_bits.len() > self.external_bits.len() {
-            self.external_bits.resize(other.external_bits.len(), false);
-        }
-        for (i, element) in other.external_bits.iter().enumerate() {
-            if element {
-                result |= !self.external_bits[i];
-                self.external_bits.set(i, element);
-            }
-        }
-        result
-    }
-
-    pub fn insert_all(&mut self, other: &Self) -> bool {
-        let mut result = false;
-        if other.eof {
-            result |= !self.eof;
-            self.eof = true;
-        }
-        if other.end_of_nonterminal_extra {
-            result |= !self.end_of_nonterminal_extra;
-            self.end_of_nonterminal_extra = true;
-        }
-        result |= self.insert_all_terminals(other);
-        result |= self.insert_all_externals(other);
-        result
-    }
-}
-
-impl FromIterator<Symbol> for TokenSet {
-    fn from_iter<T: IntoIterator<Item = Symbol>>(iter: T) -> Self {
-        let mut result = Self::new();
-        for symbol in iter {
-            result.insert(symbol);
-        }
-        result
-    }
-}
-
-fn add_metadata<T: FnOnce(&mut MetadataParams)>(input: Rule, f: T) -> Rule {
-    match input {
-        Rule::Metadata { rule, mut params } if !params.is_token => {
-            f(&mut params);
-            Rule::Metadata { rule, params }
-        }
-        _ => {
-            let mut params = MetadataParams::default();
-            f(&mut params);
-            Rule::Metadata {
-                rule: Box::new(input),
-                params,
-            }
-        }
-    }
-}
-
-fn choice_helper(result: &mut Vec<Rule>, rule: Rule) {
-    match rule {
-        Rule::Choice(elements) => {
-            for element in elements {
-                choice_helper(result, element);
-            }
-        }
-        _ => {
-            if !result.contains(&rule) {
-                result.push(rule);
-            }
-        }
-    }
-}
-
-impl fmt::Display for Precedence {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Integer(i) => write!(f, "{i}"),
-            Self::Name(s) => write!(f, "'{s}'"),
-            Self::None => write!(f, "none"),
-        }
-    }
-}

cli/generate/src/tables.rs

@@ -1,166 +0,0 @@
-use std::collections::BTreeMap;
-
-use super::{
-    nfa::CharacterSet,
-    rules::{Alias, Symbol, TokenSet},
-};
-pub type ProductionInfoId = usize;
-pub type ParseStateId = usize;
-pub type LexStateId = usize;
-
-use std::hash::BuildHasherDefault;
-
-use indexmap::IndexMap;
-use rustc_hash::FxHasher;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum ParseAction {
-    Accept,
-    Shift {
-        state: ParseStateId,
-        is_repetition: bool,
-    },
-    ShiftExtra,
-    Recover,
-    Reduce {
-        symbol: Symbol,
-        child_count: usize,
-        dynamic_precedence: i32,
-        production_id: ProductionInfoId,
-    },
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum GotoAction {
-    Goto(ParseStateId),
-    ShiftExtra,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct ParseTableEntry {
-    pub actions: Vec<ParseAction>,
-    pub reusable: bool,
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct ParseState {
-    pub id: ParseStateId,
-    pub terminal_entries: IndexMap<Symbol, ParseTableEntry, BuildHasherDefault<FxHasher>>,
-    pub nonterminal_entries: IndexMap<Symbol, GotoAction, BuildHasherDefault<FxHasher>>,
-    pub lex_state_id: usize,
-    pub external_lex_state_id: usize,
-    pub core_id: usize,
-}
-
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct FieldLocation {
-    pub index: usize,
-    pub inherited: bool,
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-pub struct ProductionInfo {
-    pub alias_sequence: Vec<Option<Alias>>,
-    pub field_map: BTreeMap<String, Vec<FieldLocation>>,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct ParseTable {
-    pub states: Vec<ParseState>,
-    pub symbols: Vec<Symbol>,
-    pub production_infos: Vec<ProductionInfo>,
-    pub max_aliased_production_length: usize,
-    pub external_lex_states: Vec<TokenSet>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
-pub struct AdvanceAction {
-    pub state: LexStateId,
-    pub in_main_token: bool,
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
-pub struct LexState {
-    pub accept_action: Option<Symbol>,
-    pub eof_action: Option<AdvanceAction>,
-    pub advance_actions: Vec<(CharacterSet, AdvanceAction)>,
-}
-
-#[derive(Debug, PartialEq, Eq, Default)]
-pub struct LexTable {
-    pub states: Vec<LexState>,
-}
-
-impl ParseTableEntry {
-    #[must_use]
-    pub const fn new() -> Self {
-        Self {
-            reusable: true,
-            actions: Vec::new(),
-        }
-    }
-}
-
-impl ParseState {
-    pub fn is_end_of_non_terminal_extra(&self) -> bool {
-        self.terminal_entries
-            .contains_key(&Symbol::end_of_nonterminal_extra())
-    }
-
-    pub fn referenced_states(&self) -> impl Iterator<Item = ParseStateId> + '_ {
-        self.terminal_entries
-            .iter()
-            .flat_map(|(_, entry)| {
-                entry.actions.iter().filter_map(|action| match action {
-                    ParseAction::Shift { state, .. } => Some(*state),
-                    _ => None,
-                })
-            })
-            .chain(self.nonterminal_entries.iter().filter_map(|(_, action)| {
-                if let GotoAction::Goto(state) = action {
-                    Some(*state)
-                } else {
-                    None
-                }
-            }))
-    }
-
-    pub fn update_referenced_states<F>(&mut self, mut f: F)
-    where
-        F: FnMut(usize, &Self) -> usize,
-    {
-        let mut updates = Vec::new();
-        for (symbol, entry) in &self.terminal_entries {
-            for (i, action) in entry.actions.iter().enumerate() {
-                if let ParseAction::Shift { state, .. } = action {
-                    let result = f(*state, self);
-                    if result != *state {
-                        updates.push((*symbol, i, result));
-                    }
-                }
-            }
-        }
-        for (symbol, action) in &self.nonterminal_entries {
-            if let GotoAction::Goto(other_state) = action {
-                let result = f(*other_state, self);
-                if result != *other_state {
-                    updates.push((*symbol, 0, result));
-                }
-            }
-        }
-        for (symbol, action_index, new_state) in updates {
-            if symbol.is_non_terminal() {
-                self.nonterminal_entries
-                    .insert(symbol, GotoAction::Goto(new_state));
-            } else {
-                let entry = self.terminal_entries.get_mut(&symbol).unwrap();
-                if let ParseAction::Shift { is_repetition, .. } = entry.actions[action_index] {
-                    entry.actions[action_index] = ParseAction::Shift {
-                        state: new_state,
-                        is_repetition,
-                    };
-                }
-            }
-        }
-    }
-}

cli/loader/emscripten-version

@@ -1 +0,0 @@
-3.1.64

cli/src/highlight.rs

@@ -1,479 +0,0 @@
-use std::{
-    collections::HashMap,
-    fmt::Write,
-    fs,
-    io::{self, Write as _},
-    path, str,
-    sync::atomic::AtomicUsize,
-    time::Instant,
-};
-
-use anstyle::{Ansi256Color, AnsiColor, Color, Effects, RgbColor};
-use anyhow::Result;
-use lazy_static::lazy_static;
-use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer};
-use serde_json::{json, Value};
-use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer};
-use tree_sitter_loader::Loader;
-
-pub const HTML_HEADER: &str = "
-<!doctype HTML>
-<head>
-  <title>Tree-sitter Highlighting</title>
-  <style>
-    body {
-      font-family: monospace
-    }
-    .line-number {
-      user-select: none;
-      text-align: right;
-      color: rgba(27,31,35,.3);
-      padding: 0 10px;
-    }
-    .line {
-      white-space: pre;
-    }
-  </style>
-</head>
-<body>
-";
-
-pub const HTML_FOOTER: &str = "
-</body>
-";
-
-lazy_static! {
-    static ref CSS_STYLES_BY_COLOR_ID: Vec<String> =
-        serde_json::from_str(include_str!("../vendor/xterm-colors.json")).unwrap();
-}
-
-#[derive(Debug, Default)]
-pub struct Style {
-    pub ansi: anstyle::Style,
-    pub css: Option<String>,
-}
-
-#[derive(Debug)]
-pub struct Theme {
-    pub styles: Vec<Style>,
-    pub highlight_names: Vec<String>,
-}
-
-#[derive(Default, Deserialize, Serialize)]
-pub struct ThemeConfig {
-    #[serde(default)]
-    pub theme: Theme,
-}
-
-impl Theme {
-    pub fn load(path: &path::Path) -> io::Result<Self> {
-        let json = fs::read_to_string(path)?;
-        Ok(serde_json::from_str(&json).unwrap_or_default())
-    }
-
-    #[must_use]
-    pub fn default_style(&self) -> Style {
-        Style::default()
-    }
-}
-
-impl<'de> Deserialize<'de> for Theme {
-    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let mut styles = Vec::new();
-        let mut highlight_names = Vec::new();
-        if let Ok(colors) = HashMap::<String, Value>::deserialize(deserializer) {
-            highlight_names.reserve(colors.len());
-            styles.reserve(colors.len());
-            for (name, style_value) in colors {
-                let mut style = Style::default();
-                parse_style(&mut style, style_value);
-                highlight_names.push(name);
-                styles.push(style);
-            }
-        }
-        Ok(Self {
-            styles,
-            highlight_names,
-        })
-    }
-}
-
-impl Serialize for Theme {
-    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut map = serializer.serialize_map(Some(self.styles.len()))?;
-        for (name, style) in self.highlight_names.iter().zip(&self.styles) {
-            let style = &style.ansi;
-            let color = style.get_fg_color().map(|color| match color {
-                Color::Ansi(color) => match color {
-                    AnsiColor::Black => json!("black"),
-                    AnsiColor::Blue => json!("blue"),
-                    AnsiColor::Cyan => json!("cyan"),
-                    AnsiColor::Green => json!("green"),
-                    AnsiColor::Magenta => json!("purple"),
-                    AnsiColor::Red => json!("red"),
-                    AnsiColor::White => json!("white"),
-                    AnsiColor::Yellow => json!("yellow"),
-                    _ => unreachable!(),
-                },
-                Color::Ansi256(Ansi256Color(n)) => json!(n),
-                Color::Rgb(RgbColor(r, g, b)) => json!(format!("#{r:x?}{g:x?}{b:x?}")),
-            });
-            let effects = style.get_effects();
-            if effects.contains(Effects::BOLD)
-                || effects.contains(Effects::ITALIC)
-                || effects.contains(Effects::UNDERLINE)
-            {
-                let mut style_json = HashMap::new();
-                if let Some(color) = color {
-                    style_json.insert("color", color);
-                }
-                if effects.contains(Effects::BOLD) {
-                    style_json.insert("bold", Value::Bool(true));
-                }
-                if effects.contains(Effects::ITALIC) {
-                    style_json.insert("italic", Value::Bool(true));
-                }
-                if effects.contains(Effects::UNDERLINE) {
-                    style_json.insert("underline", Value::Bool(true));
-                }
-                map.serialize_entry(&name, &style_json)?;
-            } else if let Some(color) = color {
-                map.serialize_entry(&name, &color)?;
-            } else {
-                map.serialize_entry(&name, &Value::Null)?;
-            }
-        }
-        map.end()
-    }
-}
-
-impl Default for Theme {
-    fn default() -> Self {
-        serde_json::from_value(json!({
-              "attribute": {"color": 124, "italic": true},
-              "comment": {"color": 245, "italic": true},
-              "constant.builtin": {"color": 94, "bold": true},
-              "constant": 94,
-              "constructor": 136,
-              "embedded": null,
-              "function.builtin": {"color": 26, "bold": true},
-              "function": 26,
-              "keyword": 56,
-              "number": {"color": 94, "bold": true},
-              "module": 136,
-              "property": 124,
-              "operator": {"color": 239, "bold": true},
-              "punctuation.bracket": 239,
-              "punctuation.delimiter": 239,
-              "string.special": 30,
-              "string": 28,
-              "tag": 18,
-              "type": 23,
-              "type.builtin": {"color": 23, "bold": true},
-              "variable.builtin": {"bold": true},
-              "variable.parameter": {"underline": true}
-        }))
-        .unwrap()
-    }
-}
-
-fn parse_style(style: &mut Style, json: Value) {
-    if let Value::Object(entries) = json {
-        for (property_name, value) in entries {
-            match property_name.as_str() {
-                "bold" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.bold();
-                    }
-                }
-                "italic" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.italic();
-                    }
-                }
-                "underline" => {
-                    if value == Value::Bool(true) {
-                        style.ansi = style.ansi.underline();
-                    }
-                }
-                "color" => {
-                    if let Some(color) = parse_color(value) {
-                        style.ansi = style.ansi.fg_color(Some(color));
-                    }
-                }
-                _ => {}
-            }
-        }
-        style.css = Some(style_to_css(style.ansi));
-    } else if let Some(color) = parse_color(json) {
-        style.ansi = style.ansi.fg_color(Some(color));
-        style.css = Some(style_to_css(style.ansi));
-    } else {
-        style.css = None;
-    }
-
-    if let Some(Color::Rgb(RgbColor(red, green, blue))) = style.ansi.get_fg_color() {
-        if !terminal_supports_truecolor() {
-            style.ansi = style
-                .ansi
-                .fg_color(Some(closest_xterm_color(red, green, blue)));
-        }
-    }
-}
-
-fn parse_color(json: Value) -> Option<Color> {
-    match json {
-        Value::Number(n) => n.as_u64().map(|n| Color::Ansi256(Ansi256Color(n as u8))),
-        Value::String(s) => match s.to_lowercase().as_str() {
-            "black" => Some(Color::Ansi(AnsiColor::Black)),
-            "blue" => Some(Color::Ansi(AnsiColor::Blue)),
-            "cyan" => Some(Color::Ansi(AnsiColor::Cyan)),
-            "green" => Some(Color::Ansi(AnsiColor::Green)),
-            "purple" => Some(Color::Ansi(AnsiColor::Magenta)),
-            "red" => Some(Color::Ansi(AnsiColor::Red)),
-            "white" => Some(Color::Ansi(AnsiColor::White)),
-            "yellow" => Some(Color::Ansi(AnsiColor::Yellow)),
-            s => {
-                if let Some((red, green, blue)) = hex_string_to_rgb(s) {
-                    Some(Color::Rgb(RgbColor(red, green, blue)))
-                } else {
-                    None
-                }
-            }
-        },
-        _ => None,
-    }
-}
-
-fn hex_string_to_rgb(s: &str) -> Option<(u8, u8, u8)> {
-    if s.starts_with('#') && s.len() >= 7 {
-        if let (Ok(red), Ok(green), Ok(blue)) = (
-            u8::from_str_radix(&s[1..3], 16),
-            u8::from_str_radix(&s[3..5], 16),
-            u8::from_str_radix(&s[5..7], 16),
-        ) {
-            Some((red, green, blue))
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-fn style_to_css(style: anstyle::Style) -> String {
-    let mut result = "style='".to_string();
-    let effects = style.get_effects();
-    if effects.contains(Effects::UNDERLINE) {
-        write!(&mut result, "text-decoration: underline;").unwrap();
-    }
-    if effects.contains(Effects::BOLD) {
-        write!(&mut result, "font-weight: bold;").unwrap();
-    }
-    if effects.contains(Effects::ITALIC) {
-        write!(&mut result, "font-style: italic;").unwrap();
-    }
-    if let Some(color) = style.get_fg_color() {
-        write_color(&mut result, color);
-    }
-    result.push('\'');
-    result
-}
-
-fn write_color(buffer: &mut String, color: Color) {
-    match color {
-        Color::Ansi(color) => match color {
-            AnsiColor::Black => write!(buffer, "color: black").unwrap(),
-            AnsiColor::Red => write!(buffer, "color: red").unwrap(),
-            AnsiColor::Green => write!(buffer, "color: green").unwrap(),
-            AnsiColor::Yellow => write!(buffer, "color: yellow").unwrap(),
-            AnsiColor::Blue => write!(buffer, "color: blue").unwrap(),
-            AnsiColor::Magenta => write!(buffer, "color: purple").unwrap(),
-            AnsiColor::Cyan => write!(buffer, "color: cyan").unwrap(),
-            AnsiColor::White => write!(buffer, "color: white").unwrap(),
-            _ => unreachable!(),
-        },
-        Color::Ansi256(Ansi256Color(n)) => {
-            write!(buffer, "color: {}", CSS_STYLES_BY_COLOR_ID[n as usize]).unwrap();
-        }
-        Color::Rgb(RgbColor(r, g, b)) => write!(buffer, "color: #{r:02x}{g:02x}{b:02x}").unwrap(),
-    }
-}
-
-fn terminal_supports_truecolor() -> bool {
-    std::env::var("COLORTERM").map_or(false, |truecolor| {
-        truecolor == "truecolor" || truecolor == "24bit"
-    })
-}
-
-fn closest_xterm_color(red: u8, green: u8, blue: u8) -> Color {
-    use std::cmp::{max, min};
-
-    let colors = CSS_STYLES_BY_COLOR_ID
-        .iter()
-        .enumerate()
-        .map(|(color_id, hex)| (color_id as u8, hex_string_to_rgb(hex).unwrap()));
-
-    // Get the xterm color with the minimum Euclidean distance to the target color
-    // i.e.  distance = √ (r2 - r1)² + (g2 - g1)² + (b2 - b1)²
-    let distances = colors.map(|(color_id, (r, g, b))| {
-        let r_delta = (max(r, red) - min(r, red)) as u32;
-        let g_delta = (max(g, green) - min(g, green)) as u32;
-        let b_delta = (max(b, blue) - min(b, blue)) as u32;
-        let distance = r_delta.pow(2) + g_delta.pow(2) + b_delta.pow(2);
-        // don't need to actually take the square root for the sake of comparison
-        (color_id, distance)
-    });
-
-    Color::Ansi256(Ansi256Color(
-        distances.min_by(|(_, d1), (_, d2)| d1.cmp(d2)).unwrap().0,
-    ))
-}
-
-pub fn ansi(
-    loader: &Loader,
-    theme: &Theme,
-    source: &[u8],
-    config: &HighlightConfiguration,
-    print_time: bool,
-    cancellation_flag: Option<&AtomicUsize>,
-) -> Result<()> {
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-    let time = Instant::now();
-    let mut highlighter = Highlighter::new();
-
-    let events = highlighter.highlight(config, source, cancellation_flag, |string| {
-        loader.highlight_config_for_injection_string(string)
-    })?;
-
-    let mut style_stack = vec![theme.default_style().ansi];
-    for event in events {
-        match event? {
-            HighlightEvent::HighlightStart(highlight) => {
-                style_stack.push(theme.styles[highlight.0].ansi);
-            }
-            HighlightEvent::HighlightEnd => {
-                style_stack.pop();
-            }
-            HighlightEvent::Source { start, end } => {
-                let style = style_stack.last().unwrap();
-                write!(&mut stdout, "{style}").unwrap();
-                stdout.write_all(&source[start..end])?;
-                write!(&mut stdout, "{style:#}").unwrap();
-            }
-        }
-    }
-
-    if print_time {
-        eprintln!("Time: {}ms", time.elapsed().as_millis());
-    }
-
-    Ok(())
-}
-
-pub fn html(
-    loader: &Loader,
-    theme: &Theme,
-    source: &[u8],
-    config: &HighlightConfiguration,
-    quiet: bool,
-    print_time: bool,
-    cancellation_flag: Option<&AtomicUsize>,
-) -> Result<()> {
-    use std::io::Write;
-
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-    let time = Instant::now();
-    let mut highlighter = Highlighter::new();
-
-    let events = highlighter.highlight(config, source, cancellation_flag, |string| {
-        loader.highlight_config_for_injection_string(string)
-    })?;
-
-    let mut renderer = HtmlRenderer::new();
-    renderer.render(events, source, &move |highlight| {
-        theme.styles[highlight.0]
-            .css
-            .as_ref()
-            .map_or_else(|| "".as_bytes(), |css_style| css_style.as_bytes())
-    })?;
-
-    if !quiet {
-        writeln!(&mut stdout, "<table>")?;
-        for (i, line) in renderer.lines().enumerate() {
-            writeln!(
-                &mut stdout,
-                "<tr><td class=line-number>{}</td><td class=line>{line}</td></tr>",
-                i + 1,
-            )?;
-        }
-
-        writeln!(&mut stdout, "</table>")?;
-    }
-
-    if print_time {
-        eprintln!("Time: {}ms", time.elapsed().as_millis());
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use std::env;
-
-    use super::*;
-
-    const JUNGLE_GREEN: &str = "#26A69A";
-    const DARK_CYAN: &str = "#00AF87";
-
-    #[test]
-    fn test_parse_style() {
-        let original_environment_variable = env::var("COLORTERM");
-
-        let mut style = Style::default();
-        assert_eq!(style.ansi.get_fg_color(), None);
-        assert_eq!(style.css, None);
-
-        // darkcyan is an ANSI color and is preserved
-        env::set_var("COLORTERM", "");
-        parse_style(&mut style, Value::String(DARK_CYAN.to_string()));
-        assert_eq!(
-            style.ansi.get_fg_color(),
-            Some(Color::Ansi256(Ansi256Color(36)))
-        );
-        assert_eq!(style.css, Some("style=\'color: #00af87\'".to_string()));
-
-        // junglegreen is not an ANSI color and is preserved when the terminal supports it
-        env::set_var("COLORTERM", "truecolor");
-        parse_style(&mut style, Value::String(JUNGLE_GREEN.to_string()));
-        assert_eq!(
-            style.ansi.get_fg_color(),
-            Some(Color::Rgb(RgbColor(38, 166, 154)))
-        );
-        assert_eq!(style.css, Some("style=\'color: #26a69a\'".to_string()));
-
-        // junglegreen gets approximated as darkcyan when the terminal does not support it
-        env::set_var("COLORTERM", "");
-        parse_style(&mut style, Value::String(JUNGLE_GREEN.to_string()));
-        assert_eq!(
-            style.ansi.get_fg_color(),
-            Some(Color::Ansi256(Ansi256Color(36)))
-        );
-        assert_eq!(style.css, Some("style=\'color: #26a69a\'".to_string()));
-
-        if let Ok(environment_variable) = original_environment_variable {
-            env::set_var("COLORTERM", environment_variable);
-        } else {
-            env::remove_var("COLORTERM");
-        }
-    }
-}

cli/src/logger.rs

@@ -1,30 +0,0 @@
-use log::{LevelFilter, Log, Metadata, Record};
-
-#[allow(dead_code)]
-struct Logger {
-    pub filter: Option<String>,
-}
-
-impl Log for Logger {
-    fn enabled(&self, _: &Metadata) -> bool {
-        true
-    }
-
-    fn log(&self, record: &Record) {
-        eprintln!(
-            "[{}] {}",
-            record
-                .module_path()
-                .unwrap_or_default()
-                .trim_start_matches("rust_tree_sitter_cli::"),
-            record.args()
-        );
-    }
-
-    fn flush(&self) {}
-}
-
-pub fn init() {
-    log::set_boxed_logger(Box::new(Logger { filter: None })).unwrap();
-    log::set_max_level(LevelFilter::Info);
-}

cli/src/parse.rs

@@ -1,482 +0,0 @@
-use std::{
-    fmt, fs,
-    io::{self, Write},
-    path::Path,
-    sync::atomic::AtomicUsize,
-    time::{Duration, Instant},
-};
-
-use anyhow::{anyhow, Context, Result};
-use tree_sitter::{ffi, InputEdit, Language, LogType, Parser, Point, Tree};
-
-use super::util;
-use crate::fuzz::edits::Edit;
-
-#[derive(Debug, Default)]
-pub struct Stats {
-    pub successful_parses: usize,
-    pub total_parses: usize,
-    pub total_bytes: usize,
-    pub total_duration: Duration,
-}
-
-impl fmt::Display for Stats {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let duration_us = self.total_duration.as_micros();
-        writeln!(
-            f,
-            "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {:.2}%; average speed: {} bytes/ms",
-            self.total_parses,
-            self.successful_parses,
-            self.total_parses - self.successful_parses,
-            ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0,
-            if duration_us != 0 {
-                ((self.total_bytes as u128) * 1_000) / duration_us
-            } else {
-                0
-            }
-        )
-    }
-}
-
-#[derive(Copy, Clone, PartialEq, Eq)]
-pub enum ParseOutput {
-    Normal,
-    Quiet,
-    Xml,
-    Dot,
-}
-
-pub struct ParseFileOptions<'a> {
-    pub language: Language,
-    pub path: &'a Path,
-    pub edits: &'a [&'a str],
-    pub max_path_length: usize,
-    pub output: ParseOutput,
-    pub print_time: bool,
-    pub timeout: u64,
-    pub debug: bool,
-    pub debug_graph: bool,
-    pub cancellation_flag: Option<&'a AtomicUsize>,
-    pub encoding: Option<u32>,
-    pub open_log: bool,
-    pub no_ranges: bool,
-}
-
-#[derive(Copy, Clone)]
-pub struct ParseResult {
-    pub successful: bool,
-    pub bytes: usize,
-    pub duration: Option<Duration>,
-}
-
-pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Result<ParseResult> {
-    let mut _log_session = None;
-    parser.set_language(&opts.language)?;
-    let mut source_code = fs::read(opts.path)
-        .with_context(|| format!("Error reading source file {:?}", opts.path))?;
-
-    // If the `--cancel` flag was passed, then cancel the parse
-    // when the user types a newline.
-    unsafe { parser.set_cancellation_flag(opts.cancellation_flag) };
-
-    // Set a timeout based on the `--time` flag.
-    parser.set_timeout_micros(opts.timeout);
-
-    // Render an HTML graph if `--debug-graph` was passed
-    if opts.debug_graph {
-        _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?);
-    }
-    // Log to stderr if `--debug` was passed
-    else if opts.debug {
-        parser.set_logger(Some(Box::new(|log_type, message| {
-            if log_type == LogType::Lex {
-                io::stderr().write_all(b"  ").unwrap();
-            }
-            writeln!(&mut io::stderr(), "{message}").unwrap();
-        })));
-    }
-
-    let time = Instant::now();
-
-    #[inline(always)]
-    fn is_utf16_bom(bom_bytes: &[u8]) -> bool {
-        bom_bytes == [0xFF, 0xFE] || bom_bytes == [0xFE, 0xFF]
-    }
-
-    let tree = match opts.encoding {
-        Some(encoding) if encoding == ffi::TSInputEncodingUTF16 => {
-            let source_code_utf16 = source_code
-                .chunks_exact(2)
-                .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
-                .collect::<Vec<_>>();
-            parser.parse_utf16(&source_code_utf16, None)
-        }
-        None if source_code.len() >= 2 && is_utf16_bom(&source_code[0..2]) => {
-            let source_code_utf16 = source_code
-                .chunks_exact(2)
-                .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
-                .collect::<Vec<_>>();
-            parser.parse_utf16(&source_code_utf16, None)
-        }
-        _ => parser.parse(&source_code, None),
-    };
-
-    parser.stop_printing_dot_graphs();
-
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    if let Some(mut tree) = tree {
-        if opts.debug_graph && !opts.edits.is_empty() {
-            println!("BEFORE:\n{}", String::from_utf8_lossy(&source_code));
-        }
-
-        for (i, edit) in opts.edits.iter().enumerate() {
-            let edit = parse_edit_flag(&source_code, edit)?;
-            perform_edit(&mut tree, &mut source_code, &edit)?;
-            tree = parser.parse(&source_code, Some(&tree)).unwrap();
-
-            if opts.debug_graph {
-                println!("AFTER {i}:\n{}", String::from_utf8_lossy(&source_code));
-            }
-        }
-
-        let duration = time.elapsed();
-        let duration_ms = duration.as_micros() as f64 / 1e3;
-        let mut cursor = tree.walk();
-
-        if opts.output == ParseOutput::Normal {
-            let mut needs_newline = false;
-            let mut indent_level = 0;
-            let mut did_visit_children = false;
-            loop {
-                let node = cursor.node();
-                let is_named = node.is_named();
-                if did_visit_children {
-                    if is_named {
-                        stdout.write_all(b")")?;
-                        needs_newline = true;
-                    }
-                    if cursor.goto_next_sibling() {
-                        did_visit_children = false;
-                    } else if cursor.goto_parent() {
-                        did_visit_children = true;
-                        indent_level -= 1;
-                    } else {
-                        break;
-                    }
-                } else {
-                    if is_named {
-                        if needs_newline {
-                            stdout.write_all(b"\n")?;
-                        }
-                        for _ in 0..indent_level {
-                            stdout.write_all(b"  ")?;
-                        }
-                        let start = node.start_position();
-                        let end = node.end_position();
-                        if let Some(field_name) = cursor.field_name() {
-                            write!(&mut stdout, "{field_name}: ")?;
-                        }
-                        write!(&mut stdout, "({}", node.kind())?;
-                        if !opts.no_ranges {
-                            write!(
-                                &mut stdout,
-                                " [{}, {}] - [{}, {}]",
-                                start.row, start.column, end.row, end.column
-                            )?;
-                        }
-                        needs_newline = true;
-                    }
-                    if cursor.goto_first_child() {
-                        did_visit_children = false;
-                        indent_level += 1;
-                    } else {
-                        did_visit_children = true;
-                    }
-                }
-            }
-            cursor.reset(tree.root_node());
-            println!();
-        }
-
-        if opts.output == ParseOutput::Xml {
-            let mut needs_newline = false;
-            let mut indent_level = 0;
-            let mut did_visit_children = false;
-            let mut had_named_children = false;
-            let mut tags = Vec::<&str>::new();
-            writeln!(&mut stdout, "<?xml version=\"1.0\"?>")?;
-            loop {
-                let node = cursor.node();
-                let is_named = node.is_named();
-                if did_visit_children {
-                    if is_named {
-                        let tag = tags.pop();
-                        if had_named_children {
-                            for _ in 0..indent_level {
-                                stdout.write_all(b"  ")?;
-                            }
-                        }
-                        write!(&mut stdout, "</{}>", tag.expect("there is a tag"))?;
-                        // we only write a line in the case where it's the last sibling
-                        if let Some(parent) = node.parent() {
-                            if parent.child(parent.child_count() - 1).unwrap() == node {
-                                stdout.write_all(b"\n")?;
-                            }
-                        }
-                        needs_newline = true;
-                    }
-                    if cursor.goto_next_sibling() {
-                        did_visit_children = false;
-                        had_named_children = false;
-                    } else if cursor.goto_parent() {
-                        did_visit_children = true;
-                        had_named_children = is_named;
-                        indent_level -= 1;
-                        if !is_named && needs_newline {
-                            stdout.write_all(b"\n")?;
-                            for _ in 0..indent_level {
-                                stdout.write_all(b"  ")?;
-                            }
-                        }
-                    } else {
-                        break;
-                    }
-                } else {
-                    if is_named {
-                        if needs_newline {
-                            stdout.write_all(b"\n")?;
-                        }
-                        for _ in 0..indent_level {
-                            stdout.write_all(b"  ")?;
-                        }
-                        write!(&mut stdout, "<{}", node.kind())?;
-                        if let Some(field_name) = cursor.field_name() {
-                            write!(&mut stdout, " field=\"{field_name}\"")?;
-                        }
-                        let start = node.start_position();
-                        let end = node.end_position();
-                        write!(&mut stdout, " srow=\"{}\"", start.row)?;
-                        write!(&mut stdout, " scol=\"{}\"", start.column)?;
-                        write!(&mut stdout, " erow=\"{}\"", end.row)?;
-                        write!(&mut stdout, " ecol=\"{}\"", end.column)?;
-                        write!(&mut stdout, ">")?;
-                        tags.push(node.kind());
-                        needs_newline = true;
-                    }
-                    if cursor.goto_first_child() {
-                        did_visit_children = false;
-                        had_named_children = false;
-                        indent_level += 1;
-                    } else {
-                        did_visit_children = true;
-                        let start = node.start_byte();
-                        let end = node.end_byte();
-                        let value =
-                            std::str::from_utf8(&source_code[start..end]).expect("has a string");
-                        // if !is_named {
-                        //     for _ in 0..indent_level {
-                        //         stdout.write_all(b"  ")?;
-                        //     }
-                        // }
-                        if !is_named && needs_newline {
-                            stdout.write_all(b"\n")?;
-                            for _ in 0..indent_level {
-                                stdout.write_all(b"  ")?;
-                            }
-                        }
-                        write!(&mut stdout, "{}", html_escape::encode_text(value))?;
-                    }
-                }
-            }
-            cursor.reset(tree.root_node());
-            println!();
-        }
-
-        if opts.output == ParseOutput::Dot {
-            util::print_tree_graph(&tree, "log.html", opts.open_log).unwrap();
-        }
-
-        let mut first_error = None;
-        loop {
-            let node = cursor.node();
-            if node.has_error() {
-                if node.is_error() || node.is_missing() {
-                    first_error = Some(node);
-                    break;
-                }
-                if !cursor.goto_first_child() {
-                    break;
-                }
-            } else if !cursor.goto_next_sibling() {
-                break;
-            }
-        }
-
-        if first_error.is_some() || opts.print_time {
-            write!(
-                &mut stdout,
-                "{:width$}\t{duration_ms:>7.2} ms\t{:>6} bytes/ms",
-                opts.path.to_str().unwrap(),
-                (source_code.len() as u128 * 1_000_000) / duration.as_nanos(),
-                width = opts.max_path_length
-            )?;
-            if let Some(node) = first_error {
-                let start = node.start_position();
-                let end = node.end_position();
-                write!(&mut stdout, "\t(")?;
-                if node.is_missing() {
-                    if node.is_named() {
-                        write!(&mut stdout, "MISSING {}", node.kind())?;
-                    } else {
-                        write!(
-                            &mut stdout,
-                            "MISSING \"{}\"",
-                            node.kind().replace('\n', "\\n")
-                        )?;
-                    }
-                } else {
-                    write!(&mut stdout, "{}", node.kind())?;
-                }
-                write!(
-                    &mut stdout,
-                    " [{}, {}] - [{}, {}])",
-                    start.row, start.column, end.row, end.column
-                )?;
-            }
-            writeln!(&mut stdout)?;
-        }
-
-        return Ok(ParseResult {
-            successful: first_error.is_none(),
-            bytes: source_code.len(),
-            duration: Some(duration),
-        });
-    }
-
-    if opts.print_time {
-        let duration = time.elapsed();
-        let duration_ms = duration.as_micros() as f64 / 1e3;
-        writeln!(
-            &mut stdout,
-            "{:width$}\t{duration_ms:>7.2} ms\t(timed out)",
-            opts.path.to_str().unwrap(),
-            width = opts.max_path_length
-        )?;
-    }
-
-    Ok(ParseResult {
-        successful: false,
-        bytes: source_code.len(),
-        duration: None,
-    })
-}
-
-pub fn perform_edit(tree: &mut Tree, input: &mut Vec<u8>, edit: &Edit) -> Result<InputEdit> {
-    let start_byte = edit.position;
-    let old_end_byte = edit.position + edit.deleted_length;
-    let new_end_byte = edit.position + edit.inserted_text.len();
-    let start_position = position_for_offset(input, start_byte)?;
-    let old_end_position = position_for_offset(input, old_end_byte)?;
-    input.splice(start_byte..old_end_byte, edit.inserted_text.iter().copied());
-    let new_end_position = position_for_offset(input, new_end_byte)?;
-    let edit = InputEdit {
-        start_byte,
-        old_end_byte,
-        new_end_byte,
-        start_position,
-        old_end_position,
-        new_end_position,
-    };
-    tree.edit(&edit);
-    Ok(edit)
-}
-
-fn parse_edit_flag(source_code: &[u8], flag: &str) -> Result<Edit> {
-    let error = || {
-        anyhow!(concat!(
-            "Invalid edit string '{}'. ",
-            "Edit strings must match the pattern '<START_BYTE_OR_POSITION> <REMOVED_LENGTH> <NEW_TEXT>'"
-        ), flag)
-    };
-
-    // Three whitespace-separated parts:
-    // * edit position
-    // * deleted length
-    // * inserted text
-    let mut parts = flag.split(' ');
-    let position = parts.next().ok_or_else(error)?;
-    let deleted_length = parts.next().ok_or_else(error)?;
-    let inserted_text = parts.collect::<Vec<_>>().join(" ").into_bytes();
-
-    // Position can either be a byte_offset or row,column pair, separated by a comma
-    let position = if position == "$" {
-        source_code.len()
-    } else if position.contains(',') {
-        let mut parts = position.split(',');
-        let row = parts.next().ok_or_else(error)?;
-        let row = row.parse::<usize>().map_err(|_| error())?;
-        let column = parts.next().ok_or_else(error)?;
-        let column = column.parse::<usize>().map_err(|_| error())?;
-        offset_for_position(source_code, Point { row, column })?
-    } else {
-        position.parse::<usize>().map_err(|_| error())?
-    };
-
-    // Deleted length must be a byte count.
-    let deleted_length = deleted_length.parse::<usize>().map_err(|_| error())?;
-
-    Ok(Edit {
-        position,
-        deleted_length,
-        inserted_text,
-    })
-}
-
-pub fn offset_for_position(input: &[u8], position: Point) -> Result<usize> {
-    let mut row = 0;
-    let mut offset = 0;
-    let mut iter = memchr::memchr_iter(b'\n', input);
-    loop {
-        if let Some(pos) = iter.next() {
-            if row < position.row {
-                row += 1;
-                offset = pos;
-                continue;
-            }
-        }
-        offset += 1;
-        break;
-    }
-    if position.row - row > 0 {
-        return Err(anyhow!("Failed to address a row: {}", position.row));
-    }
-    if let Some(pos) = iter.next() {
-        if (pos - offset < position.column) || (input[offset] == b'\n' && position.column > 0) {
-            return Err(anyhow!("Failed to address a column: {}", position.column));
-        };
-    } else if input.len() - offset < position.column {
-        return Err(anyhow!("Failed to address a column over the end"));
-    }
-    Ok(offset + position.column)
-}
-
-pub fn position_for_offset(input: &[u8], offset: usize) -> Result<Point> {
-    if offset > input.len() {
-        return Err(anyhow!("Failed to address an offset: {offset}"));
-    }
-    let mut result = Point { row: 0, column: 0 };
-    let mut last = 0;
-    for pos in memchr::memchr_iter(b'\n', &input[..offset]) {
-        result.row += 1;
-        last = pos;
-    }
-    result.column = if result.row > 0 {
-        offset - last - 1
-    } else {
-        offset
-    };
-    Ok(result)
-}

cli/src/playground.html

@@ -1,176 +0,0 @@
-<head>
-  <meta charset="utf-8">
-  <title>tree-sitter THE_LANGUAGE_NAME</title>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.45.0/codemirror.min.css">
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.18.0/clusterize.min.css">
-  <link rel="icon" type="image/png" href="https://tree-sitter.github.io/tree-sitter/assets/images/favicon-32x32.png" sizes="32x32" />
-  <link rel="icon" type="image/png" href="https://tree-sitter.github.io/tree-sitter/assets/images/favicon-16x16.png" sizes="16x16" />
-</head>
-
-<body>
-  <div id="playground-container" style="visibility: hidden;">
-    <header>
-      <div class=header-item>
-        <bold>THE_LANGUAGE_NAME</bold>
-      </div>
-
-      <div class=header-item>
-        <label for="logging-checkbox">log</label>
-        <input id="logging-checkbox" type="checkbox"></input>
-      </div>
-
-      <div class=header-item>
-        <label for="query-checkbox">query</label>
-        <input id="query-checkbox" type="checkbox"></input>
-      </div>
-
-      <div class=header-item>
-        <label for="update-time">parse time: </label>
-        <span id="update-time"></span>
-      </div>
-
-      <div class=header-item>
-        <a href="https://tree-sitter.github.io/tree-sitter/playground#about">(?)</a>
-      </div>
-
-      <select id="language-select" style="display: none;">
-        <option value="parser">Parser</option>
-      </select>
-    </header>
-
-    <main>
-      <div id="input-pane">
-        <div id="code-container">
-          <textarea id="code-input"></textarea>
-        </div>
-
-        <div id="query-container" style="visibility: hidden; position: absolute;">
-          <textarea id="query-input"></textarea>
-        </div>
-      </div>
-
-      <div id="output-container-scroll">
-        <pre id="output-container" class="highlight"></pre>
-      </div>
-    </main>
-  </div>
-
-  <script
-    src="https://code.jquery.com/jquery-3.3.1.min.js"
-    crossorigin="anonymous">
-  </script>
-
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.45.0/codemirror.min.js"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.18.0/clusterize.min.js"></script>
-
-  <script>LANGUAGE_BASE_URL = "";</script>
-  <script src=tree-sitter.js></script>
-  <script src=playground.js></script>
-
-  <style>
-    body {
-      margin: 0;
-      padding: 0;
-    }
-
-    #playground-container {
-      width: 100%;
-      height: 100%;
-      display: flex;
-      flex-direction: column;
-    }
-
-    header {
-      box-sizing: border-box;
-      display: flex;
-      padding: 20px;
-      height: 60px;
-      border-bottom: 1px solid #aaa;
-    }
-
-    main {
-      flex: 1;
-      position: relative;
-    }
-
-    #input-pane {
-      position: absolute;
-      top: 0;
-      left: 0;
-      bottom: 0;
-      right: 50%;
-      display: flex;
-      flex-direction: column;
-    }
-
-    #code-container, #query-container {
-      flex: 1;
-      position: relative;
-      overflow: hidden;
-      border-right: 1px solid #aaa;
-      border-bottom: 1px solid #aaa;
-    }
-
-    #output-container-scroll {
-      position: absolute;
-      top: 0;
-      left: 50%;
-      bottom: 0;
-      right: 0;
-    }
-
-    .header-item {
-      margin-right: 30px;
-    }
-
-    #playground-container .CodeMirror {
-      position: absolute;
-      top: 0;
-      bottom: 0;
-      left: 0;
-      right: 0;
-      height: 100%;
-    }
-
-    #output-container-scroll {
-      flex: 1;
-      padding: 0;
-      overflow: auto;
-    }
-
-    #output-container {
-      padding: 0 10px;
-      margin: 0;
-    }
-
-    #logging-checkbox {
-      vertical-align: middle;
-    }
-
-    .CodeMirror div.CodeMirror-cursor {
-      border-left: 3px solid red;
-    }
-
-    a {
-      text-decoration: none;
-      color: #040404;
-      padding: 2px;
-    }
-
-    a:hover {
-      text-decoration: underline;
-    }
-
-    a.highlighted {
-      background-color: #d9d9d9;
-      color: red;
-      border-radius: 3px;
-      text-decoration: underline;
-    }
-
-    .query-error {
-      text-decoration: underline red dashed;
-      -webkit-text-decoration: underline red dashed;
-    }
-  </style>
-</body>

cli/src/query.rs

@@ -1,146 +0,0 @@
-use std::{
-    fs,
-    io::{self, Write},
-    ops::Range,
-    path::Path,
-    time::Instant,
-};
-
-use anstyle::AnsiColor;
-use anyhow::{Context, Result};
-use streaming_iterator::StreamingIterator;
-use tree_sitter::{Language, Parser, Point, Query, QueryCursor};
-
-use crate::{
-    query_testing::{self, to_utf8_point},
-    test::paint,
-};
-
-#[allow(clippy::too_many_arguments)]
-pub fn query_files_at_paths(
-    language: &Language,
-    paths: Vec<String>,
-    query_path: &Path,
-    ordered_captures: bool,
-    byte_range: Option<Range<usize>>,
-    point_range: Option<Range<Point>>,
-    should_test: bool,
-    quiet: bool,
-    print_time: bool,
-) -> Result<()> {
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    let query_source = fs::read_to_string(query_path)
-        .with_context(|| format!("Error reading query file {query_path:?}"))?;
-    let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?;
-
-    let mut query_cursor = QueryCursor::new();
-    if let Some(range) = byte_range {
-        query_cursor.set_byte_range(range);
-    }
-    if let Some(range) = point_range {
-        query_cursor.set_point_range(range);
-    }
-
-    let mut parser = Parser::new();
-    parser.set_language(language)?;
-
-    for path in paths {
-        let mut results = Vec::new();
-
-        if !should_test {
-            writeln!(&mut stdout, "{path}")?;
-        }
-
-        let source_code =
-            fs::read(&path).with_context(|| format!("Error reading source file {path:?}"))?;
-        let tree = parser.parse(&source_code, None).unwrap();
-
-        let start = Instant::now();
-        if ordered_captures {
-            let mut captures =
-                query_cursor.captures(&query, tree.root_node(), source_code.as_slice());
-            while let Some((mat, capture_index)) = captures.next() {
-                let capture = mat.captures[*capture_index];
-                let capture_name = &query.capture_names()[capture.index as usize];
-                if !quiet && !should_test {
-                    writeln!(
-                        &mut stdout,
-                        "    pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`",
-                        mat.pattern_index,
-                        capture.index,
-                        capture.node.start_position(),
-                        capture.node.end_position(),
-                        capture.node.utf8_text(&source_code).unwrap_or("")
-                    )?;
-                }
-                results.push(query_testing::CaptureInfo {
-                    name: (*capture_name).to_string(),
-                    start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
-                    end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
-                });
-            }
-        } else {
-            let mut matches =
-                query_cursor.matches(&query, tree.root_node(), source_code.as_slice());
-            while let Some(m) = matches.next() {
-                if !quiet && !should_test {
-                    writeln!(&mut stdout, "  pattern: {}", m.pattern_index)?;
-                }
-                for capture in m.captures {
-                    let start = capture.node.start_position();
-                    let end = capture.node.end_position();
-                    let capture_name = &query.capture_names()[capture.index as usize];
-                    if !quiet && !should_test {
-                        if end.row == start.row {
-                            writeln!(
-                                &mut stdout,
-                                "    capture: {} - {capture_name}, start: {start}, end: {end}, text: `{}`",
-                                capture.index,
-                                capture.node.utf8_text(&source_code).unwrap_or("")
-                            )?;
-                        } else {
-                            writeln!(
-                                &mut stdout,
-                                "    capture: {capture_name}, start: {start}, end: {end}",
-                            )?;
-                        }
-                    }
-                    results.push(query_testing::CaptureInfo {
-                        name: (*capture_name).to_string(),
-                        start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
-                        end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
-                    });
-                }
-            }
-        }
-        if query_cursor.did_exceed_match_limit() {
-            writeln!(
-                &mut stdout,
-                "  WARNING: Query exceeded maximum number of in-progress captures!"
-            )?;
-        }
-        if should_test {
-            let path_name = Path::new(&path).file_name().unwrap().to_str().unwrap();
-            match query_testing::assert_expected_captures(&results, &path, &mut parser, language) {
-                Ok(assertion_count) => {
-                    println!(
-                        "  ✓ {} ({} assertions)",
-                        paint(Some(AnsiColor::Green), path_name),
-                        assertion_count
-                    );
-                }
-                Err(e) => {
-                    println!("  ✗ {}", paint(Some(AnsiColor::Red), path_name));
-                    return Err(e);
-                }
-            }
-        }
-        if print_time {
-            writeln!(&mut stdout, "{:?}", start.elapsed())?;
-        }
-    }
-
-    Ok(())
-}

cli/src/tags.rs

@@ -1,98 +0,0 @@
-use std::{
-    fs,
-    io::{self, Write},
-    path::Path,
-    str,
-    time::Instant,
-};
-
-use anyhow::{anyhow, Result};
-use tree_sitter_loader::{Config, Loader};
-use tree_sitter_tags::TagsContext;
-
-use super::util;
-
-pub fn generate_tags(
-    loader: &Loader,
-    loader_config: &Config,
-    scope: Option<&str>,
-    paths: &[String],
-    quiet: bool,
-    time: bool,
-) -> Result<()> {
-    let mut lang = None;
-    if let Some(scope) = scope {
-        lang = loader.language_configuration_for_scope(scope)?;
-        if lang.is_none() {
-            return Err(anyhow!("Unknown scope '{scope}'"));
-        }
-    }
-
-    let mut context = TagsContext::new();
-    let cancellation_flag = util::cancel_on_signal();
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    for path in paths {
-        let path = Path::new(&path);
-        let (language, language_config) = match lang.clone() {
-            Some(v) => v,
-            None => {
-                if let Some(v) = loader.language_configuration_for_file_name(path)? {
-                    v
-                } else {
-                    eprintln!("{}", util::lang_not_found_for_path(path, loader_config));
-                    continue;
-                }
-            }
-        };
-
-        if let Some(tags_config) = language_config.tags_config(language)? {
-            let indent = if paths.len() > 1 {
-                if !quiet {
-                    writeln!(&mut stdout, "{}", path.to_string_lossy())?;
-                }
-                "\t"
-            } else {
-                ""
-            };
-
-            let source = fs::read(path)?;
-            let t0 = Instant::now();
-            for tag in context
-                .generate_tags(tags_config, &source, Some(&cancellation_flag))?
-                .0
-            {
-                let tag = tag?;
-                if !quiet {
-                    write!(
-                        &mut stdout,
-                        "{indent}{:<10}\t | {:<8}\t{} {} - {} `{}`",
-                        str::from_utf8(&source[tag.name_range]).unwrap_or(""),
-                        &tags_config.syntax_type_name(tag.syntax_type_id),
-                        if tag.is_definition { "def" } else { "ref" },
-                        tag.span.start,
-                        tag.span.end,
-                        str::from_utf8(&source[tag.line_range]).unwrap_or(""),
-                    )?;
-                    if let Some(docs) = tag.docs {
-                        if docs.len() > 120 {
-                            write!(&mut stdout, "\t{:?}...", docs.get(0..120).unwrap_or(""))?;
-                        } else {
-                            write!(&mut stdout, "\t{:?}", &docs)?;
-                        }
-                    }
-                    writeln!(&mut stdout)?;
-                }
-            }
-
-            if time {
-                writeln!(&mut stdout, "{indent}time: {}ms", t0.elapsed().as_millis(),)?;
-            }
-        } else {
-            eprintln!("No tags config found for path {path:?}");
-        }
-    }
-
-    Ok(())
-}

cli/src/templates/__init__.py

@@ -1,42 +0,0 @@
-"""PARSER_DESCRIPTION"""
-
-from importlib.resources import files as _files
-
-from ._binding import language
-
-
-def _get_query(name, file):
-    query = _files(f"{__package__}.queries") / file
-    globals()[name] = query.read_text()
-    return globals()[name]
-
-
-def __getattr__(name):
-    # NOTE: uncomment these to include any queries that this grammar contains:
-
-    # if name == "HIGHLIGHTS_QUERY":
-    #     return _get_query("HIGHLIGHTS_QUERY", "highlights.scm")
-    # if name == "INJECTIONS_QUERY":
-    #     return _get_query("INJECTIONS_QUERY", "injections.scm")
-    # if name == "LOCALS_QUERY":
-    #     return _get_query("LOCALS_QUERY", "locals.scm")
-    # if name == "TAGS_QUERY":
-    #     return _get_query("TAGS_QUERY", "tags.scm")
-
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
-__all__ = [
-    "language",
-    # "HIGHLIGHTS_QUERY",
-    # "INJECTIONS_QUERY",
-    # "LOCALS_QUERY",
-    # "TAGS_QUERY",
-]
-
-
-def __dir__():
-    return sorted(__all__ + [
-        "__all__", "__builtins__", "__cached__", "__doc__", "__file__",
-        "__loader__", "__name__", "__package__", "__path__", "__spec__",
-    ])

cli/src/templates/__init__.pyi

@@ -1,10 +0,0 @@
-from typing import Final
-
-# NOTE: uncomment these to include any queries that this grammar contains:
-
-# HIGHLIGHTS_QUERY: Final[str]
-# INJECTIONS_QUERY: Final[str]
-# LOCALS_QUERY: Final[str]
-# TAGS_QUERY: Final[str]
-
-def language() -> object: ...

cli/src/templates/binding_test.js

@@ -1,9 +0,0 @@
-/// <reference types="node" />
-
-const assert = require("node:assert");
-const { test } = require("node:test");
-
-test("can load grammar", () => {
-  const parser = new (require("tree-sitter"))();
-  assert.doesNotThrow(() => parser.setLanguage(require(".")));
-});

cli/src/templates/build.rs

@@ -1,22 +0,0 @@
-fn main() {
-    let src_dir = std::path::Path::new("src");
-
-    let mut c_config = cc::Build::new();
-    c_config.std("c11").include(src_dir);
-
-    #[cfg(target_env = "msvc")]
-    c_config.flag("-utf-8");
-
-    let parser_path = src_dir.join("parser.c");
-    c_config.file(&parser_path);
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
-
-    // NOTE: if your language uses an external scanner, uncomment this block:
-    /*
-    let scanner_path = src_dir.join("scanner.c");
-    c_config.file(&scanner_path);
-    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
-
-    c_config.compile("tree-sitter-PARSER_NAME");
-}

cli/src/templates/cmakelists.txt

@@ -1,54 +0,0 @@
-cmake_minimum_required(VERSION 3.13)
-
-project(tree-sitter-PARSER_NAME
-        VERSION "0.0.1"
-        DESCRIPTION "CAMEL_PARSER_NAME grammar for tree-sitter"
-        HOMEPAGE_URL "https://github.com/tree-sitter/tree-sitter-PARSER_NAME"
-        LANGUAGES C)
-
-option(BUILD_SHARED_LIBS "Build using shared libraries" ON)
-
-set(TREE_SITTER_ABI_VERSION ABI_VERSION_MAX CACHE STRING "Tree-sitter ABI version")
-if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$")
-    unset(TREE_SITTER_ABI_VERSION CACHE)
-    message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer")
-endif()
-
-find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI")
-
-add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c"
-                   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json"
-                   COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json
-                            --abi=${TREE_SITTER_ABI_VERSION}
-                   WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-                   COMMENT "Generating parser.c")
-
-add_library(tree-sitter-PARSER_NAME src/parser.c)
-if(EXISTS src/scanner.c)
-  target_sources(tree-sitter-PARSER_NAME PRIVATE src/scanner.c)
-endif()
-target_include_directories(tree-sitter-PARSER_NAME PRIVATE src)
-
-set_target_properties(tree-sitter-PARSER_NAME
-                      PROPERTIES
-                      C_STANDARD 11
-                      POSITION_INDEPENDENT_CODE ON
-                      SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}")
-
-configure_file(bindings/c/tree-sitter-PARSER_NAME.pc.in
-               "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc" @ONLY)
-
-include(GNUInstallDirs)
-
-install(FILES bindings/c/tree-sitter-PARSER_NAME.h
-        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter")
-install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc"
-        DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig")
-install(TARGETS tree-sitter-PARSER_NAME
-        LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-
-add_custom_target(test "${TREE_SITTER_CLI}" test
-                  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-                  COMMENT "tree-sitter test")
-
-# vim:ft=cmake:

cli/src/templates/gitattributes

@@ -1,13 +0,0 @@
-* text=auto eol=lf
-
-src/*.json linguist-generated
-src/parser.c linguist-generated
-src/tree_sitter/* linguist-generated
-
-bindings/** linguist-generated
-binding.gyp linguist-generated
-setup.py linguist-generated
-Makefile linguist-generated
-CMakeLists.txt linguist-generated
-Package.swift linguist-generated
-go.mod linguist-generated

cli/src/templates/go.mod

@@ -1,5 +0,0 @@
-module PARSER_URL_STRIPPED
-
-go 1.22
-
-require github.com/tree-sitter/go-tree-sitter v0.23.1

cli/src/templates/index.d.ts

@@ -1,28 +0,0 @@
-type BaseNode = {
-  type: string;
-  named: boolean;
-};
-
-type ChildNode = {
-  multiple: boolean;
-  required: boolean;
-  types: BaseNode[];
-};
-
-type NodeInfo =
-  | (BaseNode & {
-      subtypes: BaseNode[];
-    })
-  | (BaseNode & {
-      fields: { [name: string]: ChildNode };
-      children: ChildNode[];
-    });
-
-type Language = {
-  name: string;
-  language: unknown;
-  nodeTypeInfo: NodeInfo[];
-};
-
-declare const language: Language;
-export = language;

cli/src/templates/index.js

@@ -1,7 +0,0 @@
-const root = require("path").join(__dirname, "..", "..");
-
-module.exports = require("node-gyp-build")(root);
-
-try {
-  module.exports.nodeTypeInfo = require("../../src/node-types.json");
-} catch (_) {}

cli/src/templates/lib.rs

@@ -1,53 +0,0 @@
-//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library.
-//!
-//! Typically, you will use the [LANGUAGE][] constant to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
-//!
-//! ```
-//! let code = r#"
-//! "#;
-//! let mut parser = tree_sitter::Parser::new();
-//! let language = tree_sitter_PARSER_NAME::LANGUAGE;
-//! parser
-//!     .set_language(&language.into())
-//!     .expect("Error loading CAMEL_PARSER_NAME parser");
-//! let tree = parser.parse(code, None).unwrap();
-//! assert!(!tree.root_node().has_error());
-//! ```
-//!
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
-//! [tree-sitter]: https://tree-sitter.github.io/
-
-use tree_sitter_language::LanguageFn;
-
-extern "C" {
-    fn tree_sitter_PARSER_NAME() -> *const ();
-}
-
-/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar.
-///
-/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
-pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) };
-
-/// The content of the [`node-types.json`][] file for this grammar.
-///
-/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
-pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
-
-// NOTE: uncomment these to include any queries that this grammar contains:
-
-// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn test_can_load_grammar() {
-        let mut parser = tree_sitter::Parser::new();
-        parser
-            .set_language(&super::LANGUAGE.into())
-            .expect("Error loading CAMEL_PARSER_NAME parser");
-    }
-}

cli/src/templates/package.swift

@@ -1,37 +0,0 @@
-// swift-tools-version:5.3
-import PackageDescription
-
-let package = Package(
-    name: "TreeSitterCAMEL_PARSER_NAME",
-    products: [
-        .library(name: "TreeSitterCAMEL_PARSER_NAME", targets: ["TreeSitterCAMEL_PARSER_NAME"]),
-    ],
-    dependencies: [
-        .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"),
-    ],
-    targets: [
-        .target(
-            name: "TreeSitterCAMEL_PARSER_NAME",
-            dependencies: [],
-            path: ".",
-            sources: [
-                "src/parser.c",
-                // NOTE: if your language has an external scanner, add it here.
-            ],
-            resources: [
-                .copy("queries")
-            ],
-            publicHeadersPath: "bindings/swift",
-            cSettings: [.headerSearchPath("src")]
-        ),
-        .testTarget(
-            name: "TreeSitterCAMEL_PARSER_NAMETests",
-            dependencies: [
-                "SwiftTreeSitter",
-                "TreeSitterCAMEL_PARSER_NAME",
-            ],
-            path: "bindings/swift/TreeSitterCAMEL_PARSER_NAMETests"
-        )
-    ],
-    cLanguageStandard: .c11
-)

cli/src/templates/setup.py

@@ -1,62 +0,0 @@
-from os.path import isdir, join
-from platform import system
-
-from setuptools import Extension, find_packages, setup
-from setuptools.command.build import build
-from wheel.bdist_wheel import bdist_wheel
-
-
-class Build(build):
-    def run(self):
-        if isdir("queries"):
-            dest = join(self.build_lib, "tree_sitter_PARSER_NAME", "queries")
-            self.copy_tree("queries", dest)
-        super().run()
-
-
-class BdistWheel(bdist_wheel):
-    def get_tag(self):
-        python, abi, platform = super().get_tag()
-        if python.startswith("cp"):
-            python, abi = "cp39", "abi3"
-        return python, abi, platform
-
-
-setup(
-    packages=find_packages("bindings/python"),
-    package_dir={"": "bindings/python"},
-    package_data={
-        "tree_sitter_LOWER_PARSER_NAME": ["*.pyi", "py.typed"],
-        "tree_sitter_LOWER_PARSER_NAME.queries": ["*.scm"],
-    },
-    ext_package="tree_sitter_LOWER_PARSER_NAME",
-    ext_modules=[
-        Extension(
-            name="_binding",
-            sources=[
-                "bindings/python/tree_sitter_LOWER_PARSER_NAME/binding.c",
-                "src/parser.c",
-                # NOTE: if your language uses an external scanner, add it here.
-            ],
-            extra_compile_args=[
-                "-std=c11",
-                "-fvisibility=hidden",
-            ] if system() != "Windows" else [
-                "/std:c11",
-                "/utf-8",
-            ],
-            define_macros=[
-                ("Py_LIMITED_API", "0x03090000"),
-                ("PY_SSIZE_T_CLEAN", None),
-                ("TREE_SITTER_HIDE_SYMBOLS", None),
-            ],
-            include_dirs=["src"],
-            py_limited_api=True,
-        )
-    ],
-    cmdclass={
-        "build": Build,
-        "bdist_wheel": BdistWheel
-    },
-    zip_safe=False
-)

cli/src/templates/test_binding.py

@@ -1,11 +0,0 @@
-from unittest import TestCase
-
-import tree_sitter, tree_sitter_LOWER_PARSER_NAME
-
-
-class TestLanguage(TestCase):
-    def test_can_load_grammar(self):
-        try:
-            tree_sitter.Language(tree_sitter_LOWER_PARSER_NAME.language())
-        except Exception:
-            self.fail("Error loading CAMEL_PARSER_NAME grammar")

cli/src/tests/async_context_test.rs

@@ -1,284 +0,0 @@
-use std::{
-    future::Future,
-    pin::{pin, Pin},
-    ptr,
-    task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker},
-};
-
-use tree_sitter::Parser;
-
-use super::helpers::fixtures::get_language;
-
-#[test]
-fn test_node_in_fut() {
-    let (ret, pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("bash");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let fut_val_fn = || async {
-            // eprintln!("fut_val_fn: {}", root.child(0).unwrap().kind());
-            yield_now().await;
-            root.child(0).unwrap().kind()
-        };
-
-        yield_now().await;
-
-        let fut_ref_fn = || async {
-            // eprintln!("fut_ref_fn: {}", root_ref.child(0).unwrap().kind());
-            yield_now().await;
-            root_ref.child(0).unwrap().kind()
-        };
-
-        let f1 = fut_val_fn().await;
-        let f2 = fut_ref_fn().await;
-        assert_eq!(f1, f2);
-
-        let fut_val = async {
-            // eprintln!("fut_val: {}", root.child(0).unwrap().kind());
-            yield_now().await;
-            root.child(0).unwrap().kind()
-        };
-
-        let fut_ref = async {
-            // eprintln!("fut_ref: {}", root_ref.child(0).unwrap().kind());
-            yield_now().await;
-            root_ref.child(0).unwrap().kind()
-        };
-
-        let f1 = fut_val.await;
-        let f2 = fut_ref.await;
-        assert_eq!(f1, f2);
-
-        f1
-    })
-    .join();
-    // eprintln!("pended: {pended:?}");
-    assert_eq!(ret, "comment");
-    assert_eq!(pended, 5);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut() {
-    let ((), pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("c");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = async {
-            yield_now().await;
-            let _ = root.to_sexp();
-        };
-
-        yield_now().await;
-
-        let fut_ref = async {
-            yield_now().await;
-            let _ = root_ref.to_sexp();
-            cursor_ref.goto_first_child();
-        };
-
-        fut_val.await;
-        fut_ref.await;
-
-        cursor_ref.goto_first_child();
-    })
-    .join();
-    assert_eq!(pended, 3);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut_with_fut_fabrics() {
-    let ((), pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("javascript");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = || async {
-            yield_now().await;
-            let _ = root.to_sexp();
-        };
-
-        yield_now().await;
-
-        let fut_ref = || async move {
-            yield_now().await;
-            let _ = root_ref.to_sexp();
-            cursor_ref.goto_first_child();
-        };
-
-        fut_val().await;
-        fut_val().await;
-        fut_ref().await;
-    })
-    .join();
-    assert_eq!(pended, 4);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut_with_inner_spawns() {
-    let (ret, pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("rust");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = || {
-            let tree = tree.clone();
-            async move {
-                let root = tree.root_node();
-                let mut cursor = tree.walk();
-                let cursor_ref = &mut cursor;
-                yield_now().await;
-                let _ = root.to_sexp();
-                cursor_ref.goto_first_child();
-            }
-        };
-
-        yield_now().await;
-
-        let fut_ref = || {
-            let tree = tree.clone();
-            async move {
-                let root = tree.root_node();
-                let root_ref = &root;
-                let mut cursor = tree.walk();
-                let cursor_ref = &mut cursor;
-                yield_now().await;
-                let _ = root_ref.to_sexp();
-                cursor_ref.goto_first_child();
-            }
-        };
-
-        let ((), p1) = tokio_like_spawn(fut_val()).await.unwrap();
-        let ((), p2) = tokio_like_spawn(fut_ref()).await.unwrap();
-
-        cursor_ref.goto_first_child();
-
-        fut_val().await;
-        fut_val().await;
-        fut_ref().await;
-
-        cursor_ref.goto_first_child();
-
-        p1 + p2
-    })
-    .join();
-    assert_eq!(pended, 4);
-    assert_eq!(ret, 2);
-}
-
-fn tokio_like_spawn<T>(future: T) -> JoinHandle<(T::Output, usize)>
-where
-    T: Future + Send + 'static,
-    T::Output: Send + 'static,
-{
-    // No runtime, just noop waker
-
-    let waker = noop_waker();
-    let mut cx = task::Context::from_waker(&waker);
-
-    let mut pending = 0;
-    let mut future = pin!(future);
-    let ret = loop {
-        match future.as_mut().poll(&mut cx) {
-            Poll::Pending => pending += 1,
-            Poll::Ready(r) => {
-                // eprintln!("ready, pended: {pending}");
-                break r;
-            }
-        }
-    };
-    JoinHandle::new((ret, pending))
-}
-
-async fn yield_now() {
-    struct SimpleYieldNow {
-        yielded: bool,
-    }
-
-    impl Future for SimpleYieldNow {
-        type Output = ();
-
-        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-            cx.waker().wake_by_ref();
-            if self.yielded {
-                return Poll::Ready(());
-            }
-            self.yielded = true;
-            Poll::Pending
-        }
-    }
-
-    SimpleYieldNow { yielded: false }.await;
-}
-
-pub fn noop_waker() -> Waker {
-    const VTABLE: RawWakerVTable = RawWakerVTable::new(
-        // Cloning just returns a new no-op raw waker
-        |_| RAW,
-        // `wake` does nothing
-        |_| {},
-        // `wake_by_ref` does nothing
-        |_| {},
-        // Dropping does nothing as we don't allocate anything
-        |_| {},
-    );
-    const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE);
-    unsafe { Waker::from_raw(RAW) }
-}
-
-struct JoinHandle<T> {
-    data: Option<T>,
-}
-
-impl<T> JoinHandle<T> {
-    #[must_use]
-    const fn new(data: T) -> Self {
-        Self { data: Some(data) }
-    }
-
-    fn join(&mut self) -> T {
-        self.data.take().unwrap()
-    }
-}
-
-impl<T: Unpin> Future for JoinHandle<T> {
-    type Output = std::result::Result<T, ()>;
-
-    fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let data = self.get_mut().data.take().unwrap();
-        Poll::Ready(Ok(data))
-    }
-}