mirror of
https://github.com/tree-sitter/tree-sitter.git
synced 2026-09-11 08:06:21 -04:00
Compare commits
78 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd5b087cd9 | ||
|
|
c0d1444118 | ||
|
|
0b04fd0533 | ||
|
|
05cf9a161a | ||
|
|
bab48517d7 | ||
|
|
e28cb5ae74 | ||
|
|
3839f6fcf5 | ||
|
|
001a926d56 | ||
|
|
6f2e8a6cf4 | ||
|
|
0ae615883e | ||
|
|
9ce156713c | ||
|
|
365b1f0f91 | ||
|
|
8e87144b61 | ||
|
|
b61eabb4d2 | ||
|
|
c802b44dff | ||
|
|
16c7bfb48f | ||
|
|
d01bd9b1e5 | ||
|
|
594f9d5580 | ||
|
|
534c4a074c | ||
|
|
0de6ea6edd | ||
|
|
6fdf7fdbc6 | ||
|
|
a6aabeb941 | ||
|
|
b1e10d5410 | ||
|
|
b3e86f75cc | ||
|
|
a226c68720 | ||
|
|
d03bb288e2 | ||
|
|
ed06ca058b | ||
|
|
fc5f1b4526 | ||
|
|
054f892546 | ||
|
|
4bcc064ecc | ||
|
|
93a5767892 | ||
|
|
a5129d5bc0 | ||
|
|
9a22dff7e5 | ||
|
|
fdc20d9c58 | ||
|
|
89e804b7e4 | ||
|
|
3b85287d6d | ||
|
|
79b60a271c | ||
|
|
ad7a629a1b | ||
|
|
325bc50d6f | ||
|
|
932bde72b2 | ||
|
|
470813116b | ||
|
|
7ec1794d6b | ||
|
|
fa8811f7f7 | ||
|
|
ef4999bf61 | ||
|
|
77e43dd116 | ||
|
|
666144d3ed | ||
|
|
ce2cb41e1f | ||
|
|
a423343bd3 | ||
|
|
c8aedb8cfa | ||
|
|
308b96d927 | ||
|
|
e98a09b6cc | ||
|
|
d3a20faff9 | ||
|
|
88a5475496 | ||
|
|
6a8a5e33d9 | ||
|
|
bc2e4e2386 | ||
|
|
f44e86628a | ||
|
|
ed6e42cbf0 | ||
|
|
4809aaaf04 | ||
|
|
152d2756fc | ||
|
|
f05efbb352 | ||
|
|
1f221c8500 | ||
|
|
fdca0718bc | ||
|
|
fa7b1b2a66 | ||
|
|
adcc4d1f7b | ||
|
|
7d9c544c96 | ||
|
|
c1e49d1571 | ||
|
|
eae6554735 | ||
|
|
48ee942c4f | ||
|
|
9ee2b87dd6 | ||
|
|
fb91deb8d9 | ||
|
|
789a966f96 | ||
|
|
3c49fef0e3 | ||
|
|
8a297b86bc | ||
|
|
ac6644016c | ||
|
|
a80765614b | ||
|
|
34602af22c | ||
|
|
c4f81931e6 | ||
|
|
25777e5a64 |
105
.github/workflows/build.yml
vendored
105
.github/workflows/build.yml
vendored
|
|
@ -46,8 +46,8 @@ jobs:
|
|||
|
||||
# Extra features
|
||||
- { platform: linux-arm64 , features: wasm }
|
||||
- { platform: linux-x64 , features: wasm }
|
||||
- { platform: macos-arm64 , features: wasm }
|
||||
- { platform: linux-x64 , features: wasm , run-wasm-test: true }
|
||||
- { platform: macos-arm64 , features: wasm , run-wasm-test: true }
|
||||
- { platform: macos-x64 , features: wasm }
|
||||
|
||||
# Cross-compilation
|
||||
|
|
@ -85,14 +85,37 @@ jobs:
|
|||
} >> $GITHUB_ENV
|
||||
|
||||
- name: Get emscripten version
|
||||
if: contains(matrix.features, 'wasm')
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
|
||||
|
||||
- name: Install Emscripten
|
||||
if: contains(matrix.features, 'wasm')
|
||||
uses: mymindstorm/setup-emsdk@v14
|
||||
- name: Cache Emscripten SDK
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
version: ${{ env.EMSCRIPTEN_VERSION }}
|
||||
path: emsdk
|
||||
key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
|
||||
|
||||
- name: Install Emscripten
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
run: |
|
||||
if [[ ! -d emsdk ]]; then
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git
|
||||
fi
|
||||
cd emsdk
|
||||
./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
|
||||
./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
|
||||
echo "$PWD" >> "$GITHUB_PATH"
|
||||
echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
echo "EMSDK=$PWD" >> "$GITHUB_ENV"
|
||||
echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
cache-dependency-path: lib/binding_web/package-lock.json
|
||||
|
||||
- name: Set up Rust
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
|
|
@ -117,34 +140,12 @@ jobs:
|
|||
if: matrix.platform == 'windows-x64'
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
update: true
|
||||
install: |
|
||||
mingw-w64-x86_64-toolchain
|
||||
mingw-w64-x86_64-clang
|
||||
mingw-w64-x86_64-make
|
||||
mingw-w64-x86_64-cmake
|
||||
|
||||
# TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
|
||||
# the `mismatched-lifetime-syntaxes` lint
|
||||
- name: Build wasmtime library (Windows x64 MSYS2)
|
||||
if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64'
|
||||
run: |
|
||||
mkdir -p target
|
||||
WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
|
||||
jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
|
||||
curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
|
||||
cd target/wasmtime-${WASMTIME_VERSION}
|
||||
cmake -S crates/c-api -B target/c-api \
|
||||
-DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
|
||||
-DWASMTIME_DISABLE_ALL_FEATURES=ON \
|
||||
-DWASMTIME_FEATURE_CRANELIFT=ON \
|
||||
-DWASMTIME_TARGET='x86_64-pc-windows-gnu'
|
||||
cmake --build target/c-api && cmake --install target/c-api
|
||||
printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
|
||||
env:
|
||||
WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
|
||||
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
|
||||
|
||||
- name: Build C library (Windows x64 MSYS2 CMake)
|
||||
if: matrix.platform == 'windows-x64'
|
||||
shell: msys2 {0}
|
||||
|
|
@ -171,26 +172,23 @@ jobs:
|
|||
env:
|
||||
WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
|
||||
|
||||
# TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
|
||||
# the `mismatched-lifetime-syntaxes` lint
|
||||
- name: Build wasmtime library
|
||||
if: contains(matrix.features, 'wasm')
|
||||
- name: Download wasmtime C API
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
run: |
|
||||
mkdir -p target
|
||||
WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
|
||||
jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
|
||||
curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
|
||||
cd target/wasmtime-${WASMTIME_VERSION}
|
||||
cmake -S crates/c-api -B target/c-api \
|
||||
-DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
|
||||
-DWASMTIME_DISABLE_ALL_FEATURES=ON \
|
||||
-DWASMTIME_FEATURE_CRANELIFT=ON \
|
||||
-DWASMTIME_TARGET='${{ matrix.target }}'
|
||||
cmake --build target/c-api && cmake --install target/c-api
|
||||
printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
|
||||
case '${{ matrix.target }}' in
|
||||
x86_64-unknown-linux-gnu) WT_TARGET=x86_64-linux ;;
|
||||
aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
|
||||
x86_64-apple-darwin) WT_TARGET=x86_64-macos ;;
|
||||
aarch64-apple-darwin) WT_TARGET=aarch64-macos ;;
|
||||
esac
|
||||
curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
|
||||
| tar xJf - -C target
|
||||
printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
|
||||
env:
|
||||
WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
|
||||
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
|
||||
|
||||
- name: Build C library (make)
|
||||
if: runner.os != 'Windows'
|
||||
|
|
@ -227,10 +225,10 @@ jobs:
|
|||
cmake --build build/shared --verbose
|
||||
env:
|
||||
CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
|
||||
WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
|
||||
WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
|
||||
|
||||
- name: Build Wasm library
|
||||
if: contains(matrix.features, 'wasm')
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
shell: bash
|
||||
run: |
|
||||
cd lib/binding_web
|
||||
|
|
@ -247,7 +245,8 @@ jobs:
|
|||
run: cargo check --no-default-features --target='${{ matrix.target }}'
|
||||
|
||||
- name: Build target
|
||||
run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE
|
||||
if: "!inputs.run-test"
|
||||
run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
|
||||
env:
|
||||
PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
|
||||
|
||||
|
|
@ -265,20 +264,20 @@ jobs:
|
|||
run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
|
||||
|
||||
- name: Generate Wasm fixtures
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true'
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
|
||||
run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
|
||||
|
||||
- name: Run main tests
|
||||
if: inputs.run-test && !matrix.no-run
|
||||
run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}'
|
||||
run: cargo test --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
|
||||
|
||||
- name: Run Wasm tests
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm')
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
|
||||
run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
|
||||
|
||||
- name: Upload CLI artifact
|
||||
if: "!matrix.no-run"
|
||||
uses: actions/upload-artifact@v5
|
||||
if: "!inputs.run-test && !matrix.no-run"
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tree-sitter.${{ matrix.platform }}
|
||||
path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
|
||||
|
|
@ -286,8 +285,8 @@ jobs:
|
|||
retention-days: 7
|
||||
|
||||
- name: Upload Wasm artifacts
|
||||
if: matrix.platform == 'linux-x64'
|
||||
uses: actions/upload-artifact@v5
|
||||
if: "!inputs.run-test && matrix.platform == 'linux-x64'"
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tree-sitter.wasm
|
||||
path: |
|
||||
|
|
|
|||
22
.github/workflows/crate_versions.yml
vendored
Normal file
22
.github/workflows/crate_versions.yml
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
name: Crate Versions Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled, opened, synchronize, reopened]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check-crates:
|
||||
runs-on: ubuntu-latest
|
||||
if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Rust
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
|
||||
- name: Check crates against crates.io
|
||||
uses: katyo/publish-crates@v2
|
||||
with:
|
||||
dry-run: true
|
||||
3
.github/workflows/release.yml
vendored
3
.github/workflows/release.yml
vendored
|
|
@ -45,6 +45,7 @@ jobs:
|
|||
for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
|
||||
exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
|
||||
gzip --stdout --name $exe > target/tree-sitter-$platform.gz
|
||||
zip -j9 target/tree-sitter-cli-$platform.zip $exe
|
||||
done
|
||||
rm -rf artifacts
|
||||
ls -l target/
|
||||
|
|
@ -54,12 +55,14 @@ jobs:
|
|||
with:
|
||||
subject-path: |
|
||||
target/tree-sitter-*.gz
|
||||
target/tree-sitter-cli-*.zip
|
||||
target/web-tree-sitter.tar.gz
|
||||
|
||||
- name: Create release
|
||||
run: |-
|
||||
gh release create $GITHUB_REF_NAME \
|
||||
target/tree-sitter-*.gz \
|
||||
target/tree-sitter-cli-*.zip \
|
||||
target/web-tree-sitter.tar.gz
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
cmake_minimum_required(VERSION 3.13)
|
||||
|
||||
project(tree-sitter
|
||||
VERSION "0.26.3"
|
||||
VERSION "0.26.8"
|
||||
DESCRIPTION "An incremental parsing system for programming tools"
|
||||
HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
|
||||
LANGUAGES C)
|
||||
|
|
@ -81,7 +81,7 @@ set_target_properties(tree-sitter
|
|||
SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
|
||||
DEFINE_SYMBOL "")
|
||||
|
||||
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
|
||||
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
|
||||
|
|
|
|||
BIN
Cargo.lock
generated
BIN
Cargo.lock
generated
Binary file not shown.
14
Cargo.toml
14
Cargo.toml
|
|
@ -14,7 +14,7 @@ members = [
|
|||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.26.3"
|
||||
version = "0.26.8"
|
||||
authors = [
|
||||
"Max Brunsfeld <maxbrunsfeld@gmail.com>",
|
||||
"Amaan Qureshi <amaanq12@gmail.com>",
|
||||
|
|
@ -153,11 +153,11 @@ walkdir = "2.5.0"
|
|||
wasmparser = "0.243.0"
|
||||
webbrowser = "1.0.5"
|
||||
|
||||
tree-sitter = { version = "0.26.3", path = "./lib" }
|
||||
tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" }
|
||||
tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" }
|
||||
tree-sitter-config = { version = "0.26.3", path = "./crates/config" }
|
||||
tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" }
|
||||
tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" }
|
||||
tree-sitter = { version = "0.26.8", path = "./lib" }
|
||||
tree-sitter-generate = { version = "0.26.8", path = "./crates/generate", default-features = false }
|
||||
tree-sitter-loader = { version = "0.26.8", path = "./crates/loader" }
|
||||
tree-sitter-config = { version = "0.26.8", path = "./crates/config" }
|
||||
tree-sitter-highlight = { version = "0.26.8", path = "./crates/highlight" }
|
||||
tree-sitter-tags = { version = "0.26.8", path = "./crates/tags" }
|
||||
|
||||
tree-sitter-language = { version = "0.1", path = "./crates/language" }
|
||||
|
|
|
|||
5
Makefile
5
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
VERSION := 0.26.3
|
||||
VERSION := 0.26.8
|
||||
DESCRIPTION := An incremental parsing system for programming tools
|
||||
HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
|
|||
ARFLAGS := rcs
|
||||
CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
|
||||
override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
|
||||
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
|
||||
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
|
||||
override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
|
||||
|
||||
# ABI versioning
|
||||
|
|
@ -122,7 +122,6 @@ test-wasm:
|
|||
|
||||
lint:
|
||||
cargo update --workspace --locked --quiet
|
||||
cargo check --workspace --all-targets
|
||||
cargo fmt --all --check
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ let package = Package(
|
|||
.headerSearchPath("src"),
|
||||
.define("_POSIX_C_SOURCE", to: "200112L"),
|
||||
.define("_DEFAULT_SOURCE"),
|
||||
.define("_BSD_SOURCE"),
|
||||
.define("_DARWIN_C_SOURCE"),
|
||||
]),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
|
|||
|
||||
lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
|
||||
lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
|
||||
lib.root_module.addCMacro("_BSD_SOURCE", "");
|
||||
lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
|
||||
|
||||
if (wasm) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
.{
|
||||
.name = .tree_sitter,
|
||||
.fingerprint = 0x841224b447ac0d4f,
|
||||
.version = "0.26.3",
|
||||
.version = "0.26.8",
|
||||
.minimum_zig_version = "0.14.1",
|
||||
.paths = .{
|
||||
"build.zig",
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ wasmparser.workspace = true
|
|||
webbrowser.workspace = true
|
||||
|
||||
tree-sitter.workspace = true
|
||||
tree-sitter-generate.workspace = true
|
||||
tree-sitter-generate = { workspace = true, features = ["load"] }
|
||||
tree-sitter-config.workspace = true
|
||||
tree-sitter-highlight.workspace = true
|
||||
tree-sitter-loader.workspace = true
|
||||
|
|
|
|||
|
|
@ -19,9 +19,7 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
|
|||
static EXAMPLE_FILTER: LazyLock<Option<String>> =
|
||||
LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
|
||||
static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
|
||||
env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
|
||||
.map(|s| s.parse::<usize>().unwrap())
|
||||
.unwrap_or(5)
|
||||
env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
|
||||
});
|
||||
static TEST_LOADER: LazyLock<Loader> =
|
||||
LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
|
||||
|
|
|
|||
4
crates/cli/npm/package-lock.json
generated
4
crates/cli/npm/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.8",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.8",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.8",
|
||||
"author": {
|
||||
"name": "Max Brunsfeld",
|
||||
"email": "maxbrunsfeld@gmail.com"
|
||||
|
|
|
|||
|
|
@ -44,11 +44,13 @@ pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
|
|||
|
||||
pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
|
||||
|
||||
pub const DEFAULT_EDIT_COUNT: usize = 3;
|
||||
pub static EDIT_COUNT: LazyLock<usize> =
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(DEFAULT_EDIT_COUNT));
|
||||
|
||||
pub const DEFAULT_ITERATION_COUNT: usize = 10;
|
||||
pub static ITERATION_COUNT: LazyLock<usize> =
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(DEFAULT_ITERATION_COUNT));
|
||||
|
||||
fn int_env_var(name: &'static str) -> Option<usize> {
|
||||
env::var(name).ok().and_then(|e| e.parse().ok())
|
||||
|
|
@ -221,7 +223,7 @@ pub fn fuzz_language_corpus(
|
|||
}
|
||||
|
||||
// Perform a random series of edits and reparse.
|
||||
let edit_count = rand.unsigned(*EDIT_COUNT);
|
||||
let edit_count = rand.unsigned(options.edits);
|
||||
let mut undo_stack = Vec::with_capacity(edit_count);
|
||||
for _ in 0..=edit_count {
|
||||
let edit = get_random_edit(&mut rand, &input);
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
|
|||
use crc32fast::hash as crc32;
|
||||
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
|
||||
use indoc::{formatdoc, indoc};
|
||||
use log::warn;
|
||||
use log::info;
|
||||
use rand::{thread_rng, Rng};
|
||||
use semver::Version;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
|
|||
const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
|
||||
const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
|
||||
|
||||
const TREE_SITTER_JSON_SCHEMA: &str =
|
||||
pub const TREE_SITTER_JSON_SCHEMA: &str =
|
||||
"https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
|
|
@ -356,7 +356,7 @@ pub fn generate_grammar_files(
|
|||
"tree-sitter-cli":"#},
|
||||
);
|
||||
if !contents.contains("module") {
|
||||
warn!("Updating package.json");
|
||||
info!("Migrating package.json to ESM");
|
||||
contents = contents.replace(
|
||||
r#""repository":"#,
|
||||
indoc! {r#"
|
||||
|
|
@ -378,6 +378,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if contents.contains("module.exports") {
|
||||
info!("Migrating grammars.js to ESM");
|
||||
contents = contents.replace("module.exports =", "export default");
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
|
|
@ -393,10 +394,16 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Zig artifacts") {
|
||||
warn!("Replacing .gitignore");
|
||||
generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
|
||||
info!("Adding zig entries to .gitignore");
|
||||
contents.push('\n');
|
||||
contents.push_str(indoc! {"
|
||||
# Zig artifacts
|
||||
.zig-cache/
|
||||
zig-cache/
|
||||
zig-out/
|
||||
"});
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|
|
@ -409,8 +416,13 @@ pub fn generate_grammar_files(
|
|||
|path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents.replace("bindings/c/* ", "bindings/c/** ");
|
||||
let c_bindings_entry = "bindings/c/* ";
|
||||
if contents.contains(c_bindings_entry) {
|
||||
info!("Updating c bindings entry in .gitattributes");
|
||||
contents = contents.replace(c_bindings_entry, "bindings/c/** ");
|
||||
}
|
||||
if !contents.contains("Zig bindings") {
|
||||
info!("Adding zig entries to .gitattributes");
|
||||
contents.push('\n');
|
||||
contents.push_str(indoc! {"
|
||||
# Zig bindings
|
||||
|
|
@ -438,39 +450,40 @@ pub fn generate_grammar_files(
|
|||
}, |path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("#[cfg(with_highlights_query)]") {
|
||||
let replacement = indoc! {r#"
|
||||
#[cfg(with_highlights_query)]
|
||||
/// The syntax highlighting query for this grammar.
|
||||
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
|
||||
info!("Updating query constants in bindings/rust/lib.rs");
|
||||
let replacement = indoc! {r#"
|
||||
#[cfg(with_highlights_query)]
|
||||
/// The syntax highlighting query for this grammar.
|
||||
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_injections_query)]
|
||||
/// The language injection query for this grammar.
|
||||
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
|
||||
#[cfg(with_injections_query)]
|
||||
/// The language injection query for this grammar.
|
||||
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_locals_query)]
|
||||
/// The local variable query for this grammar.
|
||||
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
|
||||
#[cfg(with_locals_query)]
|
||||
/// The local variable query for this grammar.
|
||||
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_tags_query)]
|
||||
/// The symbol tagging query for this grammar.
|
||||
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
|
||||
"#}
|
||||
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
|
||||
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
|
||||
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
|
||||
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r#"
|
||||
// NOTE: uncomment these to include any queries that this grammar contains:
|
||||
#[cfg(with_tags_query)]
|
||||
/// The symbol tagging query for this grammar.
|
||||
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
|
||||
"#}
|
||||
.replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
|
||||
.replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
|
||||
.replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
|
||||
.replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r#"
|
||||
// NOTE: uncomment these to include any queries that this grammar contains:
|
||||
|
||||
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
|
||||
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
|
||||
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
|
||||
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
|
||||
"#},
|
||||
&replacement,
|
||||
);
|
||||
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
|
||||
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
|
||||
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
|
||||
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
|
||||
"#},
|
||||
&replacement,
|
||||
);
|
||||
}
|
||||
write_file(path, contents)?;
|
||||
Ok(())
|
||||
|
|
@ -483,6 +496,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("wasm32-unknown-unknown") {
|
||||
info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
|
||||
let replacement = indoc!{r#"
|
||||
c_config.flag("-utf-8");
|
||||
|
||||
|
|
@ -503,19 +517,18 @@ pub fn generate_grammar_files(
|
|||
wasm_src.join("string.c"),
|
||||
]);
|
||||
}
|
||||
"#};
|
||||
|
||||
let indented_replacement = replacement
|
||||
"#}
|
||||
.lines()
|
||||
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &indented_replacement);
|
||||
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement);
|
||||
}
|
||||
|
||||
// Introduce configuration variables for dynamic query inclusion
|
||||
if !contents.contains("with_highlights_query") {
|
||||
info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
|
||||
let replaced = indoc! {r#"
|
||||
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
|
||||
}"#}
|
||||
|
|
@ -542,10 +555,10 @@ pub fn generate_grammar_files(
|
|||
}
|
||||
}"#}
|
||||
.replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
|
||||
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
|
||||
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
|
||||
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
|
||||
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
|
||||
.replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
|
||||
.replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
|
||||
.replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
|
||||
.replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
|
||||
|
||||
contents = contents.replace(
|
||||
&replaced,
|
||||
|
|
@ -572,6 +585,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("\"LICENSE\"") {
|
||||
info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
|
||||
write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -592,7 +606,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Object.defineProperty") {
|
||||
warn!("Replacing index.js");
|
||||
info!("Replacing index.js");
|
||||
generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -606,7 +620,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("export default binding") {
|
||||
warn!("Replacing index.d.ts");
|
||||
info!("Replacing index.d.ts");
|
||||
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -627,7 +641,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("import") {
|
||||
warn!("Replacing binding_test.js");
|
||||
info!("Replacing binding_test.js");
|
||||
generate_file(
|
||||
path,
|
||||
BINDING_TEST_JS_TEMPLATE,
|
||||
|
|
@ -650,6 +664,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("fs.exists(") {
|
||||
info!("Replacing `fs.exists` calls in binding.gyp");
|
||||
write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -662,14 +677,17 @@ pub fn generate_grammar_files(
|
|||
|
||||
// Generate C bindings
|
||||
if tree_sitter_config.bindings.c {
|
||||
let kebab_case_name = language_name.to_kebab_case();
|
||||
missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
|
||||
let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
|
||||
let header_name = format!("tree-sitter-{kebab_case_name}.h");
|
||||
let old_file = &path.join(&header_name);
|
||||
if allow_update && fs::exists(old_file).unwrap_or(false) {
|
||||
info!("Removing bindings/c/{header_name}");
|
||||
fs::remove_file(old_file)?;
|
||||
}
|
||||
missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
|
||||
missing_path(
|
||||
include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
|
||||
include_path.join(&header_name),
|
||||
|path| {
|
||||
generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
|
||||
},
|
||||
|
|
@ -678,7 +696,7 @@ pub fn generate_grammar_files(
|
|||
})?;
|
||||
|
||||
missing_path(
|
||||
path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
|
||||
path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
|
||||
|path| {
|
||||
generate_file(
|
||||
path,
|
||||
|
|
@ -698,23 +716,27 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
|
||||
warn!("Replacing Makefile");
|
||||
info!("Replacing Makefile");
|
||||
generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
|
||||
} else {
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r"
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"},
|
||||
indoc! {r"
|
||||
$(SRC_DIR)/grammar.json: grammar.js
|
||||
$(TS) generate --no-parser $^
|
||||
let replaced = indoc! {r"
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"};
|
||||
if contents.contains(replaced) {
|
||||
info!("Adding --no-parser target to Makefile");
|
||||
contents = contents
|
||||
.replace(
|
||||
replaced,
|
||||
indoc! {r"
|
||||
$(SRC_DIR)/grammar.json: grammar.js
|
||||
$(TS) generate --no-parser $^
|
||||
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"}
|
||||
);
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"}
|
||||
);
|
||||
}
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -726,8 +748,8 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let replaced_contents = contents
|
||||
.replace("add_custom_target(test", "add_custom_target(ts-test")
|
||||
.replace(
|
||||
&formatdoc! {r#"
|
||||
|
|
@ -775,7 +797,10 @@ pub fn generate_grammar_files(
|
|||
COMMENT "Generating parser.c")
|
||||
"#}
|
||||
);
|
||||
write_file(path, contents)?;
|
||||
if !replaced_contents.eq(&contents) {
|
||||
info!("Updating CMakeLists.txt");
|
||||
write_file(path, replaced_contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -811,7 +836,8 @@ pub fn generate_grammar_files(
|
|||
// Generate Python bindings
|
||||
if tree_sitter_config.bindings.python {
|
||||
missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
|
||||
let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
|
||||
let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
|
||||
let lang_path = path.join(&snake_case_grammar_name);
|
||||
missing_path(&lang_path, create_dir)?;
|
||||
|
||||
missing_path_else(
|
||||
|
|
@ -821,6 +847,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("PyModuleDef_Init") {
|
||||
info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
|
||||
contents = contents
|
||||
.replace("PyModule_Create", "PyModuleDef_Init")
|
||||
.replace(
|
||||
|
|
@ -861,8 +888,8 @@ pub fn generate_grammar_files(
|
|||
},
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("uncomment these to include any queries") {
|
||||
warn!("Replacing __init__.py");
|
||||
if contents.contains("uncomment these to include any queries") {
|
||||
info!("Replacing __init__.py");
|
||||
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -876,9 +903,10 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if contents.contains("uncomment these to include any queries") {
|
||||
warn!("Replacing __init__.pyi");
|
||||
info!("Replacing __init__.pyi");
|
||||
generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
|
||||
} else if !contents.contains("CapsuleType") {
|
||||
info!("Updating __init__.pyi");
|
||||
contents = contents
|
||||
.replace(
|
||||
"from typing import Final",
|
||||
|
|
@ -910,6 +938,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Parser(Language(") {
|
||||
info!("Updating Language function in bindings/python/tests/test_binding.py");
|
||||
contents = contents
|
||||
.replace("tree_sitter.Language(", "Parser(Language(")
|
||||
.replace(".language())\n", ".language()))\n")
|
||||
|
|
@ -930,11 +959,19 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("build_ext") {
|
||||
warn!("Replacing setup.py");
|
||||
info!("Replacing setup.py");
|
||||
generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
if !contents.contains(" and not get_config_var") {
|
||||
info!("Updating Python free-threading support in setup.py");
|
||||
contents = contents.replace(
|
||||
r#"startswith("cp"):"#,
|
||||
r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
|
||||
);
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -953,6 +990,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("cp310-*") {
|
||||
info!("Updating dependencies in pyproject.toml");
|
||||
contents = contents
|
||||
.replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
|
||||
.replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
|
||||
|
|
@ -990,15 +1028,18 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let replaced_contents = contents
|
||||
.replace(
|
||||
"https://github.com/ChimeHQ/SwiftTreeSitter",
|
||||
"https://github.com/tree-sitter/swift-tree-sitter",
|
||||
)
|
||||
.replace("version: \"0.8.0\")", "version: \"0.9.0\")")
|
||||
.replace("(url:", "(name: \"SwiftTreeSitter\", url:");
|
||||
write_file(path, contents)?;
|
||||
if !replaced_contents.eq(&contents) {
|
||||
info!("Updating tree-sitter dependency in Package.swift");
|
||||
write_file(path, replaced_contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -1016,7 +1057,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("b.pkg_hash.len") {
|
||||
warn!("Replacing build.zig");
|
||||
info!("Replacing build.zig");
|
||||
generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1031,7 +1072,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains(".name = .tree_sitter_") {
|
||||
warn!("Replacing build.zig.zon");
|
||||
info!("Replacing build.zig.zon");
|
||||
generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1047,7 +1088,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("ts.Language") {
|
||||
warn!("Replacing root.zig");
|
||||
info!("Replacing root.zig");
|
||||
generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1189,17 +1230,20 @@ fn generate_file(
|
|||
.replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name)
|
||||
.replace(
|
||||
HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
|
||||
generate_opts.highlights_query_path,
|
||||
&generate_opts.highlights_query_path.replace('\\', "/"),
|
||||
)
|
||||
.replace(
|
||||
INJECTIONS_QUERY_PATH_PLACEHOLDER,
|
||||
generate_opts.injections_query_path,
|
||||
&generate_opts.injections_query_path.replace('\\', "/"),
|
||||
)
|
||||
.replace(
|
||||
LOCALS_QUERY_PATH_PLACEHOLDER,
|
||||
generate_opts.locals_query_path,
|
||||
&generate_opts.locals_query_path.replace('\\', "/"),
|
||||
)
|
||||
.replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
|
||||
.replace(
|
||||
TAGS_QUERY_PATH_PLACEHOLDER,
|
||||
&generate_opts.tags_query_path.replace('\\', "/"),
|
||||
);
|
||||
|
||||
if let Some(name) = generate_opts.author_name {
|
||||
replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);
|
||||
|
|
|
|||
|
|
@ -16,11 +16,11 @@ use semver::Version as SemverVersion;
|
|||
use tree_sitter::{ffi, Parser, Point};
|
||||
use tree_sitter_cli::{
|
||||
fuzz::{
|
||||
fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED,
|
||||
LOG_GRAPH_ENABLED, START_SEED,
|
||||
fuzz_language_corpus, FuzzOptions, DEFAULT_EDIT_COUNT, DEFAULT_ITERATION_COUNT, EDIT_COUNT,
|
||||
ITERATION_COUNT, LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED,
|
||||
},
|
||||
highlight::{self, HighlightOptions},
|
||||
init::{generate_grammar_files, JsonConfigOpts},
|
||||
init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
|
||||
input::{get_input, get_tmp_source_file, CliInput},
|
||||
logger,
|
||||
parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
|
||||
|
|
@ -234,7 +234,7 @@ struct Parse {
|
|||
/// Output the parse data in a pretty-printed CST format
|
||||
#[arg(long = "cst", short = 'c')]
|
||||
pub output_cst: bool,
|
||||
/// Show parsing statistic
|
||||
/// Show parsing statistics
|
||||
#[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")]
|
||||
pub stat: bool,
|
||||
/// Interrupt the parsing process by timeout (µs)
|
||||
|
|
@ -391,11 +391,15 @@ struct Fuzz {
|
|||
/// library's language function
|
||||
#[arg(long)]
|
||||
pub lang_name: Option<String>,
|
||||
/// Maximum number of edits to perform per fuzz test
|
||||
#[arg(long)]
|
||||
#[arg(
|
||||
long,
|
||||
help=format!("Maximum number of edits to perform per fuzz test (Default: {DEFAULT_EDIT_COUNT})")
|
||||
)]
|
||||
pub edits: Option<usize>,
|
||||
/// Number of fuzzing iterations to run per test
|
||||
#[arg(long)]
|
||||
#[arg(
|
||||
long,
|
||||
help=format!("Number of fuzzing iterations to run per test (Default: {DEFAULT_ITERATION_COUNT})")
|
||||
)]
|
||||
pub iterations: Option<usize>,
|
||||
/// Only fuzz corpus test cases whose name matches the given regex
|
||||
#[arg(long, short)]
|
||||
|
|
@ -867,10 +871,26 @@ impl Init {
|
|||
|
||||
(opts.name.clone(), Some(opts))
|
||||
} else {
|
||||
let mut json = serde_json::from_str::<TreeSitterJSON>(
|
||||
&fs::read_to_string(current_dir.join("tree-sitter.json"))
|
||||
.with_context(|| "Failed to read tree-sitter.json")?,
|
||||
)?;
|
||||
let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
|
||||
.with_context(|| "Failed to read tree-sitter.json")?;
|
||||
|
||||
let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
|
||||
if json.schema.is_none() {
|
||||
json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
|
||||
}
|
||||
|
||||
let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
|
||||
// Write the re-serialized config back, as newly added optional boolean fields
|
||||
// will be included with explicit `false`s rather than implict `null`s
|
||||
if self.update && !old_config.trim().eq(new_config.trim()) {
|
||||
info!("Updating tree-sitter.json");
|
||||
fs::write(
|
||||
current_dir.join("tree-sitter.json"),
|
||||
serde_json::to_string_pretty(&json)?,
|
||||
)
|
||||
.with_context(|| "Failed to write tree-sitter.json")?;
|
||||
}
|
||||
|
||||
(json.grammars.swap_remove(0).name, None)
|
||||
};
|
||||
|
||||
|
|
@ -955,11 +975,21 @@ impl Build {
|
|||
} else {
|
||||
let output_path = if let Some(ref path) = self.output {
|
||||
let path = Path::new(path);
|
||||
if path.is_absolute() {
|
||||
let full_path = if path.is_absolute() {
|
||||
path.to_path_buf()
|
||||
} else {
|
||||
current_dir.join(path)
|
||||
}
|
||||
};
|
||||
let parent_path = full_path
|
||||
.parent()
|
||||
.context("Output path must have a parent")?;
|
||||
let name = full_path
|
||||
.file_name()
|
||||
.context("Ouput path must have a filename")?;
|
||||
fs::create_dir_all(parent_path).context("Failed to create output path")?;
|
||||
let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
|
||||
canon_path.push(name);
|
||||
canon_path
|
||||
} else {
|
||||
let file_name = grammar_path
|
||||
.file_stem()
|
||||
|
|
@ -984,7 +1014,7 @@ impl Build {
|
|||
|
||||
loader
|
||||
.compile_parser_at_path(&grammar_path, output_path, flags)
|
||||
.unwrap();
|
||||
.context("Failed to compile parser")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1622,6 +1652,7 @@ impl Highlight {
|
|||
let loader_config = config.get()?;
|
||||
loader.find_all_languages(&loader_config)?;
|
||||
loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
|
||||
let cancellation_flag = util::cancel_on_signal();
|
||||
|
||||
|
|
@ -1702,7 +1733,6 @@ impl Highlight {
|
|||
} => {
|
||||
let path = get_tmp_source_file(&contents)?;
|
||||
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
let language = languages
|
||||
.iter()
|
||||
.find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
|
||||
|
|
@ -1733,7 +1763,6 @@ impl Highlight {
|
|||
if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
|
||||
(l, lc)
|
||||
} else {
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
let language = languages
|
||||
.first()
|
||||
.map(|(l, _)| l.clone())
|
||||
|
|
|
|||
|
|
@ -433,7 +433,7 @@ pub fn parse_file_at_path(
|
|||
let parse_duration = parse_time.elapsed();
|
||||
|
||||
let stdout = io::stdout();
|
||||
let mut stdout = stdout.lock();
|
||||
let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
|
||||
|
||||
if let Some(mut tree) = tree {
|
||||
if opts.debug_graph && !opts.edits.is_empty() {
|
||||
|
|
@ -510,12 +510,11 @@ pub fn parse_file_at_path(
|
|||
}
|
||||
}
|
||||
cursor.reset(tree.root_node());
|
||||
println!();
|
||||
writeln!(&mut stdout)?;
|
||||
}
|
||||
|
||||
if opts.output == ParseOutput::Cst {
|
||||
render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
|
||||
println!();
|
||||
}
|
||||
|
||||
if opts.output == ParseOutput::Xml {
|
||||
|
|
@ -582,11 +581,11 @@ pub fn parse_file_at_path(
|
|||
}
|
||||
let start = node.start_position();
|
||||
let end = node.end_position();
|
||||
write!(&mut stdout, " srow=\"{}\"", start.row)?;
|
||||
write!(&mut stdout, " scol=\"{}\"", start.column)?;
|
||||
write!(&mut stdout, " erow=\"{}\"", end.row)?;
|
||||
write!(&mut stdout, " ecol=\"{}\"", end.column)?;
|
||||
write!(&mut stdout, ">")?;
|
||||
write!(
|
||||
&mut stdout,
|
||||
" srow=\"{}\" scol=\"{}\" erow=\"{}\" ecol=\"{}\">",
|
||||
start.row, start.column, end.row, end.column
|
||||
)?;
|
||||
tags.push(node.kind());
|
||||
needs_newline = true;
|
||||
}
|
||||
|
|
@ -785,7 +784,7 @@ pub fn render_cst<'a, 'b: 'a>(
|
|||
.map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
|
||||
.max()
|
||||
.unwrap_or(1);
|
||||
let mut indent_level = 1;
|
||||
let mut indent_level = usize::from(!opts.no_ranges);
|
||||
let mut did_visit_children = false;
|
||||
let mut in_error = false;
|
||||
loop {
|
||||
|
|
@ -883,35 +882,24 @@ fn write_node_text(
|
|||
0
|
||||
};
|
||||
let formatted_line = render_line_feed(line, opts);
|
||||
if !opts.no_ranges {
|
||||
write!(
|
||||
out,
|
||||
"{}{}{}{}{}{}",
|
||||
if multiline { "\n" } else { "" },
|
||||
if multiline {
|
||||
render_node_range(opts, cursor, is_named, true, total_width, node_range)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if multiline {
|
||||
" ".repeat(indent_level + 1)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
paint(quote_color, &String::from(quote)),
|
||||
&paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
} else {
|
||||
write!(
|
||||
out,
|
||||
"\n{}{}{}{}",
|
||||
" ".repeat(indent_level + 1),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
&paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
}
|
||||
write!(
|
||||
out,
|
||||
"{}{}{}{}{}{}",
|
||||
if multiline { "\n" } else { " " },
|
||||
if multiline && !opts.no_ranges {
|
||||
render_node_range(opts, cursor, is_named, true, total_width, node_range)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if multiline {
|
||||
" ".repeat(indent_level + 1)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
paint(quote_color, &String::from(quote)),
|
||||
paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1011,10 +999,9 @@ fn cst_render_node(
|
|||
} else {
|
||||
opts.parse_theme.node_kind
|
||||
};
|
||||
write!(out, "{}", paint(kind_color, node.kind()),)?;
|
||||
write!(out, "{}", paint(kind_color, node.kind()))?;
|
||||
|
||||
if node.child_count() == 0 {
|
||||
write!(out, " ")?;
|
||||
// Node text from a pattern or external scanner
|
||||
write_node_text(
|
||||
opts,
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ pub fn query_file_at_path(
|
|||
test_summary: Option<&mut TestSummary>,
|
||||
) -> Result<()> {
|
||||
let stdout = io::stdout();
|
||||
let mut stdout = stdout.lock();
|
||||
let mut stdout = io::BufWriter::with_capacity(64 * 1024, stdout.lock());
|
||||
|
||||
let query_source = fs::read_to_string(query_path)
|
||||
.with_context(|| format!("Error reading query file {}", query_path.display()))?;
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class BuildExt(build_ext):
|
|||
class BdistWheel(bdist_wheel):
|
||||
def get_tag(self):
|
||||
python, abi, platform = super().get_tag()
|
||||
if python.startswith("cp"):
|
||||
if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
|
||||
python, abi = "cp310", "abi3"
|
||||
return python, abi, platform
|
||||
|
||||
|
|
|
|||
|
|
@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
|
|||
render_assertion_results("queries", &self.query_results)?;
|
||||
}
|
||||
|
||||
write!(f, "{}", self.parse_stats)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -605,11 +607,13 @@ pub fn run_tests_at_path(
|
|||
test_summary: &mut TestSummary,
|
||||
) -> Result<()> {
|
||||
let test_entry = parse_tests(&opts.path)?;
|
||||
let mut _log_session = None;
|
||||
|
||||
if opts.debug_graph {
|
||||
_log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?);
|
||||
} else if opts.debug {
|
||||
let _log_session = if opts.debug_graph {
|
||||
Some(util::log_graphs(parser, "log.html", opts.open_log)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if opts.debug {
|
||||
parser.set_logger(Some(Box::new(|log_type, message| {
|
||||
if log_type == LogType::Lex {
|
||||
io::stderr().write_all(b" ").unwrap();
|
||||
|
|
@ -867,13 +871,14 @@ fn run_tests(
|
|||
let tree = parser.parse(&input, None).unwrap();
|
||||
let parse_rate = {
|
||||
let parse_time = start.elapsed();
|
||||
let true_parse_rate = tree.root_node().byte_range().len() as f64
|
||||
/ (parse_time.as_nanos() as f64 / 1_000_000.0);
|
||||
let byte_len = tree.root_node().byte_range().len();
|
||||
let true_parse_rate =
|
||||
byte_len as f64 / (parse_time.as_nanos() as f64 / 1_000_000.0);
|
||||
let adj_parse_rate = adjusted_parse_rate(&tree, parse_time);
|
||||
|
||||
test_summary.parse_stats.total_parses += 1;
|
||||
test_summary.parse_stats.total_duration += parse_time;
|
||||
test_summary.parse_stats.total_bytes += tree.root_node().byte_range().len();
|
||||
test_summary.parse_stats.total_bytes += byte_len;
|
||||
|
||||
Some((true_parse_rate, adj_parse_rate))
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3075,6 +3075,74 @@ fn test_query_matches_with_deeply_nested_patterns_with_fields() {
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_alternation_with_inner_quantifier() {
|
||||
let language = get_language("c");
|
||||
let source_code = "#include <foo>
|
||||
#include <bar>
|
||||
#include <baz>
|
||||
|
||||
// comment";
|
||||
let matches = &[
|
||||
(
|
||||
0,
|
||||
vec![
|
||||
("capture", "#include <foo>\n"),
|
||||
("capture", "#include <bar>\n"),
|
||||
("capture", "#include <baz>\n"),
|
||||
],
|
||||
),
|
||||
(0, vec![("capture", "// comment")]),
|
||||
];
|
||||
|
||||
let query = "[
|
||||
(preproc_include)+
|
||||
(comment)
|
||||
] @capture";
|
||||
let query = Query::new(&language, query).unwrap();
|
||||
assert_query_matches(&language, &query, source_code, matches);
|
||||
|
||||
let query = "[
|
||||
(comment)
|
||||
(preproc_include)+
|
||||
] @capture";
|
||||
let query = Query::new(&language, query).unwrap();
|
||||
assert_query_matches(&language, &query, source_code, matches);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_alternation_with_outer_quantifier() {
|
||||
let language = get_language("c");
|
||||
let source_code = "#include <foo>
|
||||
#include <bar>
|
||||
#include <baz>
|
||||
|
||||
// comment";
|
||||
let matches = &[(
|
||||
0,
|
||||
vec![
|
||||
("capture", "#include <foo>\n"),
|
||||
("capture", "#include <bar>\n"),
|
||||
("capture", "#include <baz>\n"),
|
||||
("capture", "// comment"),
|
||||
],
|
||||
)];
|
||||
|
||||
let query = "[
|
||||
(preproc_include)
|
||||
(comment)
|
||||
]+ @capture";
|
||||
let query = Query::new(&language, query).unwrap();
|
||||
assert_query_matches(&language, &query, source_code, matches);
|
||||
|
||||
let query = "([
|
||||
(preproc_include)
|
||||
(comment)
|
||||
] (_)?)+ @capture";
|
||||
let query = Query::new(&language, query).unwrap();
|
||||
assert_query_matches(&language, &query, source_code, matches);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_matches_with_alternations_and_predicates() {
|
||||
allocations::record(|| {
|
||||
|
|
|
|||
|
|
@ -118,6 +118,32 @@ fn test_load_fixture_language_wasm() {
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_realloc_smaller_size() {
|
||||
allocations::record(|| {
|
||||
let store = WasmStore::new(&ENGINE).unwrap();
|
||||
let mut parser = Parser::new();
|
||||
let language = get_test_fixture_language_wasm("wasm_realloc_overflow_heap");
|
||||
parser.set_wasm_store(store).unwrap();
|
||||
parser.set_language(&language).unwrap();
|
||||
let tree = parser.parse("hello", None).unwrap();
|
||||
assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_realloc_clobber_region() {
|
||||
allocations::record(|| {
|
||||
let store = WasmStore::new(&ENGINE).unwrap();
|
||||
let mut parser = Parser::new();
|
||||
let language = get_test_fixture_language_wasm("wasm_realloc_clobber_region");
|
||||
parser.set_wasm_store(store).unwrap();
|
||||
parser.set_language(&language).unwrap();
|
||||
let tree = parser.parse("hello", None).unwrap();
|
||||
assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_multiple_wasm_languages() {
|
||||
allocations::record(|| {
|
||||
|
|
@ -273,6 +299,55 @@ fn test_load_wasm_errors() {
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_wasm_language_with_reserved_words() {
|
||||
// This test exercises a grammar with multiple reserved word sets loaded via WASM.
|
||||
allocations::record(|| {
|
||||
let store = WasmStore::new(&ENGINE).unwrap();
|
||||
let language = get_test_fixture_language_wasm("reserved_words");
|
||||
|
||||
let mut parser = Parser::new();
|
||||
parser.set_wasm_store(store).unwrap();
|
||||
parser.set_language(&language).unwrap();
|
||||
|
||||
// "if" and "while" are globally reserved, so using them as identifiers
|
||||
// should produce an error recovery.
|
||||
let tree = parser
|
||||
.parse("var a =\n\nif (something) {\n c();\n}", None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
tree.root_node().to_sexp(),
|
||||
concat!(
|
||||
"(program ",
|
||||
"(ERROR (identifier)) ",
|
||||
"(if_statement (parenthesized_expression (identifier)) ",
|
||||
"(block (expression_statement (call_expression (identifier))))))",
|
||||
)
|
||||
);
|
||||
|
||||
// "if" and "while" are NOT reserved in the 'property' context, so they
|
||||
// can appear as object keys without error.
|
||||
let tree = parser
|
||||
.parse("var x = {\n if: a,\n while: b,\n};", None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
tree.root_node().to_sexp(),
|
||||
concat!(
|
||||
"(program (var_declaration (identifier) (object ",
|
||||
"(pair (identifier) (identifier)) (pair (identifier) (identifier)))))"
|
||||
)
|
||||
);
|
||||
|
||||
// "var" IS reserved in the 'property' context, so using it as a property
|
||||
// key triggers error recovery.
|
||||
let tree = parser.parse("var x = {\nvar y = z;", None).unwrap();
|
||||
assert_eq!(
|
||||
tree.root_node().to_sexp(),
|
||||
"(program (ERROR (identifier)) (var_declaration (identifier) (identifier)))"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_oom() {
|
||||
allocations::record(|| {
|
||||
|
|
|
|||
|
|
@ -292,6 +292,9 @@ impl Version {
|
|||
} else {
|
||||
self.current_dir.join("Makefile")
|
||||
};
|
||||
if !makefile_path.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.update_file_with(&makefile_path, |content| {
|
||||
content
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use std::{
|
|||
};
|
||||
|
||||
use bitflags::bitflags;
|
||||
#[cfg(feature = "load")]
|
||||
use log::warn;
|
||||
use node_types::VariableInfo;
|
||||
use regex::{Regex, RegexBuilder};
|
||||
|
|
@ -41,7 +42,7 @@ pub use parse_grammar::ParseGrammarError;
|
|||
use prepare_grammar::prepare_grammar;
|
||||
pub use prepare_grammar::PrepareGrammarError;
|
||||
use render::render_c_code;
|
||||
pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
|
||||
pub use render::{RenderError, ABI_VERSION_MAX, ABI_VERSION_MIN};
|
||||
|
||||
static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
|
||||
RegexBuilder::new("^\\s*//.*")
|
||||
|
|
@ -93,6 +94,8 @@ pub enum GenerateError {
|
|||
VariableInfo(#[from] VariableInfoError),
|
||||
#[error(transparent)]
|
||||
BuildTables(#[from] ParseTableBuilderError),
|
||||
#[error(transparent)]
|
||||
Render(#[from] RenderError),
|
||||
#[cfg(feature = "load")]
|
||||
#[error(transparent)]
|
||||
ParseVersion(#[from] ParseVersionError),
|
||||
|
|
@ -106,6 +109,7 @@ pub struct IoError {
|
|||
pub path: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "load")]
|
||||
impl IoError {
|
||||
fn new(error: &std::io::Error, path: Option<&Path>) -> Self {
|
||||
Self {
|
||||
|
|
@ -398,7 +402,7 @@ fn generate_parser_for_grammar_with_opts(
|
|||
abi_version,
|
||||
semantic_version,
|
||||
supertype_symbol_map,
|
||||
);
|
||||
)?;
|
||||
Ok(GeneratedParser {
|
||||
c_code,
|
||||
#[cfg(feature = "load")]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
#[cfg(feature = "load")]
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
||||
use serde::Serialize;
|
||||
use thiserror::Error;
|
||||
|
|
@ -584,7 +586,13 @@ pub fn generate_node_types_json(
|
|||
kind: node_type_json.kind.clone(),
|
||||
named: true,
|
||||
};
|
||||
subtype_map.push((supertype, subtypes.clone()));
|
||||
|
||||
// We only add to the subtype map if there are visible subtypes.
|
||||
// A supertype may have zero subtypes if its children are all
|
||||
// hidden (e.g., wrapping a hidden external token).
|
||||
if !subtypes.is_empty() {
|
||||
subtype_map.push((supertype, subtypes.clone()));
|
||||
}
|
||||
node_type_json.subtypes = Some(subtypes);
|
||||
} else if !syntax_grammar.variables_to_inline.contains(&symbol) {
|
||||
// If a rule is aliased under multiple names, then its information
|
||||
|
|
@ -1256,6 +1264,49 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// A supertype whose only child is a hidden external token
|
||||
/// xgust not cause generation to panic. The subtype map must
|
||||
/// skip entries with empty subtypes to avoid a lookup failure
|
||||
/// in the topological sort.
|
||||
#[test]
|
||||
fn test_node_types_supertype_with_only_hidden_child() {
|
||||
let node_types = get_node_types(&InputGrammar {
|
||||
supertype_symbols: vec!["_type_a".to_string(), "_type_b".to_string()],
|
||||
variables: vec![
|
||||
Variable {
|
||||
name: "v1".to_string(),
|
||||
kind: VariableType::Named,
|
||||
rule: Rule::seq(vec![Rule::named("_type_a"), Rule::named("_type_b")]),
|
||||
},
|
||||
// Supertype A: a normal choice of named subtypes
|
||||
Variable {
|
||||
name: "_type_a".to_string(),
|
||||
kind: VariableType::Hidden,
|
||||
rule: Rule::choice(vec![Rule::named("v2"), Rule::named("v3")]),
|
||||
},
|
||||
Variable {
|
||||
name: "v2".to_string(),
|
||||
kind: VariableType::Named,
|
||||
rule: Rule::string("x"),
|
||||
},
|
||||
Variable {
|
||||
name: "v3".to_string(),
|
||||
kind: VariableType::Named,
|
||||
rule: Rule::string("y"),
|
||||
},
|
||||
// Supertype B: a hidden external token with no subtypes
|
||||
Variable {
|
||||
name: "_type_b".to_string(),
|
||||
kind: VariableType::Hidden,
|
||||
rule: Rule::external(0),
|
||||
},
|
||||
],
|
||||
external_tokens: vec![Rule::named("_hidden_ext")],
|
||||
..Default::default()
|
||||
});
|
||||
assert!(node_types.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_node_types_for_children_without_fields() {
|
||||
let node_types = get_node_types(&InputGrammar {
|
||||
|
|
|
|||
|
|
@ -274,9 +274,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
|
|||
};
|
||||
let matches_empty = match inner_rule {
|
||||
Rule::String(rule_str) => rule_str.is_empty(),
|
||||
Rule::Pattern(ref value, _) => Regex::new(value)
|
||||
.map(|reg| reg.is_match(""))
|
||||
.unwrap_or(false),
|
||||
Rule::Pattern(ref value, _) => Regex::new(value).is_ok_and(|reg| reg.is_match("")),
|
||||
_ => false,
|
||||
};
|
||||
if matches_empty {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ unless they are used only as the grammar's start rule.
|
|||
"
|
||||
)]
|
||||
EmptyString(String),
|
||||
#[error("Terminal rule '{0}' cannot be used as a supertype")]
|
||||
SupertypeTerminal(String),
|
||||
#[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
|
||||
ExternalTokenNonTerminal(String),
|
||||
#[error("Non-symbol rules cannot be used as external tokens")]
|
||||
|
|
@ -128,11 +130,18 @@ pub(super) fn extract_tokens(
|
|||
})
|
||||
.collect();
|
||||
|
||||
let supertype_symbols = grammar
|
||||
let supertype_symbols: Vec<Symbol> = grammar
|
||||
.supertype_symbols
|
||||
.into_iter()
|
||||
.map(|symbol| symbol_replacer.replace_symbol(symbol))
|
||||
.collect();
|
||||
for supertype_symbol in &supertype_symbols {
|
||||
if supertype_symbol.is_terminal() {
|
||||
Err(ExtractTokensError::SupertypeTerminal(
|
||||
lexical_variables[supertype_symbol.index].name.clone(),
|
||||
))?;
|
||||
}
|
||||
}
|
||||
|
||||
let variables_to_inline = grammar
|
||||
.variables_to_inline
|
||||
|
|
|
|||
|
|
@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
|
|||
let production_map = production_indices_by_step_id
|
||||
.into_iter()
|
||||
.map(|(step_id, production_indices)| {
|
||||
let production = step_id.variable_index.map_or_else(
|
||||
|| &productions[step_id.production_index],
|
||||
|variable_index| {
|
||||
&grammar.variables[variable_index].productions[step_id.production_index]
|
||||
},
|
||||
) as *const Production;
|
||||
let production =
|
||||
core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
|
||||
|| &productions[step_id.production_index],
|
||||
|variable_index| {
|
||||
&grammar.variables[variable_index].productions[step_id.production_index]
|
||||
},
|
||||
));
|
||||
((production, step_id.step_index as u32), production_indices)
|
||||
})
|
||||
.collect();
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ use std::{
|
|||
|
||||
use crate::LANGUAGE_VERSION;
|
||||
use indoc::indoc;
|
||||
use serde::Serialize;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::{
|
||||
build_tables::Tables,
|
||||
|
|
@ -25,6 +27,16 @@ pub const ABI_VERSION_MIN: usize = 14;
|
|||
pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION;
|
||||
const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
|
||||
|
||||
pub type RenderResult<T> = Result<T, RenderError>;
|
||||
|
||||
#[derive(Debug, Error, Serialize)]
|
||||
pub enum RenderError {
|
||||
#[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
|
||||
ParseTable(usize),
|
||||
#[error("This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {0}")]
|
||||
ABI(usize),
|
||||
}
|
||||
|
||||
#[clippy::format_args]
|
||||
macro_rules! add {
|
||||
($this: tt, $($arg: tt)*) => {{
|
||||
|
|
@ -104,7 +116,7 @@ struct Metadata {
|
|||
}
|
||||
|
||||
impl Generator {
|
||||
fn generate(mut self) -> String {
|
||||
fn generate(mut self) -> RenderResult<String> {
|
||||
self.init();
|
||||
self.add_header();
|
||||
self.add_includes();
|
||||
|
|
@ -161,7 +173,7 @@ impl Generator {
|
|||
self.add_reserved_word_sets();
|
||||
}
|
||||
|
||||
self.add_parse_table();
|
||||
self.add_parse_table()?;
|
||||
|
||||
if !self.syntax_grammar.external_tokens.is_empty() {
|
||||
self.add_external_token_enum();
|
||||
|
|
@ -171,7 +183,7 @@ impl Generator {
|
|||
|
||||
self.add_parser_export();
|
||||
|
||||
self.buffer
|
||||
Ok(self.buffer)
|
||||
}
|
||||
|
||||
fn init(&mut self) {
|
||||
|
|
@ -1273,7 +1285,7 @@ impl Generator {
|
|||
add_line!(self, "");
|
||||
}
|
||||
|
||||
fn add_parse_table(&mut self) {
|
||||
fn add_parse_table(&mut self) -> RenderResult<()> {
|
||||
let mut parse_table_entries = HashMap::new();
|
||||
let mut next_parse_action_list_index = 0;
|
||||
|
||||
|
|
@ -1443,6 +1455,9 @@ impl Generator {
|
|||
add_line!(self, "}};");
|
||||
add_line!(self, "");
|
||||
}
|
||||
if next_parse_action_list_index >= usize::from(u16::MAX) {
|
||||
Err(RenderError::ParseTable(next_parse_action_list_index))?;
|
||||
}
|
||||
|
||||
let mut parse_table_entries = parse_table_entries
|
||||
.into_iter()
|
||||
|
|
@ -1450,6 +1465,8 @@ impl Generator {
|
|||
.collect::<Vec<_>>();
|
||||
parse_table_entries.sort_by_key(|(index, _)| *index);
|
||||
self.add_parse_action_list(parse_table_entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
|
||||
|
|
@ -1942,11 +1959,10 @@ pub fn render_c_code(
|
|||
abi_version: usize,
|
||||
semantic_version: Option<(u8, u8, u8)>,
|
||||
supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
|
||||
) -> String {
|
||||
assert!(
|
||||
(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version),
|
||||
"This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {abi_version}",
|
||||
);
|
||||
) -> RenderResult<String> {
|
||||
if !(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version) {
|
||||
Err(RenderError::ABI(abi_version))?;
|
||||
}
|
||||
|
||||
Generator {
|
||||
language_name: name.to_string(),
|
||||
|
|
|
|||
|
|
@ -52,67 +52,96 @@ extern "C" {
|
|||
|
||||
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
|
||||
/// less than the array's current capacity, this function has no effect.
|
||||
#define array_reserve(self, new_capacity) \
|
||||
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
|
||||
#define array_reserve(self, new_capacity) \
|
||||
((self)->contents = _array__reserve( \
|
||||
(void *)(self)->contents, &(self)->capacity, \
|
||||
array_elem_size(self), new_capacity) \
|
||||
)
|
||||
|
||||
/// Free any memory allocated for this array. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_delete(self) _array__delete((Array *)(self))
|
||||
#define array_delete(self) \
|
||||
do { \
|
||||
if ((self)->contents) ts_free((self)->contents); \
|
||||
(self)->contents = NULL; \
|
||||
(self)->size = 0; \
|
||||
(self)->capacity = 0; \
|
||||
} while (0)
|
||||
|
||||
/// Push a new `element` onto the end of the array.
|
||||
#define array_push(self, element) \
|
||||
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
|
||||
(self)->contents[(self)->size++] = (element))
|
||||
#define array_push(self, element) \
|
||||
do { \
|
||||
(self)->contents = _array__grow( \
|
||||
(void *)(self)->contents, (self)->size, &(self)->capacity, \
|
||||
1, array_elem_size(self) \
|
||||
); \
|
||||
(self)->contents[(self)->size++] = (element); \
|
||||
} while(0)
|
||||
|
||||
/// Increase the array's size by `count` elements.
|
||||
/// New elements are zero-initialized.
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
_array__grow((Array *)(self), count, array_elem_size(self)); \
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
(self)->contents = _array__grow( \
|
||||
(self)->contents, (self)->size, &(self)->capacity, \
|
||||
count, array_elem_size(self) \
|
||||
); \
|
||||
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
|
||||
(self)->size += (count); \
|
||||
(self)->size += (count); \
|
||||
} while (0)
|
||||
|
||||
/// Append all elements from one array to the end of another.
|
||||
#define array_push_all(self, other) \
|
||||
#define array_push_all(self, other) \
|
||||
array_extend((self), (other)->size, (other)->contents)
|
||||
|
||||
/// Append `count` elements to the end of the array, reading their values from the
|
||||
/// `contents` pointer.
|
||||
#define array_extend(self, count, contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), (self)->size, \
|
||||
0, count, contents \
|
||||
#define array_extend(self, count, other_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), (self)->size, 0, count, other_contents \
|
||||
)
|
||||
|
||||
/// Remove `old_count` elements from the array starting at the given `index`. At
|
||||
/// the same index, insert `new_count` new elements, reading their values from the
|
||||
/// `new_contents` pointer.
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), _index, \
|
||||
old_count, new_count, new_contents \
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, old_count, new_count, new_contents \
|
||||
)
|
||||
|
||||
/// Insert one `element` into the array at the given `index`.
|
||||
#define array_insert(self, _index, element) \
|
||||
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
|
||||
#define array_insert(self, _index, element) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, 0, 1, &(element) \
|
||||
)
|
||||
|
||||
/// Remove one element from the array at the given `index`.
|
||||
#define array_erase(self, _index) \
|
||||
_array__erase((Array *)(self), array_elem_size(self), _index)
|
||||
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
|
||||
|
||||
/// Pop the last element off the array, returning the element by value.
|
||||
#define array_pop(self) ((self)->contents[--(self)->size])
|
||||
|
||||
/// Assign the contents of one array to another, reallocating if necessary.
|
||||
#define array_assign(self, other) \
|
||||
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
|
||||
#define array_assign(self, other) \
|
||||
(self)->contents = _array__assign( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
|
||||
)
|
||||
|
||||
/// Swap one array with another
|
||||
#define array_swap(self, other) \
|
||||
_array__swap((Array *)(self), (Array *)(other))
|
||||
#define array_swap(self, other) \
|
||||
do { \
|
||||
void *_array_swap_tmp = (void *)(self)->contents; \
|
||||
(self)->contents = (other)->contents; \
|
||||
(other)->contents = _array_swap_tmp; \
|
||||
_array__swap(&(self)->size, &(self)->capacity, \
|
||||
&(other)->size, &(other)->capacity); \
|
||||
} while (0)
|
||||
|
||||
/// Get the size of the array contents
|
||||
#define array_elem_size(self) (sizeof *(self)->contents)
|
||||
|
|
@ -157,82 +186,90 @@ extern "C" {
|
|||
|
||||
// Private
|
||||
|
||||
typedef Array(void) Array;
|
||||
|
||||
/// This is not what you're looking for, see `array_delete`.
|
||||
static inline void _array__delete(Array *self) {
|
||||
if (self->contents) {
|
||||
ts_free(self->contents);
|
||||
self->contents = NULL;
|
||||
self->size = 0;
|
||||
self->capacity = 0;
|
||||
}
|
||||
}
|
||||
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
|
||||
// are passed to the various `_array__*` functions below to address strict aliasing
|
||||
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
|
||||
//
|
||||
// The `Array` type itself was not altered as a solution in order to avoid breakage
|
||||
// with existing consumers (in particular, parsers with external scanners).
|
||||
|
||||
/// This is not what you're looking for, see `array_erase`.
|
||||
static inline void _array__erase(Array *self, size_t element_size,
|
||||
uint32_t index) {
|
||||
assert(index < self->size);
|
||||
char *contents = (char *)self->contents;
|
||||
static inline void _array__erase(void* self_contents, uint32_t *size,
|
||||
size_t element_size, uint32_t index) {
|
||||
assert(index < *size);
|
||||
char *contents = (char *)self_contents;
|
||||
memmove(contents + index * element_size, contents + (index + 1) * element_size,
|
||||
(self->size - index - 1) * element_size);
|
||||
self->size--;
|
||||
(*size - index - 1) * element_size);
|
||||
(*size)--;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_reserve`.
|
||||
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
|
||||
if (new_capacity > self->capacity) {
|
||||
if (self->contents) {
|
||||
self->contents = ts_realloc(self->contents, new_capacity * element_size);
|
||||
static inline void *_array__reserve(void *contents, uint32_t *capacity,
|
||||
size_t element_size, uint32_t new_capacity) {
|
||||
void *new_contents = contents;
|
||||
if (new_capacity > *capacity) {
|
||||
if (contents) {
|
||||
new_contents = ts_realloc(contents, new_capacity * element_size);
|
||||
} else {
|
||||
self->contents = ts_malloc(new_capacity * element_size);
|
||||
new_contents = ts_malloc(new_capacity * element_size);
|
||||
}
|
||||
self->capacity = new_capacity;
|
||||
*capacity = new_capacity;
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_assign`.
|
||||
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
|
||||
_array__reserve(self, element_size, other->size);
|
||||
self->size = other->size;
|
||||
memcpy(self->contents, other->contents, self->size * element_size);
|
||||
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
|
||||
const void *other_contents, uint32_t other_size, size_t element_size) {
|
||||
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
|
||||
*self_size = other_size;
|
||||
memcpy(new_contents, other_contents, *self_size * element_size);
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_swap`.
|
||||
static inline void _array__swap(Array *self, Array *other) {
|
||||
Array swap = *other;
|
||||
*other = *self;
|
||||
*self = swap;
|
||||
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
|
||||
uint32_t *other_size, uint32_t *other_capacity) {
|
||||
uint32_t tmp_size = *self_size;
|
||||
uint32_t tmp_capacity = *self_capacity;
|
||||
*self_size = *other_size;
|
||||
*self_capacity = *other_capacity;
|
||||
*other_size = tmp_size;
|
||||
*other_capacity = tmp_capacity;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
|
||||
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
|
||||
uint32_t new_size = self->size + count;
|
||||
if (new_size > self->capacity) {
|
||||
uint32_t new_capacity = self->capacity * 2;
|
||||
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
|
||||
uint32_t count, size_t element_size) {
|
||||
void *new_contents = contents;
|
||||
uint32_t new_size = size + count;
|
||||
if (new_size > *capacity) {
|
||||
uint32_t new_capacity = *capacity * 2;
|
||||
if (new_capacity < 8) new_capacity = 8;
|
||||
if (new_capacity < new_size) new_capacity = new_size;
|
||||
_array__reserve(self, element_size, new_capacity);
|
||||
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_splice`.
|
||||
static inline void _array__splice(Array *self, size_t element_size,
|
||||
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
|
||||
size_t element_size,
|
||||
uint32_t index, uint32_t old_count,
|
||||
uint32_t new_count, const void *elements) {
|
||||
uint32_t new_size = self->size + new_count - old_count;
|
||||
uint32_t new_size = *size + new_count - old_count;
|
||||
uint32_t old_end = index + old_count;
|
||||
uint32_t new_end = index + new_count;
|
||||
assert(old_end <= self->size);
|
||||
assert(old_end <= *size);
|
||||
|
||||
_array__reserve(self, element_size, new_size);
|
||||
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
|
||||
|
||||
char *contents = (char *)self->contents;
|
||||
if (self->size > old_end) {
|
||||
char *contents = (char *)new_contents;
|
||||
if (*size > old_end) {
|
||||
memmove(
|
||||
contents + new_end * element_size,
|
||||
contents + old_end * element_size,
|
||||
(self->size - old_end) * element_size
|
||||
(*size - old_end) * element_size
|
||||
);
|
||||
}
|
||||
if (new_count > 0) {
|
||||
|
|
@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
|
|||
);
|
||||
}
|
||||
}
|
||||
self->size += new_count - old_count;
|
||||
*size += new_count - old_count;
|
||||
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "tree-sitter-language"
|
||||
description = "The tree-sitter Language type, used by the library and by language implementations"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version = "1.77"
|
||||
|
|
|
|||
|
|
@ -23,9 +23,15 @@ typedef long unsigned int size_t;
|
|||
|
||||
typedef long unsigned int uintptr_t;
|
||||
|
||||
#define UINT16_MAX 65535
|
||||
#define INT8_MAX 127
|
||||
#define INT16_MAX 32767
|
||||
#define INT32_MAX 2147483647L
|
||||
#define INT64_MAX 9223372036854775807LL
|
||||
|
||||
#define UINT8_MAX 255
|
||||
#define UINT16_MAX 65535
|
||||
#define UINT32_MAX 4294967295U
|
||||
#define UINT64_MAX 18446744073709551615ULL
|
||||
|
||||
#if defined(__wasm32__)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
|
|||
|
||||
int strncmp(const char *left, const char *right, size_t n);
|
||||
|
||||
size_t strlen(const char *str);
|
||||
|
||||
#endif // TREE_SITTER_WASM_STRING_H_
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
bool left_justify; // -
|
||||
|
|
@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
|
|||
return 2 + len;
|
||||
}
|
||||
|
||||
size_t strlen(const char *str) {
|
||||
const char *s = str;
|
||||
while (*s) s++;
|
||||
return s - str;
|
||||
}
|
||||
|
||||
char *strncpy(char *dest, const char *src, size_t n) {
|
||||
char *d = dest;
|
||||
const char *s = src;
|
||||
|
|
|
|||
|
|
@ -48,6 +48,19 @@ static int grow_heap(size_t size) {
|
|||
return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX;
|
||||
}
|
||||
|
||||
// Grows the heap if necessary to fit a region at the _end_ of the heap
|
||||
// ending at `region_end` by `size` bytes.
|
||||
//
|
||||
// Returns 0 if the heap could not be grown, 1 otherwise.
|
||||
static inline int grow_heap_for_region(Region *region_end, size_t size) {
|
||||
if (region_end > heap_end) {
|
||||
if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) return 0;
|
||||
if (!grow_heap(size)) return 0;
|
||||
heap_end = get_heap_end();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Clear out the heap, and move it to the given address.
|
||||
void reset_heap(void *new_heap_start) {
|
||||
heap_start = new_heap_start;
|
||||
|
|
@ -76,13 +89,7 @@ void *malloc(size_t size) {
|
|||
|
||||
Region *region_end = region_after(next, size);
|
||||
|
||||
if (region_end > heap_end) {
|
||||
if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) {
|
||||
return NULL;
|
||||
}
|
||||
if (!grow_heap(size)) return NULL;
|
||||
heap_end = get_heap_end();
|
||||
}
|
||||
if (!grow_heap_for_region(region_end, size)) return NULL;
|
||||
|
||||
void *result = &next->data;
|
||||
next->size = size;
|
||||
|
|
@ -109,6 +116,7 @@ void free(void *ptr) {
|
|||
|
||||
void *calloc(size_t count, size_t size) {
|
||||
void *result = malloc(count * size);
|
||||
if (!result) return NULL;
|
||||
memset(result, 0, count * size);
|
||||
return result;
|
||||
}
|
||||
|
|
@ -117,19 +125,36 @@ void *realloc(void *ptr, size_t new_size) {
|
|||
if (ptr == NULL) {
|
||||
return malloc(new_size);
|
||||
}
|
||||
if (new_size == 0) {
|
||||
free(ptr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
Region *region = region_for_ptr(ptr);
|
||||
Region *region_end = region_after(region, region->size);
|
||||
|
||||
// When reallocating the last allocated region, return
|
||||
// the same pointer, and skip copying the data.
|
||||
// When reallocating the last allocated region, resize
|
||||
// in place if possible, return the same pointer, and
|
||||
// skip copying the data.
|
||||
if (region_end == next) {
|
||||
next = region;
|
||||
return malloc(new_size);
|
||||
Region *new_region_end = region_after(region, new_size);
|
||||
|
||||
size_t additional_size = (char *)new_region_end - (char *)heap_end;
|
||||
if (!grow_heap_for_region(new_region_end, additional_size)) return NULL;
|
||||
|
||||
region->size = new_size;
|
||||
next = new_region_end;
|
||||
return ®ion->data;
|
||||
}
|
||||
|
||||
void *result = malloc(new_size);
|
||||
memcpy(result, ®ion->data, region->size);
|
||||
if (!result) return NULL;
|
||||
|
||||
size_t copy_size = region->size < new_size ? region->size : new_size;
|
||||
memcpy(result, ®ion->data, copy_size);
|
||||
|
||||
free(ptr);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t strlen(const char *str) {
|
||||
const char *s = str;
|
||||
while (*s) s++;
|
||||
return s - str;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::fmt::Write as _;
|
||||
#[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))]
|
||||
use std::ops::Range;
|
||||
#[cfg(feature = "tree-sitter-highlight")]
|
||||
|
|
@ -8,6 +10,7 @@ use std::sync::Mutex;
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
env, fs,
|
||||
hash::{Hash as _, Hasher as _},
|
||||
io::{BufRead, BufReader},
|
||||
marker::PhantomData,
|
||||
mem,
|
||||
|
|
@ -75,8 +78,6 @@ pub enum LoaderError {
|
|||
NoLanguage,
|
||||
#[error(transparent)]
|
||||
Query(LoaderQueryError),
|
||||
#[error(transparent)]
|
||||
ScannerSymbols(ScannerSymbolError),
|
||||
#[error("Failed to load language for scope '{0}':\n{1}")]
|
||||
ScopeLoad(String, Box<Self>),
|
||||
#[error(transparent)]
|
||||
|
|
@ -199,28 +200,6 @@ impl std::fmt::Display for SymbolError {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub struct ScannerSymbolError {
|
||||
pub missing: Vec<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScannerSymbolError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
writeln!(
|
||||
f,
|
||||
"Missing required functions in the external scanner, parsing won't work without these!\n"
|
||||
)?;
|
||||
for symbol in &self.missing {
|
||||
writeln!(f, " `{symbol}`")?;
|
||||
}
|
||||
writeln!(
|
||||
f,
|
||||
"You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n"
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub struct WasiSDKClangError {
|
||||
pub wasi_sdk_dir: String,
|
||||
|
|
@ -1025,20 +1004,26 @@ impl Loader {
|
|||
return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
|
||||
}
|
||||
|
||||
// Create a unique lock path based on the output path hash to prevent
|
||||
// interference when multiple processes build the same grammar (by name)
|
||||
// to different output locations
|
||||
let lock_hash = {
|
||||
let mut hasher = std::hash::DefaultHasher::new();
|
||||
output_path.hash(&mut hasher);
|
||||
format!("{:x}", hasher.finish())
|
||||
};
|
||||
|
||||
let lock_path = if env::var("CROSS_RUNNER").is_ok() {
|
||||
tempfile::tempdir()
|
||||
.unwrap()
|
||||
.expect("create a temp dir")
|
||||
.path()
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}.lock", config.name))
|
||||
.to_path_buf()
|
||||
} else {
|
||||
etcetera::choose_base_strategy()?
|
||||
.cache_dir()
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}.lock", config.name))
|
||||
};
|
||||
etcetera::choose_base_strategy()?.cache_dir()
|
||||
}
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}-{lock_hash}.lock", config.name));
|
||||
|
||||
if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
|
||||
recompile = false;
|
||||
|
|
@ -1085,10 +1070,30 @@ impl Loader {
|
|||
self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?;
|
||||
|
||||
if config.scanner_path.is_some() {
|
||||
self.check_external_scanner(&config.name, &output_path)?;
|
||||
self.check_external_scanner(&output_path)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the dynamic library exists before trying to load it. This can
|
||||
// happen in race conditions where we couldn't acquire the lock because
|
||||
// another process was compiling but it still hasn't finished by the
|
||||
// time we reach this point, so the output file still doesn't exist.
|
||||
//
|
||||
// Instead of allowing the `load_language` call below to fail, return a
|
||||
// clearer error to the user here.
|
||||
if !output_path.exists() {
|
||||
let msg = format!(
|
||||
"Dynamic library `{}` not found after build attempt. \
|
||||
Are you running multiple processes building to the same output location?",
|
||||
output_path.display()
|
||||
);
|
||||
|
||||
Err(LoaderError::IO(IoError::new(
|
||||
std::io::Error::new(std::io::ErrorKind::NotFound, msg),
|
||||
Some(output_path.as_path()),
|
||||
)))?;
|
||||
}
|
||||
|
||||
Self::load_language(&output_path, &language_fn_name)
|
||||
}
|
||||
|
||||
|
|
@ -1187,6 +1192,9 @@ impl Loader {
|
|||
command.arg("-UTREE_SITTER_REUSE_ALLOCATOR");
|
||||
} else {
|
||||
command.arg("-shared");
|
||||
command.arg("-Wl,--no-undefined");
|
||||
#[cfg(target_os = "openbsd")]
|
||||
command.arg("-lc");
|
||||
}
|
||||
command.args(cc_config.get_files());
|
||||
command.arg("-o").arg(output_path);
|
||||
|
|
@ -1221,25 +1229,15 @@ impl Loader {
|
|||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> {
|
||||
let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) {
|
||||
"_"
|
||||
fn check_external_scanner(&self, library_path: &Path) -> LoaderResult<()> {
|
||||
let section = " T ";
|
||||
// Older ppc toolchains incorrectly report functions in the Data section. This bug has been
|
||||
// fixed, but we still need to account for older systems.
|
||||
let old_ppc_section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
|
||||
Some(" D ")
|
||||
} else {
|
||||
""
|
||||
None
|
||||
};
|
||||
let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
|
||||
" D "
|
||||
} else {
|
||||
" T "
|
||||
};
|
||||
let mut must_have = vec![
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_create"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_destroy"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_serialize"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_deserialize"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_scan"),
|
||||
];
|
||||
|
||||
let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned());
|
||||
let command = Command::new(nm_cmd)
|
||||
.arg("--defined-only")
|
||||
|
|
@ -1247,54 +1245,41 @@ impl Loader {
|
|||
.output();
|
||||
if let Ok(output) = command {
|
||||
if output.status.success() {
|
||||
let mut found_non_static = false;
|
||||
let mut non_static_symbols = String::new();
|
||||
for line in String::from_utf8_lossy(&output.stdout).lines() {
|
||||
if line.contains(section) {
|
||||
if line.contains(section) || old_ppc_section.is_some_and(|s| line.contains(s)) {
|
||||
if let Some(function_name) =
|
||||
line.split_whitespace().collect::<Vec<_>>().get(2)
|
||||
{
|
||||
if !line.contains("tree_sitter_") {
|
||||
if !found_non_static {
|
||||
found_non_static = true;
|
||||
warn!("Found non-static non-tree-sitter functions in the external scanner");
|
||||
}
|
||||
warn!(" `{function_name}`");
|
||||
} else {
|
||||
must_have.retain(|f| f != function_name);
|
||||
writeln!(&mut non_static_symbols, " `{function_name}`").unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if found_non_static {
|
||||
warn!(concat!(
|
||||
"Consider making these functions static, they can cause conflicts ",
|
||||
"when another tree-sitter project uses the same function name."
|
||||
));
|
||||
}
|
||||
|
||||
if !must_have.is_empty() {
|
||||
return Err(LoaderError::ScannerSymbols(ScannerSymbolError {
|
||||
missing: must_have,
|
||||
}));
|
||||
if !non_static_symbols.is_empty() {
|
||||
warn!(
|
||||
"Found non-static non-tree-sitter functions in the external scanner\n{non_static_symbols}\n{}",
|
||||
concat!(
|
||||
"Consider making these functions static, they can cause conflicts ",
|
||||
"when another tree-sitter project uses the same function name."
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
"Failed to run `nm` to verify symbols in {}",
|
||||
library_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> {
|
||||
fn check_external_scanner(&self, _library_path: &Path) -> LoaderResult<()> {
|
||||
// TODO: there's no nm command on windows, whoever wants to implement this can and should :)
|
||||
|
||||
// let mut must_have = vec![
|
||||
// format!("tree_sitter_{name}_external_scanner_create"),
|
||||
// format!("tree_sitter_{name}_external_scanner_destroy"),
|
||||
// format!("tree_sitter_{name}_external_scanner_serialize"),
|
||||
// format!("tree_sitter_{name}_external_scanner_deserialize"),
|
||||
// format!("tree_sitter_{name}_external_scanner_scan"),
|
||||
// ];
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -1309,6 +1294,7 @@ impl Loader {
|
|||
|
||||
let mut command = Command::new(&clang_executable);
|
||||
command.current_dir(src_path).args([
|
||||
"--target=wasm32-unknown-wasi",
|
||||
"-o",
|
||||
output_path.to_str().unwrap(),
|
||||
"-fPIC",
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
|
|||
"-D", "NDEBUG=",
|
||||
"-D", "_POSIX_C_SOURCE=200112L",
|
||||
"-D", "_DEFAULT_SOURCE=",
|
||||
"-D", "_BSD_SOURCE=",
|
||||
"-D", "_DARWIN_C_SOURCE=",
|
||||
"-I", "lib/src",
|
||||
"-I", "lib/include",
|
||||
|
|
|
|||
|
|
@ -1,19 +1,74 @@
|
|||
use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION};
|
||||
use crate::{bail_on_err, root_dir, EMSCRIPTEN_VERSION};
|
||||
use anyhow::Result;
|
||||
use std::{fs, process::Command};
|
||||
use std::{fs, path::Path, process::Command};
|
||||
|
||||
pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
||||
enum FixtureRef<'a> {
|
||||
Tag(&'a str),
|
||||
Branch(&'a str),
|
||||
}
|
||||
|
||||
impl<'a> FixtureRef<'a> {
|
||||
#[allow(clippy::use_self)]
|
||||
const fn new(tag: &'a str, branch: Option<&'a str>) -> FixtureRef<'a> {
|
||||
if let Some(b) = branch {
|
||||
Self::Branch(b)
|
||||
} else {
|
||||
Self::Tag(tag)
|
||||
}
|
||||
}
|
||||
|
||||
const fn ref_type(&self) -> &'static str {
|
||||
match self {
|
||||
FixtureRef::Tag(_) => "tag",
|
||||
FixtureRef::Branch(_) => "branch",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FixtureRef<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FixtureRef::Tag(tag) => write!(f, "{tag}"),
|
||||
FixtureRef::Branch(branch) => write!(f, "{branch}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn current_ref_name(grammar_dir: &Path) -> Result<(String, Option<&'static str>)> {
|
||||
let tag_args = ["describe", "--tags", "--exact-match", "HEAD"];
|
||||
let branch_args = ["rev-parse", "--abbrev-ref", "HEAD"];
|
||||
|
||||
for (args, ref_type) in [tag_args.as_ref(), branch_args.as_ref()]
|
||||
.iter()
|
||||
.zip(&["tag", "branch"])
|
||||
{
|
||||
let name_cmd = Command::new("git")
|
||||
.current_dir(grammar_dir)
|
||||
.args(*args)
|
||||
.output()?;
|
||||
let name = String::from_utf8_lossy(&name_cmd.stdout);
|
||||
let name = name.trim();
|
||||
if !name.is_empty() {
|
||||
return Ok((name.to_string(), Some(ref_type)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(("<unknown>".to_string(), None))
|
||||
}
|
||||
|
||||
pub fn run_fixtures() -> Result<()> {
|
||||
let fixtures_dir = root_dir().join("test").join("fixtures");
|
||||
let grammars_dir = fixtures_dir.join("grammars");
|
||||
let fixtures_path = fixtures_dir.join("fixtures.json");
|
||||
|
||||
// grammar name, tag
|
||||
let mut fixtures: Vec<(String, String)> =
|
||||
// grammar name, tag, [branch]
|
||||
let fixtures: Vec<(String, String, Option<String>)> =
|
||||
serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?;
|
||||
|
||||
for (grammar, tag) in &mut fixtures {
|
||||
let grammar_dir = grammars_dir.join(&grammar);
|
||||
for (grammar, tag, branch) in &fixtures {
|
||||
let grammar_dir = grammars_dir.join(grammar);
|
||||
let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}");
|
||||
let target_ref = FixtureRef::new(tag, branch.as_deref());
|
||||
|
||||
println!("Fetching the {grammar} grammar...");
|
||||
|
||||
|
|
@ -24,7 +79,7 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
|||
"--depth",
|
||||
"1",
|
||||
"--branch",
|
||||
tag,
|
||||
&target_ref.to_string(),
|
||||
&grammar_url,
|
||||
&grammar_dir.to_string_lossy(),
|
||||
]);
|
||||
|
|
@ -33,31 +88,71 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
|||
&format!("Failed to clone the {grammar} grammar"),
|
||||
)?;
|
||||
} else {
|
||||
let mut describe_command = Command::new("git");
|
||||
describe_command.current_dir(&grammar_dir).args([
|
||||
"describe",
|
||||
"--tags",
|
||||
"--exact-match",
|
||||
"HEAD",
|
||||
]);
|
||||
let (current_ref, current_ref_type) = current_ref_name(&grammar_dir)?;
|
||||
if current_ref != target_ref.to_string() {
|
||||
println!(
|
||||
"Updating {grammar} grammar from {} {current_ref} to {} {target_ref}...",
|
||||
current_ref_type.unwrap_or("<unknown>"),
|
||||
target_ref.ref_type(),
|
||||
);
|
||||
|
||||
let output = describe_command.output()?;
|
||||
let current_tag = String::from_utf8_lossy(&output.stdout);
|
||||
let current_tag = current_tag.trim();
|
||||
|
||||
if current_tag != tag {
|
||||
println!("Updating {grammar} grammar from {current_tag} to {tag}...");
|
||||
|
||||
let mut fetch_command = Command::new("git");
|
||||
fetch_command.current_dir(&grammar_dir).args([
|
||||
"fetch",
|
||||
"origin",
|
||||
&format!("refs/tags/{tag}:refs/tags/{tag}"),
|
||||
]);
|
||||
bail_on_err(
|
||||
&fetch_command.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to fetch tag {tag} for {grammar} grammar"),
|
||||
)?;
|
||||
match target_ref {
|
||||
FixtureRef::Branch(branch) => {
|
||||
let mut fetch_cmd = Command::new("git");
|
||||
fetch_cmd.current_dir(&grammar_dir).args([
|
||||
"fetch",
|
||||
"--update-shallow",
|
||||
"origin",
|
||||
&format!("+refs/heads/{branch}:refs/remotes/origin/{branch}"),
|
||||
]);
|
||||
bail_on_err(
|
||||
&fetch_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to fetch branch {branch}"),
|
||||
)?;
|
||||
let mut switch_cmd = Command::new("git");
|
||||
switch_cmd
|
||||
.current_dir(&grammar_dir)
|
||||
.args(["switch", branch]);
|
||||
bail_on_err(
|
||||
&switch_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to checkout branch {branch}"),
|
||||
)?;
|
||||
let mut set_upstream_cmd = Command::new("git");
|
||||
set_upstream_cmd.current_dir(&grammar_dir).args([
|
||||
"branch",
|
||||
"--set-upstream-to",
|
||||
&format!("origin/{branch}"),
|
||||
branch,
|
||||
]);
|
||||
bail_on_err(
|
||||
&set_upstream_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to set upstream for branch {branch}"),
|
||||
)?;
|
||||
let mut pull_cmd = Command::new("git");
|
||||
pull_cmd
|
||||
.current_dir(&grammar_dir)
|
||||
.args(["pull", "origin", branch]);
|
||||
bail_on_err(
|
||||
&pull_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to pull latest from branch {branch}"),
|
||||
)?;
|
||||
}
|
||||
FixtureRef::Tag(tag) => {
|
||||
let mut fetch_command = Command::new("git");
|
||||
fetch_command.current_dir(&grammar_dir).args([
|
||||
"fetch",
|
||||
"origin",
|
||||
&format!("refs/tags/{tag}:refs/tags/{tag}"),
|
||||
]);
|
||||
bail_on_err(
|
||||
&fetch_command.spawn()?.wait_with_output()?,
|
||||
&format!(
|
||||
"Failed to fetch {} {target_ref} for {grammar} grammar",
|
||||
target_ref.ref_type()
|
||||
),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut reset_command = Command::new("git");
|
||||
reset_command
|
||||
|
|
@ -71,29 +166,23 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
|||
let mut checkout_command = Command::new("git");
|
||||
checkout_command
|
||||
.current_dir(&grammar_dir)
|
||||
.args(["checkout", tag]);
|
||||
.args(["checkout", &target_ref.to_string()]);
|
||||
bail_on_err(
|
||||
&checkout_command.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to checkout tag {tag} for {grammar} grammar"),
|
||||
&format!(
|
||||
"Failed to checkout {} {target_ref} for {grammar} grammar",
|
||||
target_ref.ref_type()
|
||||
),
|
||||
)?;
|
||||
} else {
|
||||
println!("{grammar} grammar is already at tag {tag}");
|
||||
println!(
|
||||
"{grammar} grammar is already at {} {target_ref}",
|
||||
target_ref.ref_type()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if args.update {
|
||||
println!("Updating the fixtures lock file");
|
||||
fs::write(
|
||||
&fixtures_path,
|
||||
// format the JSON without extra newlines
|
||||
serde_json::to_string(&fixtures)?
|
||||
.replace("[[", "[\n [")
|
||||
.replace("],", "],\n ")
|
||||
.replace("]]", "]\n]"),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ enum Commands {
|
|||
/// Fetches emscripten.
|
||||
FetchEmscripten,
|
||||
/// Fetches the fixtures for testing tree-sitter.
|
||||
FetchFixtures(FetchFixtures),
|
||||
FetchFixtures,
|
||||
/// Generate the Rust bindings from the C library.
|
||||
GenerateBindings,
|
||||
/// Generates the fixtures for testing tree-sitter.
|
||||
|
|
@ -118,13 +118,6 @@ struct Clippy {
|
|||
package: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct FetchFixtures {
|
||||
/// Update all fixtures to the latest tag
|
||||
#[arg(long, short)]
|
||||
update: bool,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct GenerateFixtures {
|
||||
/// Generates the parser to Wasm
|
||||
|
|
@ -232,8 +225,8 @@ fn run() -> Result<()> {
|
|||
Commands::CheckWasmExports(check_options) => check_wasm_exports::run(&check_options)?,
|
||||
Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?,
|
||||
Commands::FetchEmscripten => fetch::run_emscripten()?,
|
||||
Commands::FetchFixtures(fetch_fixture_options) => {
|
||||
fetch::run_fixtures(&fetch_fixture_options)?;
|
||||
Commands::FetchFixtures => {
|
||||
fetch::run_fixtures()?;
|
||||
}
|
||||
Commands::GenerateBindings => generate::run_bindings()?,
|
||||
Commands::GenerateFixtures(generate_fixtures_options) => {
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
|
|||
}
|
||||
|
||||
window.initializePlayground = async (opts) => {
|
||||
const { Parser, Language } = window.TreeSitter;
|
||||
const { Parser, Language, Query } = window.TreeSitter;
|
||||
|
||||
const { local } = opts;
|
||||
if (local) {
|
||||
|
|
@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
|
|||
marks.forEach((m) => m.clear());
|
||||
|
||||
if (tree && query) {
|
||||
const captures = query.captures(
|
||||
tree.rootNode,
|
||||
{ row: startRow, column: 0 },
|
||||
{ row: endRow, column: 0 },
|
||||
);
|
||||
const captures = query.captures(tree.rootNode, {
|
||||
startPosition: { row: startRow, column: 0 },
|
||||
endPosition: { row: endRow, column: 0 },
|
||||
});
|
||||
let lastNodeId;
|
||||
for (const { name, node } of captures) {
|
||||
if (node.id === lastNodeId) continue;
|
||||
|
|
@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
|
|||
const queryText = queryEditor.getValue();
|
||||
|
||||
try {
|
||||
query = parser.language.query(queryText);
|
||||
query = new Query(parser.language, queryText);
|
||||
let match;
|
||||
|
||||
let row = 0;
|
||||
|
|
|
|||
|
|
@ -31,11 +31,13 @@ If `--lib-path` is used, the name of the language used to extract the library's
|
|||
|
||||
### `--edits <EDITS>`
|
||||
|
||||
The maximum number of edits to perform. The default is 3.
|
||||
The maximum number of edits to perform. The default is 3. This value can also be set via the `TREE_SITTER_EDITS` environment
|
||||
variable.
|
||||
|
||||
### `--iterations <ITERATIONS>`
|
||||
|
||||
The number of iterations to run. The default is 10.
|
||||
The number of iterations to run. The default is 10. This value can also be set via the `TREE_SITTER_ITERATIONS` environment
|
||||
variable.
|
||||
|
||||
### `-i/--include <INCLUDE>`
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,10 @@ cd tree-sitter-${LOWER_PARSER_NAME}
|
|||
The `LOWER_` prefix here means the "lowercase" name of the language.
|
||||
```
|
||||
|
||||
```admonish warning
|
||||
Dashes are not permitted via the CLI's `init` command and should not be used in parser names.
|
||||
```
|
||||
|
||||
### Init
|
||||
|
||||
Once you've installed the `tree-sitter` CLI tool, you can start setting up your project, which will allow your parser to
|
||||
|
|
|
|||
|
|
@ -110,6 +110,69 @@ This pattern would match a set of possible keyword tokens, capturing them as `@k
|
|||
] @keyword
|
||||
```
|
||||
|
||||
Alternations can have quantified alternants, and then can have their own
|
||||
quantifiers as well. See the following examples for an illustration of how these
|
||||
cases work:
|
||||
|
||||
```query
|
||||
;;; SOURCE CODE ;;;
|
||||
; #include <foo>
|
||||
; #include <bar>
|
||||
; #include <baz>
|
||||
; // comment
|
||||
;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
[
|
||||
(preproc_include)
|
||||
(comment)
|
||||
]+ @capture
|
||||
; ^ Produces one match with four captures:
|
||||
; [
|
||||
; "#include <foo>\n",
|
||||
; "#include <bar>\n",
|
||||
; "#include <baz>\n",
|
||||
; "// comment",
|
||||
; ]
|
||||
;
|
||||
; Regex equivalent: [ab]+
|
||||
|
||||
[
|
||||
(preproc_include)+
|
||||
(comment)
|
||||
] @capture
|
||||
; ^ Produces two matches; one with three captures, and one with one capture:
|
||||
; [
|
||||
; "#include <foo>\n",
|
||||
; "#include <bar>\n",
|
||||
; "#include <baz>\n",
|
||||
; ],
|
||||
; [
|
||||
; "// comment",
|
||||
; ]
|
||||
;
|
||||
; Regex equivalent: a+|b
|
||||
|
||||
[
|
||||
(preproc_include)
|
||||
(comment)
|
||||
] @capture
|
||||
; ^ Produces four matches, each with one capture:
|
||||
; [
|
||||
; "#include <foo>\n",
|
||||
; ],
|
||||
; [
|
||||
; "#include <bar>\n",
|
||||
; ],
|
||||
; [
|
||||
; "#include <baz>\n",
|
||||
; ],
|
||||
; [
|
||||
; "// comment",
|
||||
; ]
|
||||
;
|
||||
; Regex equivalent: [ab]
|
||||
```
|
||||
|
||||
## Anchors
|
||||
|
||||
The anchor operator, `.`, is used to constrain the ways in which child patterns are matched. It has different behaviors
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
eachSystem = lib.genAttrs systems;
|
||||
pkgsFor = inputs.nixpkgs.legacyPackages;
|
||||
|
||||
version = "0.26.3";
|
||||
version = "0.26.8";
|
||||
|
||||
fs = lib.fileset;
|
||||
src = fs.toSource {
|
||||
|
|
@ -333,6 +333,7 @@
|
|||
pkg-config
|
||||
llvm
|
||||
clang
|
||||
clang-tools
|
||||
libclang
|
||||
|
||||
nodejs_22
|
||||
|
|
@ -348,6 +349,7 @@
|
|||
];
|
||||
|
||||
shellHook = ''
|
||||
export PATH="${pkgs.clang-tools}/bin:$PATH"
|
||||
echo "Tree-sitter Dev Environment"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ tree-sitter-language.workspace = true
|
|||
streaming-iterator = "0.1.9"
|
||||
|
||||
[dependencies.wasmtime-c-api]
|
||||
version = "33.0.2"
|
||||
version = "36.0.6"
|
||||
optional = true
|
||||
package = "wasmtime-c-api-impl"
|
||||
default-features = false
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ pub struct TSLogger {
|
|||
),
|
||||
>,
|
||||
}
|
||||
#[doc = " A summary of a change to a text document.\n\n The `start_byte` and `start_point` values must be less than or equal to the\n `old_end_byte` and `old_end_point` values, respectively. Passing an edit\n that violates these invariants may produce nonsensical results."]
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct TSInputEdit {
|
||||
|
|
@ -298,7 +299,7 @@ extern "C" {
|
|||
pub fn ts_tree_included_ranges(self_: *const TSTree, length: *mut u32) -> *mut TSRange;
|
||||
}
|
||||
extern "C" {
|
||||
#[doc = " Edit the syntax tree to keep it in sync with source code that has been\n edited.\n\n You must describe the edit both in terms of byte offsets and in terms of\n (row, column) coordinates."]
|
||||
#[doc = " Edit the syntax tree to keep it in sync with source code that has been\n edited.\n\n You must describe the edit both in terms of byte offsets and in terms of\n (row, column) coordinates.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
|
||||
pub fn ts_tree_edit(self_: *mut TSTree, edit: *const TSInputEdit);
|
||||
}
|
||||
extern "C" {
|
||||
|
|
@ -488,7 +489,7 @@ extern "C" {
|
|||
) -> TSNode;
|
||||
}
|
||||
extern "C" {
|
||||
#[doc = " Edit the node to keep it in-sync with source code that has been edited.\n\n This function is only rarely needed. When you edit a syntax tree with the\n [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree\n afterward will already reflect the edit. You only need to use [`ts_node_edit`]\n when you have a [`TSNode`] instance that you want to keep and continue to use\n after an edit."]
|
||||
#[doc = " Edit the node to keep it in-sync with source code that has been edited.\n\n This function is only rarely needed. When you edit a syntax tree with the\n [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree\n afterward will already reflect the edit. You only need to use [`ts_node_edit`]\n when you have a [`TSNode`] instance that you want to keep and continue to use\n after an edit.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
|
||||
pub fn ts_node_edit(self_: *mut TSNode, edit: *const TSInputEdit);
|
||||
}
|
||||
extern "C" {
|
||||
|
|
@ -496,11 +497,11 @@ extern "C" {
|
|||
pub fn ts_node_eq(self_: TSNode, other: TSNode) -> bool;
|
||||
}
|
||||
extern "C" {
|
||||
#[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance."]
|
||||
#[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
|
||||
pub fn ts_point_edit(point: *mut TSPoint, point_byte: *mut u32, edit: *const TSInputEdit);
|
||||
}
|
||||
extern "C" {
|
||||
#[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance."]
|
||||
#[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance.\n\n The edit's `start_byte` must be less than or equal to its `old_end_byte`,\n and its `start_point` must be less than or equal to its `old_end_point`."]
|
||||
pub fn ts_range_edit(range: *mut TSRange, edit: *const TSInputEdit);
|
||||
}
|
||||
extern "C" {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ fn main() {
|
|||
.include(&include_path)
|
||||
.define("_POSIX_C_SOURCE", "200112L")
|
||||
.define("_DEFAULT_SOURCE", None)
|
||||
.define("_BSD_SOURCE", None)
|
||||
.define("_DARWIN_C_SOURCE", None)
|
||||
.warnings(false)
|
||||
.file(src_path.join("lib.c"))
|
||||
|
|
|
|||
|
|
@ -3746,11 +3746,11 @@ impl fmt::Display for QueryError {
|
|||
#[must_use]
|
||||
pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
|
||||
let mut indent_level = initial_indent_level;
|
||||
let mut formatted = String::new();
|
||||
let mut formatted = String::with_capacity(sexp.len());
|
||||
let mut has_field = false;
|
||||
|
||||
let mut c_iter = sexp.chars().peekable();
|
||||
let mut s = String::with_capacity(sexp.len());
|
||||
let mut scratch = String::with_capacity(sexp.len());
|
||||
let mut quote = '\0';
|
||||
let mut saw_paren = false;
|
||||
let mut did_last = false;
|
||||
|
|
@ -3796,12 +3796,12 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
|
|||
Some(())
|
||||
};
|
||||
|
||||
while fetch_next_str(&mut s).is_some() {
|
||||
if s.is_empty() && indent_level > 0 {
|
||||
while fetch_next_str(&mut scratch).is_some() {
|
||||
if scratch.is_empty() && indent_level > 0 {
|
||||
// ")"
|
||||
indent_level -= 1;
|
||||
write!(formatted, ")").unwrap();
|
||||
} else if s.starts_with('(') {
|
||||
} else if scratch.starts_with('(') {
|
||||
if has_field {
|
||||
has_field = false;
|
||||
} else {
|
||||
|
|
@ -3815,27 +3815,27 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
|
|||
}
|
||||
|
||||
// "(node_name"
|
||||
write!(formatted, "{s}").unwrap();
|
||||
write!(formatted, "{scratch}").unwrap();
|
||||
|
||||
// "(MISSING node_name" or "(UNEXPECTED 'x'"
|
||||
if s.starts_with("(MISSING") || s.starts_with("(UNEXPECTED") {
|
||||
fetch_next_str(&mut s).unwrap();
|
||||
if s.is_empty() {
|
||||
if scratch.starts_with("(MISSING") || scratch.starts_with("(UNEXPECTED") {
|
||||
fetch_next_str(&mut scratch).unwrap();
|
||||
if scratch.is_empty() {
|
||||
while indent_level > 0 {
|
||||
indent_level -= 1;
|
||||
write!(formatted, ")").unwrap();
|
||||
}
|
||||
} else {
|
||||
write!(formatted, " {s}").unwrap();
|
||||
write!(formatted, " {scratch}").unwrap();
|
||||
}
|
||||
}
|
||||
} else if s.ends_with(':') {
|
||||
} else if scratch.ends_with(':') {
|
||||
// "field:"
|
||||
writeln!(formatted).unwrap();
|
||||
for _ in 0..indent_level {
|
||||
write!(formatted, " ").unwrap();
|
||||
}
|
||||
write!(formatted, "{s} ").unwrap();
|
||||
write!(formatted, "{scratch} ").unwrap();
|
||||
has_field = true;
|
||||
indent_level += 1;
|
||||
}
|
||||
|
|
|
|||
4
lib/binding_web/package-lock.json
generated
4
lib/binding_web/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.8",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.8",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.8",
|
||||
"description": "Tree-sitter bindings for the web",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ async function build() {
|
|||
keepNames: true,
|
||||
external: ['fs/*', 'fs/promises'],
|
||||
resolveExtensions: ['.ts', '.js', format === 'esm' ? '.mjs' : '.cjs'],
|
||||
...(format === 'cjs' ? {
|
||||
footer: { js: 'module.exports.default = module.exports;' },
|
||||
} : {}),
|
||||
});
|
||||
|
||||
// Copy the Wasm files to the appropriate spot, as esbuild doesn't "bundle" Wasm files
|
||||
|
|
|
|||
|
|
@ -114,6 +114,13 @@ typedef struct TSLogger {
|
|||
void (*log)(void *payload, TSLogType log_type, const char *buffer);
|
||||
} TSLogger;
|
||||
|
||||
/**
|
||||
* A summary of a change to a text document.
|
||||
*
|
||||
* The `start_byte` and `start_point` values must be less than or equal to the
|
||||
* `old_end_byte` and `old_end_point` values, respectively. Passing an edit
|
||||
* that violates these invariants may produce nonsensical results.
|
||||
*/
|
||||
typedef struct TSInputEdit {
|
||||
uint32_t start_byte;
|
||||
uint32_t old_end_byte;
|
||||
|
|
@ -437,6 +444,9 @@ TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length);
|
|||
*
|
||||
* You must describe the edit both in terms of byte offsets and in terms of
|
||||
* (row, column) coordinates.
|
||||
*
|
||||
* The edit's `start_byte` must be less than or equal to its `old_end_byte`,
|
||||
* and its `start_point` must be less than or equal to its `old_end_point`.
|
||||
*/
|
||||
void ts_tree_edit(TSTree *self, const TSInputEdit *edit);
|
||||
|
||||
|
|
@ -700,6 +710,9 @@ TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPo
|
|||
* afterward will already reflect the edit. You only need to use [`ts_node_edit`]
|
||||
* when you have a [`TSNode`] instance that you want to keep and continue to use
|
||||
* after an edit.
|
||||
*
|
||||
* The edit's `start_byte` must be less than or equal to its `old_end_byte`,
|
||||
* and its `start_point` must be less than or equal to its `old_end_point`.
|
||||
*/
|
||||
void ts_node_edit(TSNode *self, const TSInputEdit *edit);
|
||||
|
||||
|
|
@ -714,6 +727,9 @@ bool ts_node_eq(TSNode self, TSNode other);
|
|||
* This function updates a single point's byte offset and row/column position
|
||||
* based on an edit operation. This is useful for editing points without
|
||||
* requiring a tree or node instance.
|
||||
*
|
||||
* The edit's `start_byte` must be less than or equal to its `old_end_byte`,
|
||||
* and its `start_point` must be less than or equal to its `old_end_point`.
|
||||
*/
|
||||
void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit);
|
||||
|
||||
|
|
@ -723,6 +739,9 @@ void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit
|
|||
* This function updates a range's start and end positions based on an edit
|
||||
* operation. This is useful for editing ranges without requiring a tree
|
||||
* or node instance.
|
||||
*
|
||||
* The edit's `start_byte` must be less than or equal to its `old_end_byte`,
|
||||
* and its `start_point` must be less than or equal to its `old_end_point`.
|
||||
*/
|
||||
void ts_range_edit(TSRange *range, const TSInputEdit *edit);
|
||||
|
||||
|
|
|
|||
181
lib/src/array.h
181
lib/src/array.h
|
|
@ -52,67 +52,96 @@ extern "C" {
|
|||
|
||||
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
|
||||
/// less than the array's current capacity, this function has no effect.
|
||||
#define array_reserve(self, new_capacity) \
|
||||
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
|
||||
#define array_reserve(self, new_capacity) \
|
||||
((self)->contents = _array__reserve( \
|
||||
(void *)(self)->contents, &(self)->capacity, \
|
||||
array_elem_size(self), new_capacity) \
|
||||
)
|
||||
|
||||
/// Free any memory allocated for this array. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_delete(self) _array__delete((Array *)(self))
|
||||
#define array_delete(self) \
|
||||
do { \
|
||||
if ((self)->contents) ts_free((self)->contents); \
|
||||
(self)->contents = NULL; \
|
||||
(self)->size = 0; \
|
||||
(self)->capacity = 0; \
|
||||
} while (0)
|
||||
|
||||
/// Push a new `element` onto the end of the array.
|
||||
#define array_push(self, element) \
|
||||
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
|
||||
(self)->contents[(self)->size++] = (element))
|
||||
#define array_push(self, element) \
|
||||
do { \
|
||||
(self)->contents = _array__grow( \
|
||||
(void *)(self)->contents, (self)->size, &(self)->capacity, \
|
||||
1, array_elem_size(self) \
|
||||
); \
|
||||
(self)->contents[(self)->size++] = (element); \
|
||||
} while(0)
|
||||
|
||||
/// Increase the array's size by `count` elements.
|
||||
/// New elements are zero-initialized.
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
_array__grow((Array *)(self), count, array_elem_size(self)); \
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
(self)->contents = _array__grow( \
|
||||
(self)->contents, (self)->size, &(self)->capacity, \
|
||||
count, array_elem_size(self) \
|
||||
); \
|
||||
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
|
||||
(self)->size += (count); \
|
||||
(self)->size += (count); \
|
||||
} while (0)
|
||||
|
||||
/// Append all elements from one array to the end of another.
|
||||
#define array_push_all(self, other) \
|
||||
#define array_push_all(self, other) \
|
||||
array_extend((self), (other)->size, (other)->contents)
|
||||
|
||||
/// Append `count` elements to the end of the array, reading their values from the
|
||||
/// `contents` pointer.
|
||||
#define array_extend(self, count, contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), (self)->size, \
|
||||
0, count, contents \
|
||||
#define array_extend(self, count, other_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), (self)->size, 0, count, other_contents \
|
||||
)
|
||||
|
||||
/// Remove `old_count` elements from the array starting at the given `index`. At
|
||||
/// the same index, insert `new_count` new elements, reading their values from the
|
||||
/// `new_contents` pointer.
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), _index, \
|
||||
old_count, new_count, new_contents \
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, old_count, new_count, new_contents \
|
||||
)
|
||||
|
||||
/// Insert one `element` into the array at the given `index`.
|
||||
#define array_insert(self, _index, element) \
|
||||
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
|
||||
#define array_insert(self, _index, element) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, 0, 1, &(element) \
|
||||
)
|
||||
|
||||
/// Remove one element from the array at the given `index`.
|
||||
#define array_erase(self, _index) \
|
||||
_array__erase((Array *)(self), array_elem_size(self), _index)
|
||||
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
|
||||
|
||||
/// Pop the last element off the array, returning the element by value.
|
||||
#define array_pop(self) ((self)->contents[--(self)->size])
|
||||
|
||||
/// Assign the contents of one array to another, reallocating if necessary.
|
||||
#define array_assign(self, other) \
|
||||
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
|
||||
#define array_assign(self, other) \
|
||||
(self)->contents = _array__assign( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
|
||||
)
|
||||
|
||||
/// Swap one array with another
|
||||
#define array_swap(self, other) \
|
||||
_array__swap((Array *)(self), (Array *)(other))
|
||||
#define array_swap(self, other) \
|
||||
do { \
|
||||
void *_array_swap_tmp = (void *)(self)->contents; \
|
||||
(self)->contents = (other)->contents; \
|
||||
(other)->contents = _array_swap_tmp; \
|
||||
_array__swap(&(self)->size, &(self)->capacity, \
|
||||
&(other)->size, &(other)->capacity); \
|
||||
} while (0)
|
||||
|
||||
/// Get the size of the array contents
|
||||
#define array_elem_size(self) (sizeof *(self)->contents)
|
||||
|
|
@ -157,82 +186,90 @@ extern "C" {
|
|||
|
||||
// Private
|
||||
|
||||
typedef Array(void) Array;
|
||||
|
||||
/// This is not what you're looking for, see `array_delete`.
|
||||
static inline void _array__delete(Array *self) {
|
||||
if (self->contents) {
|
||||
ts_free(self->contents);
|
||||
self->contents = NULL;
|
||||
self->size = 0;
|
||||
self->capacity = 0;
|
||||
}
|
||||
}
|
||||
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
|
||||
// are passed to the various `_array__*` functions below to address strict aliasing
|
||||
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
|
||||
//
|
||||
// The `Array` type itself was not altered as a solution in order to avoid breakage
|
||||
// with existing consumers (in particular, parsers with external scanners).
|
||||
|
||||
/// This is not what you're looking for, see `array_erase`.
|
||||
static inline void _array__erase(Array *self, size_t element_size,
|
||||
uint32_t index) {
|
||||
ts_assert(index < self->size);
|
||||
char *contents = (char *)self->contents;
|
||||
static inline void _array__erase(void* self_contents, uint32_t *size,
|
||||
size_t element_size, uint32_t index) {
|
||||
ts_assert(index < *size);
|
||||
char *contents = (char *)self_contents;
|
||||
memmove(contents + index * element_size, contents + (index + 1) * element_size,
|
||||
(self->size - index - 1) * element_size);
|
||||
self->size--;
|
||||
(*size - index - 1) * element_size);
|
||||
(*size)--;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_reserve`.
|
||||
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
|
||||
if (new_capacity > self->capacity) {
|
||||
if (self->contents) {
|
||||
self->contents = ts_realloc(self->contents, new_capacity * element_size);
|
||||
static inline void *_array__reserve(void *contents, uint32_t *capacity,
|
||||
size_t element_size, uint32_t new_capacity) {
|
||||
void *new_contents = contents;
|
||||
if (new_capacity > *capacity) {
|
||||
if (contents) {
|
||||
new_contents = ts_realloc(contents, new_capacity * element_size);
|
||||
} else {
|
||||
self->contents = ts_malloc(new_capacity * element_size);
|
||||
new_contents = ts_malloc(new_capacity * element_size);
|
||||
}
|
||||
self->capacity = new_capacity;
|
||||
*capacity = new_capacity;
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_assign`.
|
||||
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
|
||||
_array__reserve(self, element_size, other->size);
|
||||
self->size = other->size;
|
||||
memcpy(self->contents, other->contents, self->size * element_size);
|
||||
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
|
||||
const void *other_contents, uint32_t other_size, size_t element_size) {
|
||||
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
|
||||
*self_size = other_size;
|
||||
memcpy(new_contents, other_contents, *self_size * element_size);
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_swap`.
|
||||
static inline void _array__swap(Array *self, Array *other) {
|
||||
Array swap = *other;
|
||||
*other = *self;
|
||||
*self = swap;
|
||||
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
|
||||
uint32_t *other_size, uint32_t *other_capacity) {
|
||||
uint32_t tmp_size = *self_size;
|
||||
uint32_t tmp_capacity = *self_capacity;
|
||||
*self_size = *other_size;
|
||||
*self_capacity = *other_capacity;
|
||||
*other_size = tmp_size;
|
||||
*other_capacity = tmp_capacity;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
|
||||
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
|
||||
uint32_t new_size = self->size + count;
|
||||
if (new_size > self->capacity) {
|
||||
uint32_t new_capacity = self->capacity * 2;
|
||||
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
|
||||
uint32_t count, size_t element_size) {
|
||||
void *new_contents = contents;
|
||||
uint32_t new_size = size + count;
|
||||
if (new_size > *capacity) {
|
||||
uint32_t new_capacity = *capacity * 2;
|
||||
if (new_capacity < 8) new_capacity = 8;
|
||||
if (new_capacity < new_size) new_capacity = new_size;
|
||||
_array__reserve(self, element_size, new_capacity);
|
||||
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_splice`.
|
||||
static inline void _array__splice(Array *self, size_t element_size,
|
||||
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
|
||||
size_t element_size,
|
||||
uint32_t index, uint32_t old_count,
|
||||
uint32_t new_count, const void *elements) {
|
||||
uint32_t new_size = self->size + new_count - old_count;
|
||||
uint32_t new_size = *size + new_count - old_count;
|
||||
uint32_t old_end = index + old_count;
|
||||
uint32_t new_end = index + new_count;
|
||||
ts_assert(old_end <= self->size);
|
||||
ts_assert(old_end <= *size);
|
||||
|
||||
_array__reserve(self, element_size, new_size);
|
||||
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
|
||||
|
||||
char *contents = (char *)self->contents;
|
||||
if (self->size > old_end) {
|
||||
char *contents = (char *)new_contents;
|
||||
if (*size > old_end) {
|
||||
memmove(
|
||||
contents + new_end * element_size,
|
||||
contents + old_end * element_size,
|
||||
(self->size - old_end) * element_size
|
||||
(*size - old_end) * element_size
|
||||
);
|
||||
}
|
||||
if (new_count > 0) {
|
||||
|
|
@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
|
|||
);
|
||||
}
|
||||
}
|
||||
self->size += new_count - old_count;
|
||||
*size += new_count - old_count;
|
||||
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
|
||||
|
|
|
|||
|
|
@ -2179,7 +2179,7 @@ balance:
|
|||
ts_assert(self->finished_tree.ptr);
|
||||
if (!ts_parser__balance_subtree(self)) {
|
||||
self->canceled_balancing = true;
|
||||
return false;
|
||||
return NULL;
|
||||
}
|
||||
self->canceled_balancing = false;
|
||||
LOG("done");
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
defined(__GNU__) || \
|
||||
defined(__HAIKU__) || \
|
||||
defined(__illumos__) || \
|
||||
defined(__redox__) || \
|
||||
defined(__NetBSD__) || \
|
||||
defined(__OpenBSD__) || \
|
||||
defined(__CYGWIN__) || \
|
||||
|
|
|
|||
106
lib/src/query.c
106
lib/src/query.c
|
|
@ -56,15 +56,15 @@ typedef struct {
|
|||
* Steps have some additional fields in order to handle the `.` (or "anchor") operator,
|
||||
* which forbids additional child nodes:
|
||||
* - `is_immediate` - Indicates that the node matching this step cannot be preceded
|
||||
* by other sibling nodes that weren't specified in the pattern.
|
||||
* by other sibling nodes that weren't specified in the pattern.
|
||||
* - `is_last_child` - Indicates that the node matching this step cannot have any
|
||||
* subsequent named siblings.
|
||||
* subsequent named siblings.
|
||||
*
|
||||
* For simple patterns, steps are matched in sequential order. But in order to
|
||||
* handle alternative/repeated/optional sub-patterns, query steps are not always
|
||||
* structured as a linear sequence; they sometimes need to split and merge. This
|
||||
* is done using the following fields:
|
||||
* - `alternative_index` - The index of a different query step that serves as
|
||||
* - `alternative_index` - The index of a different query step that serves as
|
||||
* an alternative to this step. A `NONE` value represents no alternative.
|
||||
* When a query state reaches a step with an alternative index, the state
|
||||
* is duplicated, with one copy remaining at the original step, and one copy
|
||||
|
|
@ -75,21 +75,22 @@ typedef struct {
|
|||
* - `is_pass_through` - Indicates that state has no matching logic of its own,
|
||||
* and exists only to split a state. One copy of the state advances immediately
|
||||
* to the next step, and one moves to the alternative step.
|
||||
* - `alternative_is_immediate` - Indicates that this step's alternative step
|
||||
* should be treated as if `is_immediate` is true.
|
||||
* - `is_inside_alternation` - Indicates that state is inside an alternation.
|
||||
* Currently only written to quantifier steps, read by logic that maintains
|
||||
* correctness for quantifiers inside alternations.
|
||||
*
|
||||
* Steps also store some derived state that summarizes how they relate to other
|
||||
* steps within the same pattern. This is used to optimize the matching process:
|
||||
* - `contains_captures` - Indicates that this step or one of its child steps
|
||||
* has a non-empty `capture_ids` list.
|
||||
* - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
|
||||
* it and all of its subsequent sibling steps within the same parent pattern
|
||||
* are guaranteed to match.
|
||||
* - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
|
||||
* for the entire top-level pattern. When iterating through a query's
|
||||
* captures using `ts_query_cursor_next_capture`, this field is used to
|
||||
* detect that a capture can safely be returned from a match that has not
|
||||
* even completed yet.
|
||||
* - `contains_captures` - Indicates that this step or one of its child steps
|
||||
* has a non-empty `capture_ids` list.
|
||||
* - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
|
||||
* it and all of its subsequent sibling steps within the same parent pattern
|
||||
* are guaranteed to match.
|
||||
* - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
|
||||
* for the entire top-level pattern. When iterating through a query's
|
||||
* captures using `ts_query_cursor_next_capture`, this field is used to
|
||||
* detect that a capture can safely be returned from a match that has not
|
||||
* even completed yet.
|
||||
*/
|
||||
typedef struct {
|
||||
TSSymbol symbol;
|
||||
|
|
@ -104,7 +105,7 @@ typedef struct {
|
|||
bool is_last_child: 1;
|
||||
bool is_pass_through: 1;
|
||||
bool is_dead_end: 1;
|
||||
bool alternative_is_immediate: 1;
|
||||
bool is_inside_alternation: 1;
|
||||
bool contains_captures: 1;
|
||||
bool root_pattern_guaranteed: 1;
|
||||
bool parent_pattern_guaranteed: 1;
|
||||
|
|
@ -816,17 +817,8 @@ static QueryStep query_step__new(
|
|||
QueryStep step = {
|
||||
.symbol = symbol,
|
||||
.depth = depth,
|
||||
.field = 0,
|
||||
.alternative_index = NONE,
|
||||
.negated_field_list_id = 0,
|
||||
.contains_captures = false,
|
||||
.is_last_child = false,
|
||||
.is_named = false,
|
||||
.is_pass_through = false,
|
||||
.is_dead_end = false,
|
||||
.root_pattern_guaranteed = false,
|
||||
.is_immediate = is_immediate,
|
||||
.alternative_is_immediate = false,
|
||||
};
|
||||
for (unsigned i = 0; i < MAX_STEP_CAPTURE_COUNT; i++) {
|
||||
step.capture_ids[i] = NONE;
|
||||
|
|
@ -2231,6 +2223,7 @@ static TSQueryError ts_query__parse_pattern(
|
|||
Stream *stream,
|
||||
uint32_t depth,
|
||||
bool is_immediate,
|
||||
bool is_inside_alternation,
|
||||
CaptureQuantifiers *capture_quantifiers
|
||||
) {
|
||||
if (stream->next == 0) return TSQueryErrorSyntax;
|
||||
|
|
@ -2264,6 +2257,7 @@ static TSQueryError ts_query__parse_pattern(
|
|||
stream,
|
||||
depth,
|
||||
is_immediate,
|
||||
true,
|
||||
&branch_capture_quantifiers
|
||||
);
|
||||
|
||||
|
|
@ -2331,6 +2325,7 @@ static TSQueryError ts_query__parse_pattern(
|
|||
stream,
|
||||
depth,
|
||||
child_is_immediate,
|
||||
is_inside_alternation,
|
||||
&child_capture_quantifiers
|
||||
);
|
||||
if (e == PARENT_DONE) {
|
||||
|
|
@ -2569,6 +2564,7 @@ static TSQueryError ts_query__parse_pattern(
|
|||
stream,
|
||||
depth + 1,
|
||||
child_is_immediate,
|
||||
is_inside_alternation,
|
||||
&child_capture_quantifiers
|
||||
);
|
||||
// In the event we only parsed a predicate, meaning no new steps were added,
|
||||
|
|
@ -2680,6 +2676,7 @@ static TSQueryError ts_query__parse_pattern(
|
|||
stream,
|
||||
depth,
|
||||
is_immediate,
|
||||
is_inside_alternation,
|
||||
&field_capture_quantifiers
|
||||
);
|
||||
if (e) {
|
||||
|
|
@ -2798,16 +2795,16 @@ static TSQueryError ts_query__parse_pattern(
|
|||
switch (quantifier) {
|
||||
case TSQuantifierOneOrMore:
|
||||
repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false);
|
||||
repeat_step.is_inside_alternation = is_inside_alternation;
|
||||
repeat_step.alternative_index = starting_step_index;
|
||||
repeat_step.is_pass_through = true;
|
||||
repeat_step.alternative_is_immediate = true;
|
||||
array_push(&self->steps, repeat_step);
|
||||
break;
|
||||
case TSQuantifierZeroOrMore:
|
||||
repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false);
|
||||
repeat_step.is_inside_alternation = is_inside_alternation;
|
||||
repeat_step.alternative_index = starting_step_index;
|
||||
repeat_step.is_pass_through = true;
|
||||
repeat_step.alternative_is_immediate = true;
|
||||
array_push(&self->steps, repeat_step);
|
||||
|
||||
// Stop when `step->alternative_index` is `NONE` or it points to
|
||||
|
|
@ -2884,7 +2881,7 @@ TSQuery *ts_query_new(
|
|||
.is_non_local = false,
|
||||
}));
|
||||
CaptureQuantifiers capture_quantifiers = capture_quantifiers_new();
|
||||
*error_type = ts_query__parse_pattern(self, &stream, 0, false, &capture_quantifiers);
|
||||
*error_type = ts_query__parse_pattern(self, &stream, 0, false, false, &capture_quantifiers);
|
||||
array_push(&self->steps, query_step__new(0, PATTERN_DONE_MARKER, false));
|
||||
|
||||
QueryPattern *pattern = array_back(&self->patterns);
|
||||
|
|
@ -2958,6 +2955,53 @@ TSQuery *ts_query_new(
|
|||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Fix up quantifier loop-backs within alternations. When a branch of an
|
||||
// alternation has a + or * quantifier, the quantifier's pass_through step
|
||||
// loops back to the branch's first step. However, the alternation linking
|
||||
// assigns that same step's `alternative_index` to point to the _next_ branch.
|
||||
// This causes the quantifier loop to incorrectly explore other alternation branches,
|
||||
// when a quantified branch matches, loops back, and then fails to match. To correct
|
||||
// this, we create "clean" copies of the branches' first steps without the link to the
|
||||
// next branch. After a quantified branch matches, it loops back to the cleaned copy.
|
||||
{
|
||||
uint32_t pat_start = pattern->steps.offset;
|
||||
uint32_t pat_end = pat_start + pattern->steps.length - 1; // exclude DONE
|
||||
|
||||
for (uint32_t i = pat_start; i < pat_end; i++) {
|
||||
QueryStep *s = array_get(&self->steps, i);
|
||||
// Ensure this step is a pass_through with a _backward_ alternative (a quantifier loop-back)
|
||||
if (!s->is_pass_through || !s->is_inside_alternation
|
||||
|| s->alternative_index == NONE || s->alternative_index >= i) continue;
|
||||
|
||||
uint32_t target_idx = s->alternative_index;
|
||||
QueryStep *target = array_get(&self->steps, target_idx);
|
||||
|
||||
// Check if the target has a forward alternative from alternation linking
|
||||
uint16_t target_alt_index = target->alternative_index;
|
||||
if (target_alt_index == NONE
|
||||
|| target_alt_index <= target_idx || target_alt_index >= pat_end) continue;
|
||||
|
||||
// Create a clean copy of the target step without the alternation alternative.
|
||||
uint32_t copy_idx = self->steps.size;
|
||||
QueryStep copy = *target;
|
||||
copy.alternative_index = NONE;
|
||||
uint16_t target_depth = target->depth;
|
||||
array_push(&self->steps, copy);
|
||||
|
||||
// Add a dead_end that redirects to the pass through step after the target,
|
||||
// so the pattern continues correctly after the cleaned copy matches.
|
||||
QueryStep redirect = query_step__new(0, target_depth, false);
|
||||
redirect.is_dead_end = true;
|
||||
redirect.alternative_index = target_idx + 1;
|
||||
array_push(&self->steps, redirect);
|
||||
|
||||
// Update the pass_through to loop back to the copy. Reacquire `s` since
|
||||
// `self->steps` may have been reallocated.
|
||||
s = array_get(&self->steps, i);
|
||||
s->alternative_index = copy_idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!ts_query__analyze_patterns(self, error_offset)) {
|
||||
|
|
@ -4130,17 +4174,17 @@ static inline bool ts_query_cursor__advance(
|
|||
QueryState *copy = ts_query_cursor__copy_state(self, &child_state);
|
||||
if (copy) {
|
||||
LOG(
|
||||
" split state for branch. pattern:%u, from_step:%u, to_step:%u, immediate:%d, capture_count: %u\n",
|
||||
" split state for branch. pattern:%u, from_step:%u, to_step:%u, pass_through:%d, capture_count:%u\n",
|
||||
copy->pattern_index,
|
||||
copy->step_index,
|
||||
next_step->alternative_index,
|
||||
next_step->alternative_is_immediate,
|
||||
next_step->is_pass_through,
|
||||
capture_list_pool_get(&self->capture_list_pool, copy->capture_list_id)->size
|
||||
);
|
||||
end_index++;
|
||||
copy_count++;
|
||||
copy->step_index = child_step->alternative_index;
|
||||
if (child_step->alternative_is_immediate) {
|
||||
if (child_step->is_pass_through) {
|
||||
copy->seeking_immediate_match = true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
|
|||
// Get a subtree's children, which are allocated immediately before the
|
||||
// tree's own heap data.
|
||||
#define ts_subtree_children(self) \
|
||||
((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
|
||||
((self).data.is_inline ? (Subtree *)NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
|
||||
|
||||
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) {
|
||||
if (self->data.is_inline) {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -80,15 +80,15 @@ typedef struct {
|
|||
} LanguageWasmInstance;
|
||||
|
||||
typedef struct {
|
||||
uint32_t reset_heap;
|
||||
uint32_t proc_exit;
|
||||
uint32_t abort;
|
||||
uint32_t assert_fail;
|
||||
uint32_t notify_memory_growth;
|
||||
uint32_t debug_message;
|
||||
uint32_t at_exit;
|
||||
uint32_t args_get;
|
||||
uint32_t args_sizes_get;
|
||||
wasmtime_func_t reset_heap;
|
||||
wasmtime_func_t proc_exit;
|
||||
wasmtime_func_t abort;
|
||||
wasmtime_func_t assert_fail;
|
||||
wasmtime_func_t notify_memory_growth;
|
||||
wasmtime_func_t debug_message;
|
||||
wasmtime_func_t at_exit;
|
||||
wasmtime_func_t args_get;
|
||||
wasmtime_func_t args_sizes_get;
|
||||
} BuiltinFunctionIndices;
|
||||
|
||||
// TSWasmStore - A struct that allows a given `Parser` to use Wasm-backed
|
||||
|
|
@ -104,7 +104,7 @@ struct TSWasmStore {
|
|||
Array(LanguageWasmInstance) language_instances;
|
||||
uint32_t current_memory_offset;
|
||||
uint32_t current_function_table_offset;
|
||||
uint32_t *stdlib_fn_indices;
|
||||
wasmtime_func_t *stdlib_fn_indices;
|
||||
BuiltinFunctionIndices builtin_fn_indices;
|
||||
wasmtime_global_t stack_pointer_global;
|
||||
wasm_globaltype_t *const_i32_type;
|
||||
|
|
@ -255,7 +255,7 @@ static bool wasm_dylink_info__parse(
|
|||
* Native callbacks exposed to Wasm modules
|
||||
*******************************************/
|
||||
|
||||
static wasm_trap_t *callback__abort(
|
||||
static wasm_trap_t *callback__abort(
|
||||
void *env,
|
||||
wasmtime_caller_t* caller,
|
||||
wasmtime_val_raw_t *args_and_results,
|
||||
|
|
@ -360,7 +360,7 @@ static wasm_trap_t *callback__lexer_eof(
|
|||
}
|
||||
|
||||
typedef struct {
|
||||
uint32_t *storage_location;
|
||||
void *storage_location;
|
||||
wasmtime_func_unchecked_callback_t callback;
|
||||
wasm_functype_t *type;
|
||||
} FunctionDefinition;
|
||||
|
|
@ -476,15 +476,11 @@ void language_id_delete(WasmLanguageId *self) {
|
|||
}
|
||||
|
||||
static wasmtime_extern_t get_builtin_extern(
|
||||
wasmtime_table_t *table,
|
||||
unsigned index
|
||||
wasmtime_func_t *func
|
||||
) {
|
||||
return (wasmtime_extern_t) {
|
||||
.kind = WASMTIME_EXTERN_FUNC,
|
||||
.of.func = (wasmtime_func_t) {
|
||||
.store_id = table->store_id,
|
||||
.__private = index
|
||||
}
|
||||
.of.func = *func
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -519,21 +515,21 @@ static bool ts_wasm_store__provide_builtin_import(
|
|||
|
||||
// Builtin functions
|
||||
else if (name_eq(import_name, "__assert_fail")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.assert_fail);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.assert_fail);
|
||||
} else if (name_eq(import_name, "__cxa_atexit")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.at_exit);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.at_exit);
|
||||
} else if (name_eq(import_name, "args_get")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_get);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.args_get);
|
||||
} else if (name_eq(import_name, "args_sizes_get")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_sizes_get);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.args_sizes_get);
|
||||
} else if (name_eq(import_name, "abort")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.abort);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.abort);
|
||||
} else if (name_eq(import_name, "proc_exit")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.proc_exit);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.proc_exit);
|
||||
} else if (name_eq(import_name, "emscripten_notify_memory_growth")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.notify_memory_growth);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.notify_memory_growth);
|
||||
} else if (name_eq(import_name, "tree_sitter_debug_message")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.debug_message);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.debug_message);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -575,6 +571,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
wasmtime_module_t *stdlib_module = NULL;
|
||||
wasm_memorytype_t *memory_type = NULL;
|
||||
wasm_tabletype_t *table_type = NULL;
|
||||
wasmtime_func_t *lexer_funcs = NULL;
|
||||
|
||||
// Define functions called by scanners via function pointers on the lexer.
|
||||
LexerInWasmMemory lexer = {
|
||||
|
|
@ -583,34 +580,34 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
};
|
||||
FunctionDefinition lexer_definitions[] = {
|
||||
{
|
||||
(uint32_t *)&lexer.advance,
|
||||
&lexer.advance,
|
||||
callback__lexer_advance,
|
||||
wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.mark_end,
|
||||
&lexer.mark_end,
|
||||
callback__lexer_mark_end,
|
||||
wasm_functype_new_1_0(wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.get_column,
|
||||
&lexer.get_column,
|
||||
callback__lexer_get_column,
|
||||
wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.is_at_included_range_start,
|
||||
&lexer.is_at_included_range_start,
|
||||
callback__lexer_is_at_included_range_start,
|
||||
wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.eof,
|
||||
&lexer.eof,
|
||||
callback__lexer_eof,
|
||||
wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
};
|
||||
|
||||
// Define builtin functions that can be imported by scanners.
|
||||
BuiltinFunctionIndices builtin_fn_indices;
|
||||
BuiltinFunctionIndices builtin_fn_indices = {0};
|
||||
FunctionDefinition builtin_definitions[] = {
|
||||
{
|
||||
&builtin_fn_indices.proc_exit,
|
||||
|
|
@ -657,18 +654,16 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
// Create all of the Wasm functions.
|
||||
unsigned builtin_definitions_len = array_len(builtin_definitions);
|
||||
unsigned lexer_definitions_len = array_len(lexer_definitions);
|
||||
lexer_funcs = ts_calloc(lexer_definitions_len, sizeof(wasmtime_func_t));
|
||||
for (unsigned i = 0; i < builtin_definitions_len; i++) {
|
||||
FunctionDefinition *definition = &builtin_definitions[i];
|
||||
wasmtime_func_t func;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
|
||||
*definition->storage_location = func.__private;
|
||||
wasmtime_func_t *func = (wasmtime_func_t *)definition->storage_location;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, func);
|
||||
wasm_functype_delete(definition->type);
|
||||
}
|
||||
for (unsigned i = 0; i < lexer_definitions_len; i++) {
|
||||
FunctionDefinition *definition = &lexer_definitions[i];
|
||||
wasmtime_func_t func;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
|
||||
*definition->storage_location = func.__private;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &lexer_funcs[i]);
|
||||
wasm_functype_delete(definition->type);
|
||||
}
|
||||
|
||||
|
|
@ -763,7 +758,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
.memory = memory,
|
||||
.function_table = function_table,
|
||||
.language_instances = array_new(),
|
||||
.stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(uint32_t)),
|
||||
.stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(wasmtime_func_t)),
|
||||
.builtin_fn_indices = builtin_fn_indices,
|
||||
.stack_pointer_global = stack_pointer_global,
|
||||
.current_memory_offset = 0,
|
||||
|
|
@ -816,7 +811,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
|
||||
// Process the stdlib module's exports.
|
||||
for (unsigned i = 0; i < stdlib_symbols_len; i++) {
|
||||
self->stdlib_fn_indices[i] = UINT32_MAX;
|
||||
self->stdlib_fn_indices[i] = (wasmtime_func_t){.store_id = 0};
|
||||
}
|
||||
wasmtime_module_exports(stdlib_module, &export_types);
|
||||
for (unsigned i = 0; i < export_types.size; i++) {
|
||||
|
|
@ -851,20 +846,20 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
}
|
||||
|
||||
if (name_eq(name, "reset_heap")) {
|
||||
self->builtin_fn_indices.reset_heap = export.of.func.__private;
|
||||
self->builtin_fn_indices.reset_heap = export.of.func;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (unsigned j = 0; j < stdlib_symbols_len; j++) {
|
||||
if (name_eq(name, STDLIB_SYMBOLS[j])) {
|
||||
self->stdlib_fn_indices[j] = export.of.func.__private;
|
||||
self->stdlib_fn_indices[j] = export.of.func;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (self->builtin_fn_indices.reset_heap == UINT32_MAX) {
|
||||
if (self->builtin_fn_indices.reset_heap.store_id == 0) {
|
||||
wasm_error->kind = TSWasmErrorKindInstantiate;
|
||||
format(
|
||||
&wasm_error->message,
|
||||
|
|
@ -874,7 +869,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
}
|
||||
|
||||
for (unsigned i = 0; i < stdlib_symbols_len; i++) {
|
||||
if (self->stdlib_fn_indices[i] == UINT32_MAX) {
|
||||
if (self->stdlib_fn_indices[i].store_id == 0) {
|
||||
wasm_error->kind = TSWasmErrorKindInstantiate;
|
||||
format(
|
||||
&wasm_error->message,
|
||||
|
|
@ -904,13 +899,13 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
}
|
||||
for (unsigned i = 0; i < lexer_definitions_len; i++) {
|
||||
FunctionDefinition *definition = &lexer_definitions[i];
|
||||
wasmtime_func_t func = {function_table.store_id, *definition->storage_location};
|
||||
wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = func};
|
||||
wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = lexer_funcs[i]};
|
||||
error = wasmtime_table_set(context, &function_table, table_index, &func_val);
|
||||
ts_assert(!error);
|
||||
*(int32_t *)(definition->storage_location) = table_index;
|
||||
table_index++;
|
||||
}
|
||||
ts_free(lexer_funcs);
|
||||
|
||||
self->current_function_table_offset = table_index;
|
||||
self->lexer_address = initial_memory_pages * MEMORY_PAGE_SIZE;
|
||||
|
|
@ -937,6 +932,7 @@ error:
|
|||
if (message.size) wasm_byte_vec_delete(&message);
|
||||
if (export_types.size) wasm_exporttype_vec_delete(&export_types);
|
||||
if (imports) ts_free(imports);
|
||||
ts_free(lexer_funcs);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -1016,8 +1012,6 @@ static bool ts_wasm_store__instantiate(
|
|||
// Construct the language function name as string.
|
||||
format(&language_function_name, "tree_sitter_%s", language_name);
|
||||
|
||||
const uint64_t store_id = self->function_table.store_id;
|
||||
|
||||
// Build the imports list for the module.
|
||||
wasm_importtype_vec_t import_types = WASM_EMPTY_VEC;
|
||||
wasmtime_module_imports(module, &import_types);
|
||||
|
|
@ -1038,8 +1032,7 @@ static bool ts_wasm_store__instantiate(
|
|||
bool defined_in_stdlib = false;
|
||||
for (unsigned j = 0; j < array_len(STDLIB_SYMBOLS); j++) {
|
||||
if (name_eq(import_name, STDLIB_SYMBOLS[j])) {
|
||||
uint16_t address = self->stdlib_fn_indices[j];
|
||||
imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = {store_id, address}};
|
||||
imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = self->stdlib_fn_indices[j]};
|
||||
defined_in_stdlib = true;
|
||||
break;
|
||||
}
|
||||
|
|
@ -1413,11 +1406,24 @@ const TSLanguage *ts_wasm_store_load_language(
|
|||
|
||||
if (language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) {
|
||||
language->name = copy_string(memory, wasm_language.name);
|
||||
language->reserved_words = copy(
|
||||
&memory[wasm_language.reserved_words],
|
||||
wasm_language.max_reserved_word_set_size * sizeof(TSSymbol)
|
||||
);
|
||||
language->max_reserved_word_set_size = wasm_language.max_reserved_word_set_size;
|
||||
|
||||
// Determine the number of reserved word sets by finding the maximum
|
||||
// reserved_word_set_id across all lex modes.
|
||||
uint16_t max_reserved_word_set_id = 0;
|
||||
for (uint32_t i = 0; i < wasm_language.state_count; i++) {
|
||||
uint16_t id = language->lex_modes[i].reserved_word_set_id;
|
||||
if (id > max_reserved_word_set_id) max_reserved_word_set_id = id;
|
||||
}
|
||||
|
||||
if (max_reserved_word_set_id > 0 && language->max_reserved_word_set_size > 0) {
|
||||
uint32_t reserved_word_count =
|
||||
(max_reserved_word_set_id + 1) * language->max_reserved_word_set_size;
|
||||
language->reserved_words = copy(
|
||||
&memory[wasm_language.reserved_words],
|
||||
reserved_word_count * sizeof(TSSymbol)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (language->external_token_count > 0) {
|
||||
|
|
@ -1546,16 +1552,13 @@ bool ts_wasm_store_add_language(
|
|||
|
||||
void ts_wasm_store_reset_heap(TSWasmStore *self) {
|
||||
wasmtime_context_t *context = wasmtime_store_context(self->store);
|
||||
wasmtime_func_t func = {
|
||||
self->function_table.store_id,
|
||||
self->builtin_fn_indices.reset_heap
|
||||
};
|
||||
wasmtime_func_t *func = &self->builtin_fn_indices.reset_heap;
|
||||
wasm_trap_t *trap = NULL;
|
||||
wasmtime_val_t args[1] = {
|
||||
{.of.i32 = ts_wasm_store__heap_address(self), .kind = WASMTIME_I32},
|
||||
};
|
||||
|
||||
wasmtime_error_t *error = wasmtime_func_call(context, &func, args, 1, NULL, 0, &trap);
|
||||
wasmtime_error_t *error = wasmtime_func_call(context, func, args, 1, NULL, 0, &trap);
|
||||
ts_assert(!error);
|
||||
ts_assert(!trap);
|
||||
}
|
||||
|
|
|
|||
32
test/fixtures/fixtures.json
vendored
32
test/fixtures/fixtures.json
vendored
|
|
@ -1,17 +1,17 @@
|
|||
[
|
||||
["bash","v0.25.0"],
|
||||
["c","v0.24.1"],
|
||||
["cpp","v0.23.4"],
|
||||
["embedded-template","v0.25.0"],
|
||||
["go","v0.25.0"],
|
||||
["html","v0.23.2"],
|
||||
["java","v0.23.5"],
|
||||
["javascript","v0.25.0"],
|
||||
["jsdoc","v0.23.2"],
|
||||
["json","v0.24.8"],
|
||||
["php","v0.24.2"],
|
||||
["python","v0.23.6"],
|
||||
["ruby","v0.23.1"],
|
||||
["rust","v0.24.0"],
|
||||
["typescript","v0.23.2"]
|
||||
]
|
||||
["bash","v0.25.0", null],
|
||||
["c","v0.24.1", null],
|
||||
["cpp","v0.23.4", null],
|
||||
["embedded-template","v0.25.0", null],
|
||||
["go","v0.25.0", null],
|
||||
["html","v0.23.2", null],
|
||||
["java","v0.23.5", null],
|
||||
["javascript","v0.25.0", null],
|
||||
["jsdoc","v0.23.2", null],
|
||||
["json","v0.24.8", null],
|
||||
["php","v0.24.2", "upstream_test_fixture"],
|
||||
["python","v0.23.6", null],
|
||||
["ruby","v0.23.1", null],
|
||||
["rust","v0.24.0", null],
|
||||
["typescript","v0.23.2", null]
|
||||
]
|
||||
|
|
|
|||
1
test/fixtures/test_grammars/terminal_supertype/expected_error.txt
vendored
Normal file
1
test/fixtures/test_grammars/terminal_supertype/expected_error.txt
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
Terminal rule 'A' cannot be used as a supertype
|
||||
18
test/fixtures/test_grammars/terminal_supertype/grammar.js
vendored
Normal file
18
test/fixtures/test_grammars/terminal_supertype/grammar.js
vendored
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
module.exports = grammar({
|
||||
name: 'terminal_supertype',
|
||||
|
||||
supertypes: $ => [$.A],
|
||||
extras: $ => [$.lineComment, $.lineComment2],
|
||||
|
||||
rules: {
|
||||
source_file: $ => choice(
|
||||
$.B,
|
||||
$.A,
|
||||
),
|
||||
lineComment: $ => /;;.*/,
|
||||
lineComment2: $ => /##.*/,
|
||||
A: $ => /xyz/,
|
||||
C: $ => "xyz",
|
||||
B: $ => seq('abc', $.C),
|
||||
}
|
||||
});
|
||||
9
test/fixtures/test_grammars/wasm_realloc_clobber_region/corpus.txt
vendored
Normal file
9
test/fixtures/test_grammars/wasm_realloc_clobber_region/corpus.txt
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
==========================
|
||||
A leading zero-width token
|
||||
==========================
|
||||
|
||||
hello
|
||||
|
||||
---
|
||||
|
||||
(document (zero_width))
|
||||
10
test/fixtures/test_grammars/wasm_realloc_clobber_region/grammar.js
vendored
Normal file
10
test/fixtures/test_grammars/wasm_realloc_clobber_region/grammar.js
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
export default grammar({
|
||||
name: 'wasm_realloc_clobber_region',
|
||||
|
||||
extras: _ => [/\s/],
|
||||
externals: $ => [$.zero_width],
|
||||
|
||||
rules: {
|
||||
document: $ => seq($.zero_width, 'hello'),
|
||||
}
|
||||
});
|
||||
68
test/fixtures/test_grammars/wasm_realloc_clobber_region/scanner.c
vendored
Normal file
68
test/fixtures/test_grammars/wasm_realloc_clobber_region/scanner.c
vendored
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
#include "tree_sitter/parser.h"
|
||||
#include <string.h>
|
||||
|
||||
#ifdef __wasm__
|
||||
#define panic(x) __builtin_trap()
|
||||
#else
|
||||
#include <assert.h>
|
||||
#define panic(x) assert(0)
|
||||
#endif
|
||||
|
||||
enum TokenType {
|
||||
ZERO_WIDTH_TOKEN
|
||||
};
|
||||
|
||||
void *tree_sitter_wasm_realloc_clobber_region_external_scanner_create(void) {
|
||||
size_t a_count = 32;
|
||||
void *a = malloc(a_count);
|
||||
memset(a, 0x41, a_count); // fill `a` with 'A's
|
||||
for (int i = 0; i < 32; i++) {
|
||||
// This condition is unreachable, but it prevents wasi-sdk clang from optimizing
|
||||
// out the allocation to `a`.
|
||||
if (((char *)a)[i] != 0x41) panic();
|
||||
}
|
||||
|
||||
size_t b_count = 16;
|
||||
void *b = malloc(b_count);
|
||||
memset(b, 0x42, b_count); // fill `b` with 'B's
|
||||
|
||||
// `a` is now first on the free list
|
||||
free(a);
|
||||
// We're re-allocating the last region, so `realloc`'s optimization to re-use the region
|
||||
// by altering its metadata's size (thus avoiding a memcpy) should be used.
|
||||
void *c = realloc(b, 24); // realloc `b` should still contain 'B's
|
||||
|
||||
// ensure the contents of `b` were not clobbered. A previous bug caused `a`s region
|
||||
// to be used for `c`.
|
||||
char *p = (char *)c;
|
||||
for (int i = 0; i < (int)b_count; i++) {
|
||||
if (p[i] != 0x42) panic();
|
||||
}
|
||||
|
||||
free(c);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool tree_sitter_wasm_realloc_clobber_region_external_scanner_scan(
|
||||
void *payload,
|
||||
TSLexer *lexer,
|
||||
const bool *valid_symbols
|
||||
) {
|
||||
lexer->result_symbol = ZERO_WIDTH_TOKEN;
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned tree_sitter_wasm_realloc_clobber_region_external_scanner_serialize(
|
||||
void *payload,
|
||||
char *buffer
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tree_sitter_wasm_realloc_clobber_region_external_scanner_deserialize(
|
||||
void *payload,
|
||||
const char *buffer,
|
||||
unsigned length
|
||||
) {}
|
||||
|
||||
void tree_sitter_wasm_realloc_clobber_region_external_scanner_destroy(void *payload) {}
|
||||
9
test/fixtures/test_grammars/wasm_realloc_overflow_heap/corpus.txt
vendored
Normal file
9
test/fixtures/test_grammars/wasm_realloc_overflow_heap/corpus.txt
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
==========================
|
||||
A leading zero-width token
|
||||
==========================
|
||||
|
||||
hello
|
||||
|
||||
---
|
||||
|
||||
(document (zero_width))
|
||||
10
test/fixtures/test_grammars/wasm_realloc_overflow_heap/grammar.js
vendored
Normal file
10
test/fixtures/test_grammars/wasm_realloc_overflow_heap/grammar.js
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
export default grammar({
|
||||
name: 'wasm_realloc_overflow_heap',
|
||||
|
||||
extras: _ => [/\s/],
|
||||
externals: $ => [$.zero_width],
|
||||
|
||||
rules: {
|
||||
document: $ => seq($.zero_width, 'hello'),
|
||||
}
|
||||
});
|
||||
43
test/fixtures/test_grammars/wasm_realloc_overflow_heap/scanner.c
vendored
Normal file
43
test/fixtures/test_grammars/wasm_realloc_overflow_heap/scanner.c
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#include "tree_sitter/parser.h"
|
||||
|
||||
// Constant copied from `crates/language/wasm/src/stdlib.c`,
|
||||
// Must be kept in sync for a reliable repro.
|
||||
#define MAX_HEAP_SIZE (4 * 1024 * 1024)
|
||||
|
||||
enum TokenType {
|
||||
ZERO_WIDTH_TOKEN
|
||||
};
|
||||
|
||||
void *tree_sitter_wasm_realloc_overflow_heap_external_scanner_create(void) {
|
||||
size_t large_size = MAX_HEAP_SIZE - 64;
|
||||
void *p = malloc(large_size);
|
||||
void *q = malloc(4);
|
||||
p = realloc(p, 4);
|
||||
free(p);
|
||||
free(q);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool tree_sitter_wasm_realloc_overflow_heap_external_scanner_scan(
|
||||
void *payload,
|
||||
TSLexer *lexer,
|
||||
const bool *valid_symbols
|
||||
) {
|
||||
lexer->result_symbol = ZERO_WIDTH_TOKEN;
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned tree_sitter_wasm_realloc_overflow_heap_external_scanner_serialize(
|
||||
void *payload,
|
||||
char *buffer
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tree_sitter_wasm_realloc_overflow_heap_external_scanner_deserialize(
|
||||
void *payload,
|
||||
const char *buffer,
|
||||
unsigned length
|
||||
) {}
|
||||
|
||||
void tree_sitter_wasm_realloc_overflow_heap_external_scanner_destroy(void *payload) {}
|
||||
Loading…
Reference in a new issue