mirror of
https://github.com/tree-sitter/tree-sitter.git
synced 2026-09-10 07:36:22 -04:00
Compare commits
60 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
534c4a074c | ||
|
|
0de6ea6edd | ||
|
|
6fdf7fdbc6 | ||
|
|
a6aabeb941 | ||
|
|
b1e10d5410 | ||
|
|
b3e86f75cc | ||
|
|
a226c68720 | ||
|
|
d03bb288e2 | ||
|
|
ed06ca058b | ||
|
|
fc5f1b4526 | ||
|
|
054f892546 | ||
|
|
4bcc064ecc | ||
|
|
93a5767892 | ||
|
|
a5129d5bc0 | ||
|
|
9a22dff7e5 | ||
|
|
fdc20d9c58 | ||
|
|
89e804b7e4 | ||
|
|
3b85287d6d | ||
|
|
79b60a271c | ||
|
|
ad7a629a1b | ||
|
|
325bc50d6f | ||
|
|
932bde72b2 | ||
|
|
470813116b | ||
|
|
7ec1794d6b | ||
|
|
fa8811f7f7 | ||
|
|
ef4999bf61 | ||
|
|
77e43dd116 | ||
|
|
666144d3ed | ||
|
|
ce2cb41e1f | ||
|
|
a423343bd3 | ||
|
|
c8aedb8cfa | ||
|
|
308b96d927 | ||
|
|
e98a09b6cc | ||
|
|
d3a20faff9 | ||
|
|
88a5475496 | ||
|
|
6a8a5e33d9 | ||
|
|
bc2e4e2386 | ||
|
|
f44e86628a | ||
|
|
ed6e42cbf0 | ||
|
|
4809aaaf04 | ||
|
|
152d2756fc | ||
|
|
f05efbb352 | ||
|
|
1f221c8500 | ||
|
|
fdca0718bc | ||
|
|
fa7b1b2a66 | ||
|
|
adcc4d1f7b | ||
|
|
7d9c544c96 | ||
|
|
c1e49d1571 | ||
|
|
eae6554735 | ||
|
|
48ee942c4f | ||
|
|
9ee2b87dd6 | ||
|
|
fb91deb8d9 | ||
|
|
789a966f96 | ||
|
|
3c49fef0e3 | ||
|
|
8a297b86bc | ||
|
|
ac6644016c | ||
|
|
a80765614b | ||
|
|
34602af22c | ||
|
|
c4f81931e6 | ||
|
|
25777e5a64 |
105
.github/workflows/build.yml
vendored
105
.github/workflows/build.yml
vendored
|
|
@ -46,8 +46,8 @@ jobs:
|
|||
|
||||
# Extra features
|
||||
- { platform: linux-arm64 , features: wasm }
|
||||
- { platform: linux-x64 , features: wasm }
|
||||
- { platform: macos-arm64 , features: wasm }
|
||||
- { platform: linux-x64 , features: wasm , run-wasm-test: true }
|
||||
- { platform: macos-arm64 , features: wasm , run-wasm-test: true }
|
||||
- { platform: macos-x64 , features: wasm }
|
||||
|
||||
# Cross-compilation
|
||||
|
|
@ -85,14 +85,37 @@ jobs:
|
|||
} >> $GITHUB_ENV
|
||||
|
||||
- name: Get emscripten version
|
||||
if: contains(matrix.features, 'wasm')
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
|
||||
|
||||
- name: Install Emscripten
|
||||
if: contains(matrix.features, 'wasm')
|
||||
uses: mymindstorm/setup-emsdk@v14
|
||||
- name: Cache Emscripten SDK
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
version: ${{ env.EMSCRIPTEN_VERSION }}
|
||||
path: emsdk
|
||||
key: emsdk-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}-${{ runner.arch }}
|
||||
|
||||
- name: Install Emscripten
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
run: |
|
||||
if [[ ! -d emsdk ]]; then
|
||||
git clone --depth 1 https://github.com/emscripten-core/emsdk.git
|
||||
fi
|
||||
cd emsdk
|
||||
./emsdk install ${{ env.EMSCRIPTEN_VERSION }}
|
||||
./emsdk activate ${{ env.EMSCRIPTEN_VERSION }}
|
||||
echo "$PWD" >> "$GITHUB_PATH"
|
||||
echo "$PWD/upstream/emscripten" >> "$GITHUB_PATH"
|
||||
echo "EMSDK=$PWD" >> "$GITHUB_ENV"
|
||||
echo "EMSDK_NODE=$PWD/node/$(ls node)/bin/node" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
cache-dependency-path: lib/binding_web/package-lock.json
|
||||
|
||||
- name: Set up Rust
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
|
|
@ -117,34 +140,12 @@ jobs:
|
|||
if: matrix.platform == 'windows-x64'
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
update: true
|
||||
install: |
|
||||
mingw-w64-x86_64-toolchain
|
||||
mingw-w64-x86_64-clang
|
||||
mingw-w64-x86_64-make
|
||||
mingw-w64-x86_64-cmake
|
||||
|
||||
# TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
|
||||
# the `mismatched-lifetime-syntaxes` lint
|
||||
- name: Build wasmtime library (Windows x64 MSYS2)
|
||||
if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64'
|
||||
run: |
|
||||
mkdir -p target
|
||||
WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
|
||||
jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
|
||||
curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
|
||||
cd target/wasmtime-${WASMTIME_VERSION}
|
||||
cmake -S crates/c-api -B target/c-api \
|
||||
-DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
|
||||
-DWASMTIME_DISABLE_ALL_FEATURES=ON \
|
||||
-DWASMTIME_FEATURE_CRANELIFT=ON \
|
||||
-DWASMTIME_TARGET='x86_64-pc-windows-gnu'
|
||||
cmake --build target/c-api && cmake --install target/c-api
|
||||
printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
|
||||
env:
|
||||
WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
|
||||
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
|
||||
|
||||
- name: Build C library (Windows x64 MSYS2 CMake)
|
||||
if: matrix.platform == 'windows-x64'
|
||||
shell: msys2 {0}
|
||||
|
|
@ -171,26 +172,23 @@ jobs:
|
|||
env:
|
||||
WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
|
||||
|
||||
# TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
|
||||
# the `mismatched-lifetime-syntaxes` lint
|
||||
- name: Build wasmtime library
|
||||
if: contains(matrix.features, 'wasm')
|
||||
- name: Download wasmtime C API
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
run: |
|
||||
mkdir -p target
|
||||
WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
|
||||
jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
|
||||
curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
|
||||
cd target/wasmtime-${WASMTIME_VERSION}
|
||||
cmake -S crates/c-api -B target/c-api \
|
||||
-DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
|
||||
-DWASMTIME_DISABLE_ALL_FEATURES=ON \
|
||||
-DWASMTIME_FEATURE_CRANELIFT=ON \
|
||||
-DWASMTIME_TARGET='${{ matrix.target }}'
|
||||
cmake --build target/c-api && cmake --install target/c-api
|
||||
printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
|
||||
case '${{ matrix.target }}' in
|
||||
x86_64-unknown-linux-gnu) WT_TARGET=x86_64-linux ;;
|
||||
aarch64-unknown-linux-gnu) WT_TARGET=aarch64-linux ;;
|
||||
x86_64-apple-darwin) WT_TARGET=x86_64-macos ;;
|
||||
aarch64-apple-darwin) WT_TARGET=aarch64-macos ;;
|
||||
esac
|
||||
curl -LSs "$WASMTIME_REPO/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api.tar.xz" \
|
||||
| tar xJf - -C target
|
||||
printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/target/wasmtime-v${WASMTIME_VERSION}-${WT_TARGET}-c-api" >> $GITHUB_ENV
|
||||
env:
|
||||
WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
|
||||
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
|
||||
|
||||
- name: Build C library (make)
|
||||
if: runner.os != 'Windows'
|
||||
|
|
@ -227,10 +225,10 @@ jobs:
|
|||
cmake --build build/shared --verbose
|
||||
env:
|
||||
CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
|
||||
WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
|
||||
WASM: ${{ contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test) && 'ON' || 'OFF' }}
|
||||
|
||||
- name: Build Wasm library
|
||||
if: contains(matrix.features, 'wasm')
|
||||
if: contains(matrix.features, 'wasm') && (matrix.run-wasm-test || !inputs.run-test)
|
||||
shell: bash
|
||||
run: |
|
||||
cd lib/binding_web
|
||||
|
|
@ -247,7 +245,8 @@ jobs:
|
|||
run: cargo check --no-default-features --target='${{ matrix.target }}'
|
||||
|
||||
- name: Build target
|
||||
run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE
|
||||
if: "!inputs.run-test"
|
||||
run: cargo build --release --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}' $PACKAGE
|
||||
env:
|
||||
PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
|
||||
|
||||
|
|
@ -265,20 +264,20 @@ jobs:
|
|||
run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
|
||||
|
||||
- name: Generate Wasm fixtures
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true'
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test && steps.cache.outputs.cache-hit != 'true'
|
||||
run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
|
||||
|
||||
- name: Run main tests
|
||||
if: inputs.run-test && !matrix.no-run
|
||||
run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}'
|
||||
run: cargo test --target='${{ matrix.target }}' --features='${{ (matrix.run-wasm-test || !inputs.run-test) && matrix.features || '' }}'
|
||||
|
||||
- name: Run Wasm tests
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm')
|
||||
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
|
||||
run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
|
||||
|
||||
- name: Upload CLI artifact
|
||||
if: "!matrix.no-run"
|
||||
uses: actions/upload-artifact@v5
|
||||
if: "!inputs.run-test && !matrix.no-run"
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tree-sitter.${{ matrix.platform }}
|
||||
path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
|
||||
|
|
@ -286,8 +285,8 @@ jobs:
|
|||
retention-days: 7
|
||||
|
||||
- name: Upload Wasm artifacts
|
||||
if: matrix.platform == 'linux-x64'
|
||||
uses: actions/upload-artifact@v5
|
||||
if: "!inputs.run-test && matrix.platform == 'linux-x64'"
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tree-sitter.wasm
|
||||
path: |
|
||||
|
|
|
|||
22
.github/workflows/crate_versions.yml
vendored
Normal file
22
.github/workflows/crate_versions.yml
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
name: Crate Versions Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled, opened, synchronize, reopened]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check-crates:
|
||||
runs-on: ubuntu-latest
|
||||
if: contains(github.event.pull_request.labels.*.name, 'ci:check release') || github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Rust
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
|
||||
- name: Check crates against crates.io
|
||||
uses: katyo/publish-crates@v2
|
||||
with:
|
||||
dry-run: true
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
cmake_minimum_required(VERSION 3.13)
|
||||
|
||||
project(tree-sitter
|
||||
VERSION "0.26.3"
|
||||
VERSION "0.26.6"
|
||||
DESCRIPTION "An incremental parsing system for programming tools"
|
||||
HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
|
||||
LANGUAGES C)
|
||||
|
|
@ -81,7 +81,7 @@ set_target_properties(tree-sitter
|
|||
SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
|
||||
DEFINE_SYMBOL "")
|
||||
|
||||
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
|
||||
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
|
||||
|
|
|
|||
BIN
Cargo.lock
generated
BIN
Cargo.lock
generated
Binary file not shown.
14
Cargo.toml
14
Cargo.toml
|
|
@ -14,7 +14,7 @@ members = [
|
|||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.26.3"
|
||||
version = "0.26.6"
|
||||
authors = [
|
||||
"Max Brunsfeld <maxbrunsfeld@gmail.com>",
|
||||
"Amaan Qureshi <amaanq12@gmail.com>",
|
||||
|
|
@ -153,11 +153,11 @@ walkdir = "2.5.0"
|
|||
wasmparser = "0.243.0"
|
||||
webbrowser = "1.0.5"
|
||||
|
||||
tree-sitter = { version = "0.26.3", path = "./lib" }
|
||||
tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" }
|
||||
tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" }
|
||||
tree-sitter-config = { version = "0.26.3", path = "./crates/config" }
|
||||
tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" }
|
||||
tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" }
|
||||
tree-sitter = { version = "0.26.6", path = "./lib" }
|
||||
tree-sitter-generate = { version = "0.26.6", path = "./crates/generate" }
|
||||
tree-sitter-loader = { version = "0.26.6", path = "./crates/loader" }
|
||||
tree-sitter-config = { version = "0.26.6", path = "./crates/config" }
|
||||
tree-sitter-highlight = { version = "0.26.6", path = "./crates/highlight" }
|
||||
tree-sitter-tags = { version = "0.26.6", path = "./crates/tags" }
|
||||
|
||||
tree-sitter-language = { version = "0.1", path = "./crates/language" }
|
||||
|
|
|
|||
5
Makefile
5
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
VERSION := 0.26.3
|
||||
VERSION := 0.26.6
|
||||
DESCRIPTION := An incremental parsing system for programming tools
|
||||
HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
|
|||
ARFLAGS := rcs
|
||||
CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
|
||||
override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
|
||||
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
|
||||
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
|
||||
override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
|
||||
|
||||
# ABI versioning
|
||||
|
|
@ -122,7 +122,6 @@ test-wasm:
|
|||
|
||||
lint:
|
||||
cargo update --workspace --locked --quiet
|
||||
cargo check --workspace --all-targets
|
||||
cargo fmt --all --check
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ let package = Package(
|
|||
.headerSearchPath("src"),
|
||||
.define("_POSIX_C_SOURCE", to: "200112L"),
|
||||
.define("_DEFAULT_SOURCE"),
|
||||
.define("_BSD_SOURCE"),
|
||||
.define("_DARWIN_C_SOURCE"),
|
||||
]),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
|
|||
|
||||
lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
|
||||
lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
|
||||
lib.root_module.addCMacro("_BSD_SOURCE", "");
|
||||
lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
|
||||
|
||||
if (wasm) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
.{
|
||||
.name = .tree_sitter,
|
||||
.fingerprint = 0x841224b447ac0d4f,
|
||||
.version = "0.26.3",
|
||||
.version = "0.26.6",
|
||||
.minimum_zig_version = "0.14.1",
|
||||
.paths = .{
|
||||
"build.zig",
|
||||
|
|
|
|||
|
|
@ -19,9 +19,7 @@ static LANGUAGE_FILTER: LazyLock<Option<String>> =
|
|||
static EXAMPLE_FILTER: LazyLock<Option<String>> =
|
||||
LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
|
||||
static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
|
||||
env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
|
||||
.map(|s| s.parse::<usize>().unwrap())
|
||||
.unwrap_or(5)
|
||||
env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
|
||||
});
|
||||
static TEST_LOADER: LazyLock<Loader> =
|
||||
LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
|
||||
|
|
|
|||
4
crates/cli/npm/package-lock.json
generated
4
crates/cli/npm/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.6",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.6",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.6",
|
||||
"author": {
|
||||
"name": "Max Brunsfeld",
|
||||
"email": "maxbrunsfeld@gmail.com"
|
||||
|
|
|
|||
|
|
@ -44,11 +44,13 @@ pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
|
|||
|
||||
pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
|
||||
|
||||
pub const DEFAULT_EDIT_COUNT: usize = 3;
|
||||
pub static EDIT_COUNT: LazyLock<usize> =
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(DEFAULT_EDIT_COUNT));
|
||||
|
||||
pub const DEFAULT_ITERATION_COUNT: usize = 10;
|
||||
pub static ITERATION_COUNT: LazyLock<usize> =
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
|
||||
LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(DEFAULT_ITERATION_COUNT));
|
||||
|
||||
fn int_env_var(name: &'static str) -> Option<usize> {
|
||||
env::var(name).ok().and_then(|e| e.parse().ok())
|
||||
|
|
@ -221,7 +223,7 @@ pub fn fuzz_language_corpus(
|
|||
}
|
||||
|
||||
// Perform a random series of edits and reparse.
|
||||
let edit_count = rand.unsigned(*EDIT_COUNT);
|
||||
let edit_count = rand.unsigned(options.edits);
|
||||
let mut undo_stack = Vec::with_capacity(edit_count);
|
||||
for _ in 0..=edit_count {
|
||||
let edit = get_random_edit(&mut rand, &input);
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
|
|||
use crc32fast::hash as crc32;
|
||||
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
|
||||
use indoc::{formatdoc, indoc};
|
||||
use log::warn;
|
||||
use log::info;
|
||||
use rand::{thread_rng, Rng};
|
||||
use semver::Version;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
|
|||
const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
|
||||
const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
|
||||
|
||||
const TREE_SITTER_JSON_SCHEMA: &str =
|
||||
pub const TREE_SITTER_JSON_SCHEMA: &str =
|
||||
"https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
|
|
@ -356,7 +356,7 @@ pub fn generate_grammar_files(
|
|||
"tree-sitter-cli":"#},
|
||||
);
|
||||
if !contents.contains("module") {
|
||||
warn!("Updating package.json");
|
||||
info!("Migrating package.json to ESM");
|
||||
contents = contents.replace(
|
||||
r#""repository":"#,
|
||||
indoc! {r#"
|
||||
|
|
@ -378,6 +378,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if contents.contains("module.exports") {
|
||||
info!("Migrating grammars.js to ESM");
|
||||
contents = contents.replace("module.exports =", "export default");
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
|
|
@ -393,10 +394,16 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Zig artifacts") {
|
||||
warn!("Replacing .gitignore");
|
||||
generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
|
||||
info!("Adding zig entries to .gitignore");
|
||||
contents.push('\n');
|
||||
contents.push_str(indoc! {"
|
||||
# Zig artifacts
|
||||
.zig-cache/
|
||||
zig-cache/
|
||||
zig-out/
|
||||
"});
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|
|
@ -409,8 +416,13 @@ pub fn generate_grammar_files(
|
|||
|path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents.replace("bindings/c/* ", "bindings/c/** ");
|
||||
let c_bindings_entry = "bindings/c/* ";
|
||||
if contents.contains(c_bindings_entry) {
|
||||
info!("Updating c bindings entry in .gitattributes");
|
||||
contents = contents.replace(c_bindings_entry, "bindings/c/** ");
|
||||
}
|
||||
if !contents.contains("Zig bindings") {
|
||||
info!("Adding zig entries to .gitattributes");
|
||||
contents.push('\n');
|
||||
contents.push_str(indoc! {"
|
||||
# Zig bindings
|
||||
|
|
@ -438,39 +450,40 @@ pub fn generate_grammar_files(
|
|||
}, |path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("#[cfg(with_highlights_query)]") {
|
||||
let replacement = indoc! {r#"
|
||||
#[cfg(with_highlights_query)]
|
||||
/// The syntax highlighting query for this grammar.
|
||||
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
|
||||
info!("Updating query constants in bindings/rust/lib.rs");
|
||||
let replacement = indoc! {r#"
|
||||
#[cfg(with_highlights_query)]
|
||||
/// The syntax highlighting query for this grammar.
|
||||
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_injections_query)]
|
||||
/// The language injection query for this grammar.
|
||||
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
|
||||
#[cfg(with_injections_query)]
|
||||
/// The language injection query for this grammar.
|
||||
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_locals_query)]
|
||||
/// The local variable query for this grammar.
|
||||
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
|
||||
#[cfg(with_locals_query)]
|
||||
/// The local variable query for this grammar.
|
||||
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_tags_query)]
|
||||
/// The symbol tagging query for this grammar.
|
||||
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
|
||||
"#}
|
||||
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
|
||||
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
|
||||
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
|
||||
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r#"
|
||||
// NOTE: uncomment these to include any queries that this grammar contains:
|
||||
#[cfg(with_tags_query)]
|
||||
/// The symbol tagging query for this grammar.
|
||||
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
|
||||
"#}
|
||||
.replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
|
||||
.replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
|
||||
.replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
|
||||
.replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r#"
|
||||
// NOTE: uncomment these to include any queries that this grammar contains:
|
||||
|
||||
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
|
||||
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
|
||||
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
|
||||
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
|
||||
"#},
|
||||
&replacement,
|
||||
);
|
||||
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
|
||||
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
|
||||
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
|
||||
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
|
||||
"#},
|
||||
&replacement,
|
||||
);
|
||||
}
|
||||
write_file(path, contents)?;
|
||||
Ok(())
|
||||
|
|
@ -483,6 +496,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("wasm32-unknown-unknown") {
|
||||
info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
|
||||
let replacement = indoc!{r#"
|
||||
c_config.flag("-utf-8");
|
||||
|
||||
|
|
@ -503,19 +517,18 @@ pub fn generate_grammar_files(
|
|||
wasm_src.join("string.c"),
|
||||
]);
|
||||
}
|
||||
"#};
|
||||
|
||||
let indented_replacement = replacement
|
||||
"#}
|
||||
.lines()
|
||||
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &indented_replacement);
|
||||
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement);
|
||||
}
|
||||
|
||||
// Introduce configuration variables for dynamic query inclusion
|
||||
if !contents.contains("with_highlights_query") {
|
||||
info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
|
||||
let replaced = indoc! {r#"
|
||||
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
|
||||
}"#}
|
||||
|
|
@ -542,10 +555,10 @@ pub fn generate_grammar_files(
|
|||
}
|
||||
}"#}
|
||||
.replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
|
||||
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
|
||||
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
|
||||
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
|
||||
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
|
||||
.replace(HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, &generate_opts.highlights_query_path.replace('\\', "/"))
|
||||
.replace(INJECTIONS_QUERY_PATH_PLACEHOLDER, &generate_opts.injections_query_path.replace('\\', "/"))
|
||||
.replace(LOCALS_QUERY_PATH_PLACEHOLDER, &generate_opts.locals_query_path.replace('\\', "/"))
|
||||
.replace(TAGS_QUERY_PATH_PLACEHOLDER, &generate_opts.tags_query_path.replace('\\', "/"));
|
||||
|
||||
contents = contents.replace(
|
||||
&replaced,
|
||||
|
|
@ -572,6 +585,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("\"LICENSE\"") {
|
||||
info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
|
||||
write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -592,7 +606,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Object.defineProperty") {
|
||||
warn!("Replacing index.js");
|
||||
info!("Replacing index.js");
|
||||
generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -606,7 +620,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("export default binding") {
|
||||
warn!("Replacing index.d.ts");
|
||||
info!("Replacing index.d.ts");
|
||||
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -627,7 +641,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("import") {
|
||||
warn!("Replacing binding_test.js");
|
||||
info!("Replacing binding_test.js");
|
||||
generate_file(
|
||||
path,
|
||||
BINDING_TEST_JS_TEMPLATE,
|
||||
|
|
@ -650,6 +664,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("fs.exists(") {
|
||||
info!("Replacing `fs.exists` calls in binding.gyp");
|
||||
write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -662,14 +677,17 @@ pub fn generate_grammar_files(
|
|||
|
||||
// Generate C bindings
|
||||
if tree_sitter_config.bindings.c {
|
||||
let kebab_case_name = language_name.to_kebab_case();
|
||||
missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
|
||||
let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
|
||||
let header_name = format!("tree-sitter-{kebab_case_name}.h");
|
||||
let old_file = &path.join(&header_name);
|
||||
if allow_update && fs::exists(old_file).unwrap_or(false) {
|
||||
info!("Removing bindings/c/{header_name}");
|
||||
fs::remove_file(old_file)?;
|
||||
}
|
||||
missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
|
||||
missing_path(
|
||||
include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
|
||||
include_path.join(&header_name),
|
||||
|path| {
|
||||
generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
|
||||
},
|
||||
|
|
@ -678,7 +696,7 @@ pub fn generate_grammar_files(
|
|||
})?;
|
||||
|
||||
missing_path(
|
||||
path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
|
||||
path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
|
||||
|path| {
|
||||
generate_file(
|
||||
path,
|
||||
|
|
@ -698,23 +716,27 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
|
||||
warn!("Replacing Makefile");
|
||||
info!("Replacing Makefile");
|
||||
generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
|
||||
} else {
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r"
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"},
|
||||
indoc! {r"
|
||||
$(SRC_DIR)/grammar.json: grammar.js
|
||||
$(TS) generate --no-parser $^
|
||||
let replaced = indoc! {r"
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"};
|
||||
if contents.contains(replaced) {
|
||||
info!("Adding --no-parser target to Makefile");
|
||||
contents = contents
|
||||
.replace(
|
||||
replaced,
|
||||
indoc! {r"
|
||||
$(SRC_DIR)/grammar.json: grammar.js
|
||||
$(TS) generate --no-parser $^
|
||||
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"}
|
||||
);
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"}
|
||||
);
|
||||
}
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -726,8 +748,8 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let replaced_contents = contents
|
||||
.replace("add_custom_target(test", "add_custom_target(ts-test")
|
||||
.replace(
|
||||
&formatdoc! {r#"
|
||||
|
|
@ -775,7 +797,10 @@ pub fn generate_grammar_files(
|
|||
COMMENT "Generating parser.c")
|
||||
"#}
|
||||
);
|
||||
write_file(path, contents)?;
|
||||
if !replaced_contents.eq(&contents) {
|
||||
info!("Updating CMakeLists.txt");
|
||||
write_file(path, replaced_contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -811,7 +836,8 @@ pub fn generate_grammar_files(
|
|||
// Generate Python bindings
|
||||
if tree_sitter_config.bindings.python {
|
||||
missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
|
||||
let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
|
||||
let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
|
||||
let lang_path = path.join(&snake_case_grammar_name);
|
||||
missing_path(&lang_path, create_dir)?;
|
||||
|
||||
missing_path_else(
|
||||
|
|
@ -821,6 +847,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("PyModuleDef_Init") {
|
||||
info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
|
||||
contents = contents
|
||||
.replace("PyModule_Create", "PyModuleDef_Init")
|
||||
.replace(
|
||||
|
|
@ -861,8 +888,8 @@ pub fn generate_grammar_files(
|
|||
},
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("uncomment these to include any queries") {
|
||||
warn!("Replacing __init__.py");
|
||||
if contents.contains("uncomment these to include any queries") {
|
||||
info!("Replacing __init__.py");
|
||||
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -876,9 +903,10 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if contents.contains("uncomment these to include any queries") {
|
||||
warn!("Replacing __init__.pyi");
|
||||
info!("Replacing __init__.pyi");
|
||||
generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
|
||||
} else if !contents.contains("CapsuleType") {
|
||||
info!("Updating __init__.pyi");
|
||||
contents = contents
|
||||
.replace(
|
||||
"from typing import Final",
|
||||
|
|
@ -910,6 +938,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Parser(Language(") {
|
||||
info!("Updating Language function in bindings/python/tests/test_binding.py");
|
||||
contents = contents
|
||||
.replace("tree_sitter.Language(", "Parser(Language(")
|
||||
.replace(".language())\n", ".language()))\n")
|
||||
|
|
@ -930,11 +959,19 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("build_ext") {
|
||||
warn!("Replacing setup.py");
|
||||
info!("Replacing setup.py");
|
||||
generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
if !contents.contains(" and not get_config_var") {
|
||||
info!("Updating Python free-threading support in setup.py");
|
||||
contents = contents.replace(
|
||||
r#"startswith("cp"):"#,
|
||||
r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
|
||||
);
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -953,6 +990,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("cp310-*") {
|
||||
info!("Updating dependencies in pyproject.toml");
|
||||
contents = contents
|
||||
.replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
|
||||
.replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
|
||||
|
|
@ -990,15 +1028,18 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let replaced_contents = contents
|
||||
.replace(
|
||||
"https://github.com/ChimeHQ/SwiftTreeSitter",
|
||||
"https://github.com/tree-sitter/swift-tree-sitter",
|
||||
)
|
||||
.replace("version: \"0.8.0\")", "version: \"0.9.0\")")
|
||||
.replace("(url:", "(name: \"SwiftTreeSitter\", url:");
|
||||
write_file(path, contents)?;
|
||||
if !replaced_contents.eq(&contents) {
|
||||
info!("Updating tree-sitter dependency in Package.swift");
|
||||
write_file(path, replaced_contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -1016,7 +1057,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("b.pkg_hash.len") {
|
||||
warn!("Replacing build.zig");
|
||||
info!("Replacing build.zig");
|
||||
generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1031,7 +1072,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains(".name = .tree_sitter_") {
|
||||
warn!("Replacing build.zig.zon");
|
||||
info!("Replacing build.zig.zon");
|
||||
generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1047,7 +1088,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("ts.Language") {
|
||||
warn!("Replacing root.zig");
|
||||
info!("Replacing root.zig");
|
||||
generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1189,17 +1230,20 @@ fn generate_file(
|
|||
.replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name)
|
||||
.replace(
|
||||
HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
|
||||
generate_opts.highlights_query_path,
|
||||
&generate_opts.highlights_query_path.replace('\\', "/"),
|
||||
)
|
||||
.replace(
|
||||
INJECTIONS_QUERY_PATH_PLACEHOLDER,
|
||||
generate_opts.injections_query_path,
|
||||
&generate_opts.injections_query_path.replace('\\', "/"),
|
||||
)
|
||||
.replace(
|
||||
LOCALS_QUERY_PATH_PLACEHOLDER,
|
||||
generate_opts.locals_query_path,
|
||||
&generate_opts.locals_query_path.replace('\\', "/"),
|
||||
)
|
||||
.replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
|
||||
.replace(
|
||||
TAGS_QUERY_PATH_PLACEHOLDER,
|
||||
&generate_opts.tags_query_path.replace('\\', "/"),
|
||||
);
|
||||
|
||||
if let Some(name) = generate_opts.author_name {
|
||||
replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);
|
||||
|
|
|
|||
|
|
@ -16,11 +16,11 @@ use semver::Version as SemverVersion;
|
|||
use tree_sitter::{ffi, Parser, Point};
|
||||
use tree_sitter_cli::{
|
||||
fuzz::{
|
||||
fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED,
|
||||
LOG_GRAPH_ENABLED, START_SEED,
|
||||
fuzz_language_corpus, FuzzOptions, DEFAULT_EDIT_COUNT, DEFAULT_ITERATION_COUNT, EDIT_COUNT,
|
||||
ITERATION_COUNT, LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED,
|
||||
},
|
||||
highlight::{self, HighlightOptions},
|
||||
init::{generate_grammar_files, JsonConfigOpts},
|
||||
init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
|
||||
input::{get_input, get_tmp_source_file, CliInput},
|
||||
logger,
|
||||
parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
|
||||
|
|
@ -391,11 +391,15 @@ struct Fuzz {
|
|||
/// library's language function
|
||||
#[arg(long)]
|
||||
pub lang_name: Option<String>,
|
||||
/// Maximum number of edits to perform per fuzz test
|
||||
#[arg(long)]
|
||||
#[arg(
|
||||
long,
|
||||
help=format!("Maximum number of edits to perform per fuzz test (Default: {DEFAULT_EDIT_COUNT})")
|
||||
)]
|
||||
pub edits: Option<usize>,
|
||||
/// Number of fuzzing iterations to run per test
|
||||
#[arg(long)]
|
||||
#[arg(
|
||||
long,
|
||||
help=format!("Number of fuzzing iterations to run per test (Default: {DEFAULT_ITERATION_COUNT})")
|
||||
)]
|
||||
pub iterations: Option<usize>,
|
||||
/// Only fuzz corpus test cases whose name matches the given regex
|
||||
#[arg(long, short)]
|
||||
|
|
@ -867,10 +871,26 @@ impl Init {
|
|||
|
||||
(opts.name.clone(), Some(opts))
|
||||
} else {
|
||||
let mut json = serde_json::from_str::<TreeSitterJSON>(
|
||||
&fs::read_to_string(current_dir.join("tree-sitter.json"))
|
||||
.with_context(|| "Failed to read tree-sitter.json")?,
|
||||
)?;
|
||||
let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
|
||||
.with_context(|| "Failed to read tree-sitter.json")?;
|
||||
|
||||
let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
|
||||
if json.schema.is_none() {
|
||||
json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
|
||||
}
|
||||
|
||||
let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
|
||||
// Write the re-serialized config back, as newly added optional boolean fields
|
||||
// will be included with explicit `false`s rather than implict `null`s
|
||||
if self.update && !old_config.trim().eq(new_config.trim()) {
|
||||
info!("Updating tree-sitter.json");
|
||||
fs::write(
|
||||
current_dir.join("tree-sitter.json"),
|
||||
serde_json::to_string_pretty(&json)?,
|
||||
)
|
||||
.with_context(|| "Failed to write tree-sitter.json")?;
|
||||
}
|
||||
|
||||
(json.grammars.swap_remove(0).name, None)
|
||||
};
|
||||
|
||||
|
|
@ -955,11 +975,21 @@ impl Build {
|
|||
} else {
|
||||
let output_path = if let Some(ref path) = self.output {
|
||||
let path = Path::new(path);
|
||||
if path.is_absolute() {
|
||||
let full_path = if path.is_absolute() {
|
||||
path.to_path_buf()
|
||||
} else {
|
||||
current_dir.join(path)
|
||||
}
|
||||
};
|
||||
let parent_path = full_path
|
||||
.parent()
|
||||
.context("Output path must have a parent")?;
|
||||
let name = full_path
|
||||
.file_name()
|
||||
.context("Ouput path must have a filename")?;
|
||||
fs::create_dir_all(parent_path).context("Failed to create output path")?;
|
||||
let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
|
||||
canon_path.push(name);
|
||||
canon_path
|
||||
} else {
|
||||
let file_name = grammar_path
|
||||
.file_stem()
|
||||
|
|
@ -984,7 +1014,7 @@ impl Build {
|
|||
|
||||
loader
|
||||
.compile_parser_at_path(&grammar_path, output_path, flags)
|
||||
.unwrap();
|
||||
.context("Failed to compile parser")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1622,6 +1652,7 @@ impl Highlight {
|
|||
let loader_config = config.get()?;
|
||||
loader.find_all_languages(&loader_config)?;
|
||||
loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
|
||||
let cancellation_flag = util::cancel_on_signal();
|
||||
|
||||
|
|
@ -1702,7 +1733,6 @@ impl Highlight {
|
|||
} => {
|
||||
let path = get_tmp_source_file(&contents)?;
|
||||
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
let language = languages
|
||||
.iter()
|
||||
.find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
|
||||
|
|
@ -1733,7 +1763,6 @@ impl Highlight {
|
|||
if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
|
||||
(l, lc)
|
||||
} else {
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
let language = languages
|
||||
.first()
|
||||
.map(|(l, _)| l.clone())
|
||||
|
|
|
|||
|
|
@ -515,7 +515,6 @@ pub fn parse_file_at_path(
|
|||
|
||||
if opts.output == ParseOutput::Cst {
|
||||
render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
|
||||
println!();
|
||||
}
|
||||
|
||||
if opts.output == ParseOutput::Xml {
|
||||
|
|
@ -785,7 +784,7 @@ pub fn render_cst<'a, 'b: 'a>(
|
|||
.map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
|
||||
.max()
|
||||
.unwrap_or(1);
|
||||
let mut indent_level = 1;
|
||||
let mut indent_level = usize::from(!opts.no_ranges);
|
||||
let mut did_visit_children = false;
|
||||
let mut in_error = false;
|
||||
loop {
|
||||
|
|
@ -883,35 +882,24 @@ fn write_node_text(
|
|||
0
|
||||
};
|
||||
let formatted_line = render_line_feed(line, opts);
|
||||
if !opts.no_ranges {
|
||||
write!(
|
||||
out,
|
||||
"{}{}{}{}{}{}",
|
||||
if multiline { "\n" } else { "" },
|
||||
if multiline {
|
||||
render_node_range(opts, cursor, is_named, true, total_width, node_range)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if multiline {
|
||||
" ".repeat(indent_level + 1)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
paint(quote_color, &String::from(quote)),
|
||||
&paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
} else {
|
||||
write!(
|
||||
out,
|
||||
"\n{}{}{}{}",
|
||||
" ".repeat(indent_level + 1),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
&paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
}
|
||||
write!(
|
||||
out,
|
||||
"{}{}{}{}{}{}",
|
||||
if multiline { "\n" } else { " " },
|
||||
if multiline && !opts.no_ranges {
|
||||
render_node_range(opts, cursor, is_named, true, total_width, node_range)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if multiline {
|
||||
" ".repeat(indent_level + 1)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
paint(quote_color, &String::from(quote)),
|
||||
paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1011,10 +999,9 @@ fn cst_render_node(
|
|||
} else {
|
||||
opts.parse_theme.node_kind
|
||||
};
|
||||
write!(out, "{}", paint(kind_color, node.kind()),)?;
|
||||
write!(out, "{}", paint(kind_color, node.kind()))?;
|
||||
|
||||
if node.child_count() == 0 {
|
||||
write!(out, " ")?;
|
||||
// Node text from a pattern or external scanner
|
||||
write_node_text(
|
||||
opts,
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class BuildExt(build_ext):
|
|||
class BdistWheel(bdist_wheel):
|
||||
def get_tag(self):
|
||||
python, abi, platform = super().get_tag()
|
||||
if python.startswith("cp"):
|
||||
if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
|
||||
python, abi = "cp310", "abi3"
|
||||
return python, abi, platform
|
||||
|
||||
|
|
|
|||
|
|
@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
|
|||
render_assertion_results("queries", &self.query_results)?;
|
||||
}
|
||||
|
||||
write!(f, "{}", self.parse_stats)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -605,11 +607,13 @@ pub fn run_tests_at_path(
|
|||
test_summary: &mut TestSummary,
|
||||
) -> Result<()> {
|
||||
let test_entry = parse_tests(&opts.path)?;
|
||||
let mut _log_session = None;
|
||||
|
||||
if opts.debug_graph {
|
||||
_log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?);
|
||||
} else if opts.debug {
|
||||
let _log_session = if opts.debug_graph {
|
||||
Some(util::log_graphs(parser, "log.html", opts.open_log)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if opts.debug {
|
||||
parser.set_logger(Some(Box::new(|log_type, message| {
|
||||
if log_type == LogType::Lex {
|
||||
io::stderr().write_all(b" ").unwrap();
|
||||
|
|
|
|||
|
|
@ -3075,6 +3075,41 @@ fn test_query_matches_with_deeply_nested_patterns_with_fields() {
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_alternation_with_inner_quantifier() {
|
||||
let language = get_language("c");
|
||||
let source_code = "#include <foo>
|
||||
#include <bar>
|
||||
#include <baz>
|
||||
|
||||
// comment";
|
||||
let matches = &[
|
||||
(
|
||||
0,
|
||||
vec![
|
||||
("capture", "#include <foo>\n"),
|
||||
("capture", "#include <bar>\n"),
|
||||
("capture", "#include <baz>\n"),
|
||||
],
|
||||
),
|
||||
(0, vec![("capture", "// comment")]),
|
||||
];
|
||||
|
||||
let query = "[
|
||||
(preproc_include)+
|
||||
(comment)
|
||||
] @capture";
|
||||
let query = Query::new(&language, query).unwrap();
|
||||
assert_query_matches(&language, &query, source_code, matches);
|
||||
|
||||
let query = "[
|
||||
(comment)
|
||||
(preproc_include)+
|
||||
] @capture";
|
||||
let query = Query::new(&language, query).unwrap();
|
||||
assert_query_matches(&language, &query, source_code, matches);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_matches_with_alternations_and_predicates() {
|
||||
allocations::record(|| {
|
||||
|
|
|
|||
|
|
@ -118,6 +118,32 @@ fn test_load_fixture_language_wasm() {
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_realloc_smaller_size() {
|
||||
allocations::record(|| {
|
||||
let store = WasmStore::new(&ENGINE).unwrap();
|
||||
let mut parser = Parser::new();
|
||||
let language = get_test_fixture_language_wasm("wasm_realloc_overflow_heap");
|
||||
parser.set_wasm_store(store).unwrap();
|
||||
parser.set_language(&language).unwrap();
|
||||
let tree = parser.parse("hello", None).unwrap();
|
||||
assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_realloc_clobber_region() {
|
||||
allocations::record(|| {
|
||||
let store = WasmStore::new(&ENGINE).unwrap();
|
||||
let mut parser = Parser::new();
|
||||
let language = get_test_fixture_language_wasm("wasm_realloc_clobber_region");
|
||||
parser.set_wasm_store(store).unwrap();
|
||||
parser.set_language(&language).unwrap();
|
||||
let tree = parser.parse("hello", None).unwrap();
|
||||
assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_multiple_wasm_languages() {
|
||||
allocations::record(|| {
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ pub use parse_grammar::ParseGrammarError;
|
|||
use prepare_grammar::prepare_grammar;
|
||||
pub use prepare_grammar::PrepareGrammarError;
|
||||
use render::render_c_code;
|
||||
pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN};
|
||||
pub use render::{RenderError, ABI_VERSION_MAX, ABI_VERSION_MIN};
|
||||
|
||||
static JSON_COMMENT_REGEX: LazyLock<Regex> = LazyLock::new(|| {
|
||||
RegexBuilder::new("^\\s*//.*")
|
||||
|
|
@ -93,6 +93,8 @@ pub enum GenerateError {
|
|||
VariableInfo(#[from] VariableInfoError),
|
||||
#[error(transparent)]
|
||||
BuildTables(#[from] ParseTableBuilderError),
|
||||
#[error(transparent)]
|
||||
Render(#[from] RenderError),
|
||||
#[cfg(feature = "load")]
|
||||
#[error(transparent)]
|
||||
ParseVersion(#[from] ParseVersionError),
|
||||
|
|
@ -398,7 +400,7 @@ fn generate_parser_for_grammar_with_opts(
|
|||
abi_version,
|
||||
semantic_version,
|
||||
supertype_symbol_map,
|
||||
);
|
||||
)?;
|
||||
Ok(GeneratedParser {
|
||||
c_code,
|
||||
#[cfg(feature = "load")]
|
||||
|
|
|
|||
|
|
@ -274,9 +274,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult<InputGrammar> {
|
|||
};
|
||||
let matches_empty = match inner_rule {
|
||||
Rule::String(rule_str) => rule_str.is_empty(),
|
||||
Rule::Pattern(ref value, _) => Regex::new(value)
|
||||
.map(|reg| reg.is_match(""))
|
||||
.unwrap_or(false),
|
||||
Rule::Pattern(ref value, _) => Regex::new(value).is_ok_and(|reg| reg.is_match("")),
|
||||
_ => false,
|
||||
};
|
||||
if matches_empty {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ unless they are used only as the grammar's start rule.
|
|||
"
|
||||
)]
|
||||
EmptyString(String),
|
||||
#[error("Terminal rule '{0}' cannot be used as a supertype")]
|
||||
SupertypeTerminal(String),
|
||||
#[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")]
|
||||
ExternalTokenNonTerminal(String),
|
||||
#[error("Non-symbol rules cannot be used as external tokens")]
|
||||
|
|
@ -128,11 +130,18 @@ pub(super) fn extract_tokens(
|
|||
})
|
||||
.collect();
|
||||
|
||||
let supertype_symbols = grammar
|
||||
let supertype_symbols: Vec<Symbol> = grammar
|
||||
.supertype_symbols
|
||||
.into_iter()
|
||||
.map(|symbol| symbol_replacer.replace_symbol(symbol))
|
||||
.collect();
|
||||
for supertype_symbol in &supertype_symbols {
|
||||
if supertype_symbol.is_terminal() {
|
||||
Err(ExtractTokensError::SupertypeTerminal(
|
||||
lexical_variables[supertype_symbol.index].name.clone(),
|
||||
))?;
|
||||
}
|
||||
}
|
||||
|
||||
let variables_to_inline = grammar
|
||||
.variables_to_inline
|
||||
|
|
|
|||
|
|
@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
|
|||
let production_map = production_indices_by_step_id
|
||||
.into_iter()
|
||||
.map(|(step_id, production_indices)| {
|
||||
let production = step_id.variable_index.map_or_else(
|
||||
|| &productions[step_id.production_index],
|
||||
|variable_index| {
|
||||
&grammar.variables[variable_index].productions[step_id.production_index]
|
||||
},
|
||||
) as *const Production;
|
||||
let production =
|
||||
core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
|
||||
|| &productions[step_id.production_index],
|
||||
|variable_index| {
|
||||
&grammar.variables[variable_index].productions[step_id.production_index]
|
||||
},
|
||||
));
|
||||
((production, step_id.step_index as u32), production_indices)
|
||||
})
|
||||
.collect();
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ use std::{
|
|||
|
||||
use crate::LANGUAGE_VERSION;
|
||||
use indoc::indoc;
|
||||
use serde::Serialize;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::{
|
||||
build_tables::Tables,
|
||||
|
|
@ -25,6 +27,16 @@ pub const ABI_VERSION_MIN: usize = 14;
|
|||
pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION;
|
||||
const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
|
||||
|
||||
pub type RenderResult<T> = Result<T, RenderError>;
|
||||
|
||||
#[derive(Debug, Error, Serialize)]
|
||||
pub enum RenderError {
|
||||
#[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
|
||||
ParseTable(usize),
|
||||
#[error("This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {0}")]
|
||||
ABI(usize),
|
||||
}
|
||||
|
||||
#[clippy::format_args]
|
||||
macro_rules! add {
|
||||
($this: tt, $($arg: tt)*) => {{
|
||||
|
|
@ -104,7 +116,7 @@ struct Metadata {
|
|||
}
|
||||
|
||||
impl Generator {
|
||||
fn generate(mut self) -> String {
|
||||
fn generate(mut self) -> RenderResult<String> {
|
||||
self.init();
|
||||
self.add_header();
|
||||
self.add_includes();
|
||||
|
|
@ -161,7 +173,7 @@ impl Generator {
|
|||
self.add_reserved_word_sets();
|
||||
}
|
||||
|
||||
self.add_parse_table();
|
||||
self.add_parse_table()?;
|
||||
|
||||
if !self.syntax_grammar.external_tokens.is_empty() {
|
||||
self.add_external_token_enum();
|
||||
|
|
@ -171,7 +183,7 @@ impl Generator {
|
|||
|
||||
self.add_parser_export();
|
||||
|
||||
self.buffer
|
||||
Ok(self.buffer)
|
||||
}
|
||||
|
||||
fn init(&mut self) {
|
||||
|
|
@ -1273,7 +1285,7 @@ impl Generator {
|
|||
add_line!(self, "");
|
||||
}
|
||||
|
||||
fn add_parse_table(&mut self) {
|
||||
fn add_parse_table(&mut self) -> RenderResult<()> {
|
||||
let mut parse_table_entries = HashMap::new();
|
||||
let mut next_parse_action_list_index = 0;
|
||||
|
||||
|
|
@ -1443,6 +1455,9 @@ impl Generator {
|
|||
add_line!(self, "}};");
|
||||
add_line!(self, "");
|
||||
}
|
||||
if next_parse_action_list_index >= usize::from(u16::MAX) {
|
||||
Err(RenderError::ParseTable(next_parse_action_list_index))?;
|
||||
}
|
||||
|
||||
let mut parse_table_entries = parse_table_entries
|
||||
.into_iter()
|
||||
|
|
@ -1450,6 +1465,8 @@ impl Generator {
|
|||
.collect::<Vec<_>>();
|
||||
parse_table_entries.sort_by_key(|(index, _)| *index);
|
||||
self.add_parse_action_list(parse_table_entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_parse_action_list(&mut self, parse_table_entries: Vec<(usize, ParseTableEntry)>) {
|
||||
|
|
@ -1942,11 +1959,10 @@ pub fn render_c_code(
|
|||
abi_version: usize,
|
||||
semantic_version: Option<(u8, u8, u8)>,
|
||||
supertype_symbol_map: BTreeMap<Symbol, Vec<ChildType>>,
|
||||
) -> String {
|
||||
assert!(
|
||||
(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version),
|
||||
"This version of Tree-sitter can only generate parsers with ABI version {ABI_VERSION_MIN} - {ABI_VERSION_MAX}, not {abi_version}",
|
||||
);
|
||||
) -> RenderResult<String> {
|
||||
if !(ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version) {
|
||||
Err(RenderError::ABI(abi_version))?;
|
||||
}
|
||||
|
||||
Generator {
|
||||
language_name: name.to_string(),
|
||||
|
|
|
|||
|
|
@ -52,67 +52,96 @@ extern "C" {
|
|||
|
||||
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
|
||||
/// less than the array's current capacity, this function has no effect.
|
||||
#define array_reserve(self, new_capacity) \
|
||||
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
|
||||
#define array_reserve(self, new_capacity) \
|
||||
((self)->contents = _array__reserve( \
|
||||
(void *)(self)->contents, &(self)->capacity, \
|
||||
array_elem_size(self), new_capacity) \
|
||||
)
|
||||
|
||||
/// Free any memory allocated for this array. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_delete(self) _array__delete((Array *)(self))
|
||||
#define array_delete(self) \
|
||||
do { \
|
||||
if ((self)->contents) ts_free((self)->contents); \
|
||||
(self)->contents = NULL; \
|
||||
(self)->size = 0; \
|
||||
(self)->capacity = 0; \
|
||||
} while (0)
|
||||
|
||||
/// Push a new `element` onto the end of the array.
|
||||
#define array_push(self, element) \
|
||||
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
|
||||
(self)->contents[(self)->size++] = (element))
|
||||
#define array_push(self, element) \
|
||||
do { \
|
||||
(self)->contents = _array__grow( \
|
||||
(void *)(self)->contents, (self)->size, &(self)->capacity, \
|
||||
1, array_elem_size(self) \
|
||||
); \
|
||||
(self)->contents[(self)->size++] = (element); \
|
||||
} while(0)
|
||||
|
||||
/// Increase the array's size by `count` elements.
|
||||
/// New elements are zero-initialized.
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
_array__grow((Array *)(self), count, array_elem_size(self)); \
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
(self)->contents = _array__grow( \
|
||||
(self)->contents, (self)->size, &(self)->capacity, \
|
||||
count, array_elem_size(self) \
|
||||
); \
|
||||
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
|
||||
(self)->size += (count); \
|
||||
(self)->size += (count); \
|
||||
} while (0)
|
||||
|
||||
/// Append all elements from one array to the end of another.
|
||||
#define array_push_all(self, other) \
|
||||
#define array_push_all(self, other) \
|
||||
array_extend((self), (other)->size, (other)->contents)
|
||||
|
||||
/// Append `count` elements to the end of the array, reading their values from the
|
||||
/// `contents` pointer.
|
||||
#define array_extend(self, count, contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), (self)->size, \
|
||||
0, count, contents \
|
||||
#define array_extend(self, count, other_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), (self)->size, 0, count, other_contents \
|
||||
)
|
||||
|
||||
/// Remove `old_count` elements from the array starting at the given `index`. At
|
||||
/// the same index, insert `new_count` new elements, reading their values from the
|
||||
/// `new_contents` pointer.
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), _index, \
|
||||
old_count, new_count, new_contents \
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, old_count, new_count, new_contents \
|
||||
)
|
||||
|
||||
/// Insert one `element` into the array at the given `index`.
|
||||
#define array_insert(self, _index, element) \
|
||||
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
|
||||
#define array_insert(self, _index, element) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, 0, 1, &(element) \
|
||||
)
|
||||
|
||||
/// Remove one element from the array at the given `index`.
|
||||
#define array_erase(self, _index) \
|
||||
_array__erase((Array *)(self), array_elem_size(self), _index)
|
||||
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
|
||||
|
||||
/// Pop the last element off the array, returning the element by value.
|
||||
#define array_pop(self) ((self)->contents[--(self)->size])
|
||||
|
||||
/// Assign the contents of one array to another, reallocating if necessary.
|
||||
#define array_assign(self, other) \
|
||||
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
|
||||
#define array_assign(self, other) \
|
||||
(self)->contents = _array__assign( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
|
||||
)
|
||||
|
||||
/// Swap one array with another
|
||||
#define array_swap(self, other) \
|
||||
_array__swap((Array *)(self), (Array *)(other))
|
||||
#define array_swap(self, other) \
|
||||
do { \
|
||||
void *_array_swap_tmp = (void *)(self)->contents; \
|
||||
(self)->contents = (other)->contents; \
|
||||
(other)->contents = _array_swap_tmp; \
|
||||
_array__swap(&(self)->size, &(self)->capacity, \
|
||||
&(other)->size, &(other)->capacity); \
|
||||
} while (0)
|
||||
|
||||
/// Get the size of the array contents
|
||||
#define array_elem_size(self) (sizeof *(self)->contents)
|
||||
|
|
@ -157,82 +186,90 @@ extern "C" {
|
|||
|
||||
// Private
|
||||
|
||||
typedef Array(void) Array;
|
||||
|
||||
/// This is not what you're looking for, see `array_delete`.
|
||||
static inline void _array__delete(Array *self) {
|
||||
if (self->contents) {
|
||||
ts_free(self->contents);
|
||||
self->contents = NULL;
|
||||
self->size = 0;
|
||||
self->capacity = 0;
|
||||
}
|
||||
}
|
||||
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
|
||||
// are passed to the various `_array__*` functions below to address strict aliasing
|
||||
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
|
||||
//
|
||||
// The `Array` type itself was not altered as a solution in order to avoid breakage
|
||||
// with existing consumers (in particular, parsers with external scanners).
|
||||
|
||||
/// This is not what you're looking for, see `array_erase`.
|
||||
static inline void _array__erase(Array *self, size_t element_size,
|
||||
uint32_t index) {
|
||||
assert(index < self->size);
|
||||
char *contents = (char *)self->contents;
|
||||
static inline void _array__erase(void* self_contents, uint32_t *size,
|
||||
size_t element_size, uint32_t index) {
|
||||
assert(index < *size);
|
||||
char *contents = (char *)self_contents;
|
||||
memmove(contents + index * element_size, contents + (index + 1) * element_size,
|
||||
(self->size - index - 1) * element_size);
|
||||
self->size--;
|
||||
(*size - index - 1) * element_size);
|
||||
(*size)--;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_reserve`.
|
||||
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
|
||||
if (new_capacity > self->capacity) {
|
||||
if (self->contents) {
|
||||
self->contents = ts_realloc(self->contents, new_capacity * element_size);
|
||||
static inline void *_array__reserve(void *contents, uint32_t *capacity,
|
||||
size_t element_size, uint32_t new_capacity) {
|
||||
void *new_contents = contents;
|
||||
if (new_capacity > *capacity) {
|
||||
if (contents) {
|
||||
new_contents = ts_realloc(contents, new_capacity * element_size);
|
||||
} else {
|
||||
self->contents = ts_malloc(new_capacity * element_size);
|
||||
new_contents = ts_malloc(new_capacity * element_size);
|
||||
}
|
||||
self->capacity = new_capacity;
|
||||
*capacity = new_capacity;
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_assign`.
|
||||
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
|
||||
_array__reserve(self, element_size, other->size);
|
||||
self->size = other->size;
|
||||
memcpy(self->contents, other->contents, self->size * element_size);
|
||||
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
|
||||
const void *other_contents, uint32_t other_size, size_t element_size) {
|
||||
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
|
||||
*self_size = other_size;
|
||||
memcpy(new_contents, other_contents, *self_size * element_size);
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_swap`.
|
||||
static inline void _array__swap(Array *self, Array *other) {
|
||||
Array swap = *other;
|
||||
*other = *self;
|
||||
*self = swap;
|
||||
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
|
||||
uint32_t *other_size, uint32_t *other_capacity) {
|
||||
uint32_t tmp_size = *self_size;
|
||||
uint32_t tmp_capacity = *self_capacity;
|
||||
*self_size = *other_size;
|
||||
*self_capacity = *other_capacity;
|
||||
*other_size = tmp_size;
|
||||
*other_capacity = tmp_capacity;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
|
||||
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
|
||||
uint32_t new_size = self->size + count;
|
||||
if (new_size > self->capacity) {
|
||||
uint32_t new_capacity = self->capacity * 2;
|
||||
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
|
||||
uint32_t count, size_t element_size) {
|
||||
void *new_contents = contents;
|
||||
uint32_t new_size = size + count;
|
||||
if (new_size > *capacity) {
|
||||
uint32_t new_capacity = *capacity * 2;
|
||||
if (new_capacity < 8) new_capacity = 8;
|
||||
if (new_capacity < new_size) new_capacity = new_size;
|
||||
_array__reserve(self, element_size, new_capacity);
|
||||
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_splice`.
|
||||
static inline void _array__splice(Array *self, size_t element_size,
|
||||
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
|
||||
size_t element_size,
|
||||
uint32_t index, uint32_t old_count,
|
||||
uint32_t new_count, const void *elements) {
|
||||
uint32_t new_size = self->size + new_count - old_count;
|
||||
uint32_t new_size = *size + new_count - old_count;
|
||||
uint32_t old_end = index + old_count;
|
||||
uint32_t new_end = index + new_count;
|
||||
assert(old_end <= self->size);
|
||||
assert(old_end <= *size);
|
||||
|
||||
_array__reserve(self, element_size, new_size);
|
||||
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
|
||||
|
||||
char *contents = (char *)self->contents;
|
||||
if (self->size > old_end) {
|
||||
char *contents = (char *)new_contents;
|
||||
if (*size > old_end) {
|
||||
memmove(
|
||||
contents + new_end * element_size,
|
||||
contents + old_end * element_size,
|
||||
(self->size - old_end) * element_size
|
||||
(*size - old_end) * element_size
|
||||
);
|
||||
}
|
||||
if (new_count > 0) {
|
||||
|
|
@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
|
|||
);
|
||||
}
|
||||
}
|
||||
self->size += new_count - old_count;
|
||||
*size += new_count - old_count;
|
||||
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "tree-sitter-language"
|
||||
description = "The tree-sitter Language type, used by the library and by language implementations"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version = "1.77"
|
||||
|
|
|
|||
|
|
@ -23,9 +23,15 @@ typedef long unsigned int size_t;
|
|||
|
||||
typedef long unsigned int uintptr_t;
|
||||
|
||||
#define UINT16_MAX 65535
|
||||
#define INT8_MAX 127
|
||||
#define INT16_MAX 32767
|
||||
#define INT32_MAX 2147483647L
|
||||
#define INT64_MAX 9223372036854775807LL
|
||||
|
||||
#define UINT8_MAX 255
|
||||
#define UINT16_MAX 65535
|
||||
#define UINT32_MAX 4294967295U
|
||||
#define UINT64_MAX 18446744073709551615ULL
|
||||
|
||||
#if defined(__wasm32__)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
|
|||
|
||||
int strncmp(const char *left, const char *right, size_t n);
|
||||
|
||||
size_t strlen(const char *str);
|
||||
|
||||
#endif // TREE_SITTER_WASM_STRING_H_
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
bool left_justify; // -
|
||||
|
|
@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
|
|||
return 2 + len;
|
||||
}
|
||||
|
||||
size_t strlen(const char *str) {
|
||||
const char *s = str;
|
||||
while (*s) s++;
|
||||
return s - str;
|
||||
}
|
||||
|
||||
char *strncpy(char *dest, const char *src, size_t n) {
|
||||
char *d = dest;
|
||||
const char *s = src;
|
||||
|
|
|
|||
|
|
@ -48,6 +48,19 @@ static int grow_heap(size_t size) {
|
|||
return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX;
|
||||
}
|
||||
|
||||
// Grows the heap if necessary to fit a region at the _end_ of the heap
|
||||
// ending at `region_end` by `size` bytes.
|
||||
//
|
||||
// Returns 0 if the heap could not be grown, 1 otherwise.
|
||||
static inline int grow_heap_for_region(Region *region_end, size_t size) {
|
||||
if (region_end > heap_end) {
|
||||
if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) return 0;
|
||||
if (!grow_heap(size)) return 0;
|
||||
heap_end = get_heap_end();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Clear out the heap, and move it to the given address.
|
||||
void reset_heap(void *new_heap_start) {
|
||||
heap_start = new_heap_start;
|
||||
|
|
@ -76,13 +89,7 @@ void *malloc(size_t size) {
|
|||
|
||||
Region *region_end = region_after(next, size);
|
||||
|
||||
if (region_end > heap_end) {
|
||||
if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) {
|
||||
return NULL;
|
||||
}
|
||||
if (!grow_heap(size)) return NULL;
|
||||
heap_end = get_heap_end();
|
||||
}
|
||||
if (!grow_heap_for_region(region_end, size)) return NULL;
|
||||
|
||||
void *result = &next->data;
|
||||
next->size = size;
|
||||
|
|
@ -109,6 +116,7 @@ void free(void *ptr) {
|
|||
|
||||
void *calloc(size_t count, size_t size) {
|
||||
void *result = malloc(count * size);
|
||||
if (!result) return NULL;
|
||||
memset(result, 0, count * size);
|
||||
return result;
|
||||
}
|
||||
|
|
@ -117,19 +125,36 @@ void *realloc(void *ptr, size_t new_size) {
|
|||
if (ptr == NULL) {
|
||||
return malloc(new_size);
|
||||
}
|
||||
if (new_size == 0) {
|
||||
free(ptr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
Region *region = region_for_ptr(ptr);
|
||||
Region *region_end = region_after(region, region->size);
|
||||
|
||||
// When reallocating the last allocated region, return
|
||||
// the same pointer, and skip copying the data.
|
||||
// When reallocating the last allocated region, resize
|
||||
// in place if possible, return the same pointer, and
|
||||
// skip copying the data.
|
||||
if (region_end == next) {
|
||||
next = region;
|
||||
return malloc(new_size);
|
||||
Region *new_region_end = region_after(region, new_size);
|
||||
|
||||
size_t additional_size = (char *)new_region_end - (char *)heap_end;
|
||||
if (!grow_heap_for_region(new_region_end, additional_size)) return NULL;
|
||||
|
||||
region->size = new_size;
|
||||
next = new_region_end;
|
||||
return ®ion->data;
|
||||
}
|
||||
|
||||
void *result = malloc(new_size);
|
||||
memcpy(result, ®ion->data, region->size);
|
||||
if (!result) return NULL;
|
||||
|
||||
size_t copy_size = region->size < new_size ? region->size : new_size;
|
||||
memcpy(result, ®ion->data, copy_size);
|
||||
|
||||
free(ptr);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t strlen(const char *str) {
|
||||
const char *s = str;
|
||||
while (*s) s++;
|
||||
return s - str;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::fmt::Write as _;
|
||||
#[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))]
|
||||
use std::ops::Range;
|
||||
#[cfg(feature = "tree-sitter-highlight")]
|
||||
|
|
@ -8,6 +10,7 @@ use std::sync::Mutex;
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
env, fs,
|
||||
hash::{Hash as _, Hasher as _},
|
||||
io::{BufRead, BufReader},
|
||||
marker::PhantomData,
|
||||
mem,
|
||||
|
|
@ -75,8 +78,6 @@ pub enum LoaderError {
|
|||
NoLanguage,
|
||||
#[error(transparent)]
|
||||
Query(LoaderQueryError),
|
||||
#[error(transparent)]
|
||||
ScannerSymbols(ScannerSymbolError),
|
||||
#[error("Failed to load language for scope '{0}':\n{1}")]
|
||||
ScopeLoad(String, Box<Self>),
|
||||
#[error(transparent)]
|
||||
|
|
@ -199,28 +200,6 @@ impl std::fmt::Display for SymbolError {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub struct ScannerSymbolError {
|
||||
pub missing: Vec<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScannerSymbolError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
writeln!(
|
||||
f,
|
||||
"Missing required functions in the external scanner, parsing won't work without these!\n"
|
||||
)?;
|
||||
for symbol in &self.missing {
|
||||
writeln!(f, " `{symbol}`")?;
|
||||
}
|
||||
writeln!(
|
||||
f,
|
||||
"You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n"
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub struct WasiSDKClangError {
|
||||
pub wasi_sdk_dir: String,
|
||||
|
|
@ -1025,20 +1004,26 @@ impl Loader {
|
|||
return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
|
||||
}
|
||||
|
||||
// Create a unique lock path based on the output path hash to prevent
|
||||
// interference when multiple processes build the same grammar (by name)
|
||||
// to different output locations
|
||||
let lock_hash = {
|
||||
let mut hasher = std::hash::DefaultHasher::new();
|
||||
output_path.hash(&mut hasher);
|
||||
format!("{:x}", hasher.finish())
|
||||
};
|
||||
|
||||
let lock_path = if env::var("CROSS_RUNNER").is_ok() {
|
||||
tempfile::tempdir()
|
||||
.unwrap()
|
||||
.expect("create a temp dir")
|
||||
.path()
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}.lock", config.name))
|
||||
.to_path_buf()
|
||||
} else {
|
||||
etcetera::choose_base_strategy()?
|
||||
.cache_dir()
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}.lock", config.name))
|
||||
};
|
||||
etcetera::choose_base_strategy()?.cache_dir()
|
||||
}
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}-{lock_hash}.lock", config.name));
|
||||
|
||||
if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
|
||||
recompile = false;
|
||||
|
|
@ -1085,10 +1070,30 @@ impl Loader {
|
|||
self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?;
|
||||
|
||||
if config.scanner_path.is_some() {
|
||||
self.check_external_scanner(&config.name, &output_path)?;
|
||||
self.check_external_scanner(&output_path)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the dynamic library exists before trying to load it. This can
|
||||
// happen in race conditions where we couldn't acquire the lock because
|
||||
// another process was compiling but it still hasn't finished by the
|
||||
// time we reach this point, so the output file still doesn't exist.
|
||||
//
|
||||
// Instead of allowing the `load_language` call below to fail, return a
|
||||
// clearer error to the user here.
|
||||
if !output_path.exists() {
|
||||
let msg = format!(
|
||||
"Dynamic library `{}` not found after build attempt. \
|
||||
Are you running multiple processes building to the same output location?",
|
||||
output_path.display()
|
||||
);
|
||||
|
||||
Err(LoaderError::IO(IoError::new(
|
||||
std::io::Error::new(std::io::ErrorKind::NotFound, msg),
|
||||
Some(output_path.as_path()),
|
||||
)))?;
|
||||
}
|
||||
|
||||
Self::load_language(&output_path, &language_fn_name)
|
||||
}
|
||||
|
||||
|
|
@ -1187,6 +1192,7 @@ impl Loader {
|
|||
command.arg("-UTREE_SITTER_REUSE_ALLOCATOR");
|
||||
} else {
|
||||
command.arg("-shared");
|
||||
command.arg("-Wl,--no-undefined");
|
||||
}
|
||||
command.args(cc_config.get_files());
|
||||
command.arg("-o").arg(output_path);
|
||||
|
|
@ -1221,25 +1227,15 @@ impl Loader {
|
|||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> {
|
||||
let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) {
|
||||
"_"
|
||||
fn check_external_scanner(&self, library_path: &Path) -> LoaderResult<()> {
|
||||
let section = " T ";
|
||||
// Older ppc toolchains incorrectly report functions in the Data section. This bug has been
|
||||
// fixed, but we still need to account for older systems.
|
||||
let old_ppc_section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
|
||||
Some(" D ")
|
||||
} else {
|
||||
""
|
||||
None
|
||||
};
|
||||
let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) {
|
||||
" D "
|
||||
} else {
|
||||
" T "
|
||||
};
|
||||
let mut must_have = vec![
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_create"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_destroy"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_serialize"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_deserialize"),
|
||||
format!("{prefix}tree_sitter_{name}_external_scanner_scan"),
|
||||
];
|
||||
|
||||
let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned());
|
||||
let command = Command::new(nm_cmd)
|
||||
.arg("--defined-only")
|
||||
|
|
@ -1247,54 +1243,41 @@ impl Loader {
|
|||
.output();
|
||||
if let Ok(output) = command {
|
||||
if output.status.success() {
|
||||
let mut found_non_static = false;
|
||||
let mut non_static_symbols = String::new();
|
||||
for line in String::from_utf8_lossy(&output.stdout).lines() {
|
||||
if line.contains(section) {
|
||||
if line.contains(section) || old_ppc_section.is_some_and(|s| line.contains(s)) {
|
||||
if let Some(function_name) =
|
||||
line.split_whitespace().collect::<Vec<_>>().get(2)
|
||||
{
|
||||
if !line.contains("tree_sitter_") {
|
||||
if !found_non_static {
|
||||
found_non_static = true;
|
||||
warn!("Found non-static non-tree-sitter functions in the external scanner");
|
||||
}
|
||||
warn!(" `{function_name}`");
|
||||
} else {
|
||||
must_have.retain(|f| f != function_name);
|
||||
writeln!(&mut non_static_symbols, " `{function_name}`").unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if found_non_static {
|
||||
warn!(concat!(
|
||||
"Consider making these functions static, they can cause conflicts ",
|
||||
"when another tree-sitter project uses the same function name."
|
||||
));
|
||||
}
|
||||
|
||||
if !must_have.is_empty() {
|
||||
return Err(LoaderError::ScannerSymbols(ScannerSymbolError {
|
||||
missing: must_have,
|
||||
}));
|
||||
if !non_static_symbols.is_empty() {
|
||||
warn!(
|
||||
"Found non-static non-tree-sitter functions in the external scanner\n{non_static_symbols}\n{}",
|
||||
concat!(
|
||||
"Consider making these functions static, they can cause conflicts ",
|
||||
"when another tree-sitter project uses the same function name."
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
"Failed to run `nm` to verify symbols in {}",
|
||||
library_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> {
|
||||
fn check_external_scanner(&self, _library_path: &Path) -> LoaderResult<()> {
|
||||
// TODO: there's no nm command on windows, whoever wants to implement this can and should :)
|
||||
|
||||
// let mut must_have = vec![
|
||||
// format!("tree_sitter_{name}_external_scanner_create"),
|
||||
// format!("tree_sitter_{name}_external_scanner_destroy"),
|
||||
// format!("tree_sitter_{name}_external_scanner_serialize"),
|
||||
// format!("tree_sitter_{name}_external_scanner_deserialize"),
|
||||
// format!("tree_sitter_{name}_external_scanner_scan"),
|
||||
// ];
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
|
|||
"-D", "NDEBUG=",
|
||||
"-D", "_POSIX_C_SOURCE=200112L",
|
||||
"-D", "_DEFAULT_SOURCE=",
|
||||
"-D", "_BSD_SOURCE=",
|
||||
"-D", "_DARWIN_C_SOURCE=",
|
||||
"-I", "lib/src",
|
||||
"-I", "lib/include",
|
||||
|
|
|
|||
|
|
@ -1,19 +1,74 @@
|
|||
use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION};
|
||||
use crate::{bail_on_err, root_dir, EMSCRIPTEN_VERSION};
|
||||
use anyhow::Result;
|
||||
use std::{fs, process::Command};
|
||||
use std::{fs, path::Path, process::Command};
|
||||
|
||||
pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
||||
enum FixtureRef<'a> {
|
||||
Tag(&'a str),
|
||||
Branch(&'a str),
|
||||
}
|
||||
|
||||
impl<'a> FixtureRef<'a> {
|
||||
#[allow(clippy::use_self)]
|
||||
const fn new(tag: &'a str, branch: Option<&'a str>) -> FixtureRef<'a> {
|
||||
if let Some(b) = branch {
|
||||
Self::Branch(b)
|
||||
} else {
|
||||
Self::Tag(tag)
|
||||
}
|
||||
}
|
||||
|
||||
const fn ref_type(&self) -> &'static str {
|
||||
match self {
|
||||
FixtureRef::Tag(_) => "tag",
|
||||
FixtureRef::Branch(_) => "branch",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FixtureRef<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FixtureRef::Tag(tag) => write!(f, "{tag}"),
|
||||
FixtureRef::Branch(branch) => write!(f, "{branch}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn current_ref_name(grammar_dir: &Path) -> Result<(String, Option<&'static str>)> {
|
||||
let tag_args = ["describe", "--tags", "--exact-match", "HEAD"];
|
||||
let branch_args = ["rev-parse", "--abbrev-ref", "HEAD"];
|
||||
|
||||
for (args, ref_type) in [tag_args.as_ref(), branch_args.as_ref()]
|
||||
.iter()
|
||||
.zip(&["tag", "branch"])
|
||||
{
|
||||
let name_cmd = Command::new("git")
|
||||
.current_dir(grammar_dir)
|
||||
.args(*args)
|
||||
.output()?;
|
||||
let name = String::from_utf8_lossy(&name_cmd.stdout);
|
||||
let name = name.trim();
|
||||
if !name.is_empty() {
|
||||
return Ok((name.to_string(), Some(ref_type)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(("<unknown>".to_string(), None))
|
||||
}
|
||||
|
||||
pub fn run_fixtures() -> Result<()> {
|
||||
let fixtures_dir = root_dir().join("test").join("fixtures");
|
||||
let grammars_dir = fixtures_dir.join("grammars");
|
||||
let fixtures_path = fixtures_dir.join("fixtures.json");
|
||||
|
||||
// grammar name, tag
|
||||
let mut fixtures: Vec<(String, String)> =
|
||||
// grammar name, tag, [branch]
|
||||
let fixtures: Vec<(String, String, Option<String>)> =
|
||||
serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?;
|
||||
|
||||
for (grammar, tag) in &mut fixtures {
|
||||
let grammar_dir = grammars_dir.join(&grammar);
|
||||
for (grammar, tag, branch) in &fixtures {
|
||||
let grammar_dir = grammars_dir.join(grammar);
|
||||
let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}");
|
||||
let target_ref = FixtureRef::new(tag, branch.as_deref());
|
||||
|
||||
println!("Fetching the {grammar} grammar...");
|
||||
|
||||
|
|
@ -24,7 +79,7 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
|||
"--depth",
|
||||
"1",
|
||||
"--branch",
|
||||
tag,
|
||||
&target_ref.to_string(),
|
||||
&grammar_url,
|
||||
&grammar_dir.to_string_lossy(),
|
||||
]);
|
||||
|
|
@ -33,31 +88,71 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
|||
&format!("Failed to clone the {grammar} grammar"),
|
||||
)?;
|
||||
} else {
|
||||
let mut describe_command = Command::new("git");
|
||||
describe_command.current_dir(&grammar_dir).args([
|
||||
"describe",
|
||||
"--tags",
|
||||
"--exact-match",
|
||||
"HEAD",
|
||||
]);
|
||||
let (current_ref, current_ref_type) = current_ref_name(&grammar_dir)?;
|
||||
if current_ref != target_ref.to_string() {
|
||||
println!(
|
||||
"Updating {grammar} grammar from {} {current_ref} to {} {target_ref}...",
|
||||
current_ref_type.unwrap_or("<unknown>"),
|
||||
target_ref.ref_type(),
|
||||
);
|
||||
|
||||
let output = describe_command.output()?;
|
||||
let current_tag = String::from_utf8_lossy(&output.stdout);
|
||||
let current_tag = current_tag.trim();
|
||||
|
||||
if current_tag != tag {
|
||||
println!("Updating {grammar} grammar from {current_tag} to {tag}...");
|
||||
|
||||
let mut fetch_command = Command::new("git");
|
||||
fetch_command.current_dir(&grammar_dir).args([
|
||||
"fetch",
|
||||
"origin",
|
||||
&format!("refs/tags/{tag}:refs/tags/{tag}"),
|
||||
]);
|
||||
bail_on_err(
|
||||
&fetch_command.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to fetch tag {tag} for {grammar} grammar"),
|
||||
)?;
|
||||
match target_ref {
|
||||
FixtureRef::Branch(branch) => {
|
||||
let mut fetch_cmd = Command::new("git");
|
||||
fetch_cmd.current_dir(&grammar_dir).args([
|
||||
"fetch",
|
||||
"--update-shallow",
|
||||
"origin",
|
||||
&format!("+refs/heads/{branch}:refs/remotes/origin/{branch}"),
|
||||
]);
|
||||
bail_on_err(
|
||||
&fetch_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to fetch branch {branch}"),
|
||||
)?;
|
||||
let mut switch_cmd = Command::new("git");
|
||||
switch_cmd
|
||||
.current_dir(&grammar_dir)
|
||||
.args(["switch", branch]);
|
||||
bail_on_err(
|
||||
&switch_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to checkout branch {branch}"),
|
||||
)?;
|
||||
let mut set_upstream_cmd = Command::new("git");
|
||||
set_upstream_cmd.current_dir(&grammar_dir).args([
|
||||
"branch",
|
||||
"--set-upstream-to",
|
||||
&format!("origin/{branch}"),
|
||||
branch,
|
||||
]);
|
||||
bail_on_err(
|
||||
&set_upstream_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to set upstream for branch {branch}"),
|
||||
)?;
|
||||
let mut pull_cmd = Command::new("git");
|
||||
pull_cmd
|
||||
.current_dir(&grammar_dir)
|
||||
.args(["pull", "origin", branch]);
|
||||
bail_on_err(
|
||||
&pull_cmd.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to pull latest from branch {branch}"),
|
||||
)?;
|
||||
}
|
||||
FixtureRef::Tag(tag) => {
|
||||
let mut fetch_command = Command::new("git");
|
||||
fetch_command.current_dir(&grammar_dir).args([
|
||||
"fetch",
|
||||
"origin",
|
||||
&format!("refs/tags/{tag}:refs/tags/{tag}"),
|
||||
]);
|
||||
bail_on_err(
|
||||
&fetch_command.spawn()?.wait_with_output()?,
|
||||
&format!(
|
||||
"Failed to fetch {} {target_ref} for {grammar} grammar",
|
||||
target_ref.ref_type()
|
||||
),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut reset_command = Command::new("git");
|
||||
reset_command
|
||||
|
|
@ -71,29 +166,23 @@ pub fn run_fixtures(args: &FetchFixtures) -> Result<()> {
|
|||
let mut checkout_command = Command::new("git");
|
||||
checkout_command
|
||||
.current_dir(&grammar_dir)
|
||||
.args(["checkout", tag]);
|
||||
.args(["checkout", &target_ref.to_string()]);
|
||||
bail_on_err(
|
||||
&checkout_command.spawn()?.wait_with_output()?,
|
||||
&format!("Failed to checkout tag {tag} for {grammar} grammar"),
|
||||
&format!(
|
||||
"Failed to checkout {} {target_ref} for {grammar} grammar",
|
||||
target_ref.ref_type()
|
||||
),
|
||||
)?;
|
||||
} else {
|
||||
println!("{grammar} grammar is already at tag {tag}");
|
||||
println!(
|
||||
"{grammar} grammar is already at {} {target_ref}",
|
||||
target_ref.ref_type()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if args.update {
|
||||
println!("Updating the fixtures lock file");
|
||||
fs::write(
|
||||
&fixtures_path,
|
||||
// format the JSON without extra newlines
|
||||
serde_json::to_string(&fixtures)?
|
||||
.replace("[[", "[\n [")
|
||||
.replace("],", "],\n ")
|
||||
.replace("]]", "]\n]"),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ enum Commands {
|
|||
/// Fetches emscripten.
|
||||
FetchEmscripten,
|
||||
/// Fetches the fixtures for testing tree-sitter.
|
||||
FetchFixtures(FetchFixtures),
|
||||
FetchFixtures,
|
||||
/// Generate the Rust bindings from the C library.
|
||||
GenerateBindings,
|
||||
/// Generates the fixtures for testing tree-sitter.
|
||||
|
|
@ -118,13 +118,6 @@ struct Clippy {
|
|||
package: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct FetchFixtures {
|
||||
/// Update all fixtures to the latest tag
|
||||
#[arg(long, short)]
|
||||
update: bool,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct GenerateFixtures {
|
||||
/// Generates the parser to Wasm
|
||||
|
|
@ -232,8 +225,8 @@ fn run() -> Result<()> {
|
|||
Commands::CheckWasmExports(check_options) => check_wasm_exports::run(&check_options)?,
|
||||
Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?,
|
||||
Commands::FetchEmscripten => fetch::run_emscripten()?,
|
||||
Commands::FetchFixtures(fetch_fixture_options) => {
|
||||
fetch::run_fixtures(&fetch_fixture_options)?;
|
||||
Commands::FetchFixtures => {
|
||||
fetch::run_fixtures()?;
|
||||
}
|
||||
Commands::GenerateBindings => generate::run_bindings()?,
|
||||
Commands::GenerateFixtures(generate_fixtures_options) => {
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
|
|||
}
|
||||
|
||||
window.initializePlayground = async (opts) => {
|
||||
const { Parser, Language } = window.TreeSitter;
|
||||
const { Parser, Language, Query } = window.TreeSitter;
|
||||
|
||||
const { local } = opts;
|
||||
if (local) {
|
||||
|
|
@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
|
|||
marks.forEach((m) => m.clear());
|
||||
|
||||
if (tree && query) {
|
||||
const captures = query.captures(
|
||||
tree.rootNode,
|
||||
{ row: startRow, column: 0 },
|
||||
{ row: endRow, column: 0 },
|
||||
);
|
||||
const captures = query.captures(tree.rootNode, {
|
||||
startPosition: { row: startRow, column: 0 },
|
||||
endPosition: { row: endRow, column: 0 },
|
||||
});
|
||||
let lastNodeId;
|
||||
for (const { name, node } of captures) {
|
||||
if (node.id === lastNodeId) continue;
|
||||
|
|
@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
|
|||
const queryText = queryEditor.getValue();
|
||||
|
||||
try {
|
||||
query = parser.language.query(queryText);
|
||||
query = new Query(parser.language, queryText);
|
||||
let match;
|
||||
|
||||
let row = 0;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
"name": {
|
||||
"type": "string",
|
||||
"description": "The name of the grammar.",
|
||||
"pattern": "^[a-z0-9_]+$"
|
||||
"pattern": "^[a-z0-9_-]+$"
|
||||
},
|
||||
"camelcase": {
|
||||
"type": "string",
|
||||
|
|
|
|||
|
|
@ -31,11 +31,13 @@ If `--lib-path` is used, the name of the language used to extract the library's
|
|||
|
||||
### `--edits <EDITS>`
|
||||
|
||||
The maximum number of edits to perform. The default is 3.
|
||||
The maximum number of edits to perform. The default is 3. This value can also be set via the `TREE_SITTER_EDITS` environment
|
||||
variable.
|
||||
|
||||
### `--iterations <ITERATIONS>`
|
||||
|
||||
The number of iterations to run. The default is 10.
|
||||
The number of iterations to run. The default is 10. This value can also be set via the `TREE_SITTER_ITERATIONS` environment
|
||||
variable.
|
||||
|
||||
### `-i/--include <INCLUDE>`
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
eachSystem = lib.genAttrs systems;
|
||||
pkgsFor = inputs.nixpkgs.legacyPackages;
|
||||
|
||||
version = "0.26.3";
|
||||
version = "0.26.6";
|
||||
|
||||
fs = lib.fileset;
|
||||
src = fs.toSource {
|
||||
|
|
@ -333,6 +333,7 @@
|
|||
pkg-config
|
||||
llvm
|
||||
clang
|
||||
clang-tools
|
||||
libclang
|
||||
|
||||
nodejs_22
|
||||
|
|
@ -348,6 +349,7 @@
|
|||
];
|
||||
|
||||
shellHook = ''
|
||||
export PATH="${pkgs.clang-tools}/bin:$PATH"
|
||||
echo "Tree-sitter Dev Environment"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ tree-sitter-language.workspace = true
|
|||
streaming-iterator = "0.1.9"
|
||||
|
||||
[dependencies.wasmtime-c-api]
|
||||
version = "33.0.2"
|
||||
version = "36.0.6"
|
||||
optional = true
|
||||
package = "wasmtime-c-api-impl"
|
||||
default-features = false
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ fn main() {
|
|||
.include(&include_path)
|
||||
.define("_POSIX_C_SOURCE", "200112L")
|
||||
.define("_DEFAULT_SOURCE", None)
|
||||
.define("_BSD_SOURCE", None)
|
||||
.define("_DARWIN_C_SOURCE", None)
|
||||
.warnings(false)
|
||||
.file(src_path.join("lib.c"))
|
||||
|
|
|
|||
4
lib/binding_web/package-lock.json
generated
4
lib/binding_web/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.6",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.6",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.26.6",
|
||||
"description": "Tree-sitter bindings for the web",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ async function build() {
|
|||
keepNames: true,
|
||||
external: ['fs/*', 'fs/promises'],
|
||||
resolveExtensions: ['.ts', '.js', format === 'esm' ? '.mjs' : '.cjs'],
|
||||
...(format === 'cjs' ? {
|
||||
footer: { js: 'module.exports.default = module.exports;' },
|
||||
} : {}),
|
||||
});
|
||||
|
||||
// Copy the Wasm files to the appropriate spot, as esbuild doesn't "bundle" Wasm files
|
||||
|
|
|
|||
181
lib/src/array.h
181
lib/src/array.h
|
|
@ -52,67 +52,96 @@ extern "C" {
|
|||
|
||||
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
|
||||
/// less than the array's current capacity, this function has no effect.
|
||||
#define array_reserve(self, new_capacity) \
|
||||
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
|
||||
#define array_reserve(self, new_capacity) \
|
||||
((self)->contents = _array__reserve( \
|
||||
(void *)(self)->contents, &(self)->capacity, \
|
||||
array_elem_size(self), new_capacity) \
|
||||
)
|
||||
|
||||
/// Free any memory allocated for this array. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_delete(self) _array__delete((Array *)(self))
|
||||
#define array_delete(self) \
|
||||
do { \
|
||||
if ((self)->contents) ts_free((self)->contents); \
|
||||
(self)->contents = NULL; \
|
||||
(self)->size = 0; \
|
||||
(self)->capacity = 0; \
|
||||
} while (0)
|
||||
|
||||
/// Push a new `element` onto the end of the array.
|
||||
#define array_push(self, element) \
|
||||
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
|
||||
(self)->contents[(self)->size++] = (element))
|
||||
#define array_push(self, element) \
|
||||
do { \
|
||||
(self)->contents = _array__grow( \
|
||||
(void *)(self)->contents, (self)->size, &(self)->capacity, \
|
||||
1, array_elem_size(self) \
|
||||
); \
|
||||
(self)->contents[(self)->size++] = (element); \
|
||||
} while(0)
|
||||
|
||||
/// Increase the array's size by `count` elements.
|
||||
/// New elements are zero-initialized.
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
_array__grow((Array *)(self), count, array_elem_size(self)); \
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
(self)->contents = _array__grow( \
|
||||
(self)->contents, (self)->size, &(self)->capacity, \
|
||||
count, array_elem_size(self) \
|
||||
); \
|
||||
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
|
||||
(self)->size += (count); \
|
||||
(self)->size += (count); \
|
||||
} while (0)
|
||||
|
||||
/// Append all elements from one array to the end of another.
|
||||
#define array_push_all(self, other) \
|
||||
#define array_push_all(self, other) \
|
||||
array_extend((self), (other)->size, (other)->contents)
|
||||
|
||||
/// Append `count` elements to the end of the array, reading their values from the
|
||||
/// `contents` pointer.
|
||||
#define array_extend(self, count, contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), (self)->size, \
|
||||
0, count, contents \
|
||||
#define array_extend(self, count, other_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), (self)->size, 0, count, other_contents \
|
||||
)
|
||||
|
||||
/// Remove `old_count` elements from the array starting at the given `index`. At
|
||||
/// the same index, insert `new_count` new elements, reading their values from the
|
||||
/// `new_contents` pointer.
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), _index, \
|
||||
old_count, new_count, new_contents \
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, old_count, new_count, new_contents \
|
||||
)
|
||||
|
||||
/// Insert one `element` into the array at the given `index`.
|
||||
#define array_insert(self, _index, element) \
|
||||
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
|
||||
#define array_insert(self, _index, element) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, 0, 1, &(element) \
|
||||
)
|
||||
|
||||
/// Remove one element from the array at the given `index`.
|
||||
#define array_erase(self, _index) \
|
||||
_array__erase((Array *)(self), array_elem_size(self), _index)
|
||||
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
|
||||
|
||||
/// Pop the last element off the array, returning the element by value.
|
||||
#define array_pop(self) ((self)->contents[--(self)->size])
|
||||
|
||||
/// Assign the contents of one array to another, reallocating if necessary.
|
||||
#define array_assign(self, other) \
|
||||
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
|
||||
#define array_assign(self, other) \
|
||||
(self)->contents = _array__assign( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
|
||||
)
|
||||
|
||||
/// Swap one array with another
|
||||
#define array_swap(self, other) \
|
||||
_array__swap((Array *)(self), (Array *)(other))
|
||||
#define array_swap(self, other) \
|
||||
do { \
|
||||
void *_array_swap_tmp = (void *)(self)->contents; \
|
||||
(self)->contents = (other)->contents; \
|
||||
(other)->contents = _array_swap_tmp; \
|
||||
_array__swap(&(self)->size, &(self)->capacity, \
|
||||
&(other)->size, &(other)->capacity); \
|
||||
} while (0)
|
||||
|
||||
/// Get the size of the array contents
|
||||
#define array_elem_size(self) (sizeof *(self)->contents)
|
||||
|
|
@ -157,82 +186,90 @@ extern "C" {
|
|||
|
||||
// Private
|
||||
|
||||
typedef Array(void) Array;
|
||||
|
||||
/// This is not what you're looking for, see `array_delete`.
|
||||
static inline void _array__delete(Array *self) {
|
||||
if (self->contents) {
|
||||
ts_free(self->contents);
|
||||
self->contents = NULL;
|
||||
self->size = 0;
|
||||
self->capacity = 0;
|
||||
}
|
||||
}
|
||||
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
|
||||
// are passed to the various `_array__*` functions below to address strict aliasing
|
||||
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
|
||||
//
|
||||
// The `Array` type itself was not altered as a solution in order to avoid breakage
|
||||
// with existing consumers (in particular, parsers with external scanners).
|
||||
|
||||
/// This is not what you're looking for, see `array_erase`.
|
||||
static inline void _array__erase(Array *self, size_t element_size,
|
||||
uint32_t index) {
|
||||
ts_assert(index < self->size);
|
||||
char *contents = (char *)self->contents;
|
||||
static inline void _array__erase(void* self_contents, uint32_t *size,
|
||||
size_t element_size, uint32_t index) {
|
||||
ts_assert(index < *size);
|
||||
char *contents = (char *)self_contents;
|
||||
memmove(contents + index * element_size, contents + (index + 1) * element_size,
|
||||
(self->size - index - 1) * element_size);
|
||||
self->size--;
|
||||
(*size - index - 1) * element_size);
|
||||
(*size)--;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_reserve`.
|
||||
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
|
||||
if (new_capacity > self->capacity) {
|
||||
if (self->contents) {
|
||||
self->contents = ts_realloc(self->contents, new_capacity * element_size);
|
||||
static inline void *_array__reserve(void *contents, uint32_t *capacity,
|
||||
size_t element_size, uint32_t new_capacity) {
|
||||
void *new_contents = contents;
|
||||
if (new_capacity > *capacity) {
|
||||
if (contents) {
|
||||
new_contents = ts_realloc(contents, new_capacity * element_size);
|
||||
} else {
|
||||
self->contents = ts_malloc(new_capacity * element_size);
|
||||
new_contents = ts_malloc(new_capacity * element_size);
|
||||
}
|
||||
self->capacity = new_capacity;
|
||||
*capacity = new_capacity;
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_assign`.
|
||||
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
|
||||
_array__reserve(self, element_size, other->size);
|
||||
self->size = other->size;
|
||||
memcpy(self->contents, other->contents, self->size * element_size);
|
||||
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
|
||||
const void *other_contents, uint32_t other_size, size_t element_size) {
|
||||
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
|
||||
*self_size = other_size;
|
||||
memcpy(new_contents, other_contents, *self_size * element_size);
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_swap`.
|
||||
static inline void _array__swap(Array *self, Array *other) {
|
||||
Array swap = *other;
|
||||
*other = *self;
|
||||
*self = swap;
|
||||
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
|
||||
uint32_t *other_size, uint32_t *other_capacity) {
|
||||
uint32_t tmp_size = *self_size;
|
||||
uint32_t tmp_capacity = *self_capacity;
|
||||
*self_size = *other_size;
|
||||
*self_capacity = *other_capacity;
|
||||
*other_size = tmp_size;
|
||||
*other_capacity = tmp_capacity;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
|
||||
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
|
||||
uint32_t new_size = self->size + count;
|
||||
if (new_size > self->capacity) {
|
||||
uint32_t new_capacity = self->capacity * 2;
|
||||
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
|
||||
uint32_t count, size_t element_size) {
|
||||
void *new_contents = contents;
|
||||
uint32_t new_size = size + count;
|
||||
if (new_size > *capacity) {
|
||||
uint32_t new_capacity = *capacity * 2;
|
||||
if (new_capacity < 8) new_capacity = 8;
|
||||
if (new_capacity < new_size) new_capacity = new_size;
|
||||
_array__reserve(self, element_size, new_capacity);
|
||||
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_splice`.
|
||||
static inline void _array__splice(Array *self, size_t element_size,
|
||||
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
|
||||
size_t element_size,
|
||||
uint32_t index, uint32_t old_count,
|
||||
uint32_t new_count, const void *elements) {
|
||||
uint32_t new_size = self->size + new_count - old_count;
|
||||
uint32_t new_size = *size + new_count - old_count;
|
||||
uint32_t old_end = index + old_count;
|
||||
uint32_t new_end = index + new_count;
|
||||
ts_assert(old_end <= self->size);
|
||||
ts_assert(old_end <= *size);
|
||||
|
||||
_array__reserve(self, element_size, new_size);
|
||||
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
|
||||
|
||||
char *contents = (char *)self->contents;
|
||||
if (self->size > old_end) {
|
||||
char *contents = (char *)new_contents;
|
||||
if (*size > old_end) {
|
||||
memmove(
|
||||
contents + new_end * element_size,
|
||||
contents + old_end * element_size,
|
||||
(self->size - old_end) * element_size
|
||||
(*size - old_end) * element_size
|
||||
);
|
||||
}
|
||||
if (new_count > 0) {
|
||||
|
|
@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
|
|||
);
|
||||
}
|
||||
}
|
||||
self->size += new_count - old_count;
|
||||
*size += new_count - old_count;
|
||||
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
defined(__GNU__) || \
|
||||
defined(__HAIKU__) || \
|
||||
defined(__illumos__) || \
|
||||
defined(__redox__) || \
|
||||
defined(__NetBSD__) || \
|
||||
defined(__OpenBSD__) || \
|
||||
defined(__CYGWIN__) || \
|
||||
|
|
|
|||
|
|
@ -2958,6 +2958,53 @@ TSQuery *ts_query_new(
|
|||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Fix up quantifier loop-backs within alternations. When a branch of an
|
||||
// alternation has a + or * quantifier, the quantifier's pass_through step
|
||||
// loops back to the branch's first step. However, the alternation linking
|
||||
// assigns that same step's `alternative_index` to point to the _next_ branch.
|
||||
// This causes the quantifier loop to incorrectly explore other alternation branches,
|
||||
// when a quantified branch matches, loops back, and then fails to match. To correct
|
||||
// this, we create "clean" copies of the branches' first steps without the link to the
|
||||
// next branch. After a quantified branch matches, it loops back to the cleaned copy.
|
||||
{
|
||||
uint32_t pat_start = pattern->steps.offset;
|
||||
uint32_t pat_end = pat_start + pattern->steps.length - 1; // exclude DONE
|
||||
|
||||
for (uint32_t i = pat_start; i < pat_end; i++) {
|
||||
QueryStep *s = array_get(&self->steps, i);
|
||||
// Ensure this step is a pass_through with a _backward_ alternative (a quantifier loop-back)
|
||||
if (!s->is_pass_through
|
||||
|| s->alternative_index == NONE || s->alternative_index >= i) continue;
|
||||
|
||||
uint32_t target_idx = s->alternative_index;
|
||||
QueryStep *target = array_get(&self->steps, target_idx);
|
||||
|
||||
// Check if the target has a forward alternative from alternation linking
|
||||
uint16_t target_alt_index = target->alternative_index;
|
||||
if (target_alt_index == NONE
|
||||
|| target_alt_index <= target_idx || target_alt_index >= pat_end) continue;
|
||||
|
||||
// Create a clean copy of the target step without the alternation alternative.
|
||||
uint32_t copy_idx = self->steps.size;
|
||||
QueryStep copy = *target;
|
||||
copy.alternative_index = NONE;
|
||||
uint16_t target_depth = target->depth;
|
||||
array_push(&self->steps, copy);
|
||||
|
||||
// Add a dead_end that redirects to the pass through step after the target,
|
||||
// so the pattern continues correctly after the cleaned copy matches.
|
||||
QueryStep redirect = query_step__new(0, target_depth, false);
|
||||
redirect.is_dead_end = true;
|
||||
redirect.alternative_index = target_idx + 1;
|
||||
array_push(&self->steps, redirect);
|
||||
|
||||
// Update the pass_through to loop back to the copy. Reacquire `s` since
|
||||
// `self->steps` may have been reallocated.
|
||||
s = array_get(&self->steps, i);
|
||||
s->alternative_index = copy_idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!ts_query__analyze_patterns(self, error_offset)) {
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
|
|||
// Get a subtree's children, which are allocated immediately before the
|
||||
// tree's own heap data.
|
||||
#define ts_subtree_children(self) \
|
||||
((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
|
||||
((self).data.is_inline ? (Subtree *)NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
|
||||
|
||||
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) {
|
||||
if (self->data.is_inline) {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -80,15 +80,15 @@ typedef struct {
|
|||
} LanguageWasmInstance;
|
||||
|
||||
typedef struct {
|
||||
uint32_t reset_heap;
|
||||
uint32_t proc_exit;
|
||||
uint32_t abort;
|
||||
uint32_t assert_fail;
|
||||
uint32_t notify_memory_growth;
|
||||
uint32_t debug_message;
|
||||
uint32_t at_exit;
|
||||
uint32_t args_get;
|
||||
uint32_t args_sizes_get;
|
||||
wasmtime_func_t reset_heap;
|
||||
wasmtime_func_t proc_exit;
|
||||
wasmtime_func_t abort;
|
||||
wasmtime_func_t assert_fail;
|
||||
wasmtime_func_t notify_memory_growth;
|
||||
wasmtime_func_t debug_message;
|
||||
wasmtime_func_t at_exit;
|
||||
wasmtime_func_t args_get;
|
||||
wasmtime_func_t args_sizes_get;
|
||||
} BuiltinFunctionIndices;
|
||||
|
||||
// TSWasmStore - A struct that allows a given `Parser` to use Wasm-backed
|
||||
|
|
@ -104,7 +104,7 @@ struct TSWasmStore {
|
|||
Array(LanguageWasmInstance) language_instances;
|
||||
uint32_t current_memory_offset;
|
||||
uint32_t current_function_table_offset;
|
||||
uint32_t *stdlib_fn_indices;
|
||||
wasmtime_func_t *stdlib_fn_indices;
|
||||
BuiltinFunctionIndices builtin_fn_indices;
|
||||
wasmtime_global_t stack_pointer_global;
|
||||
wasm_globaltype_t *const_i32_type;
|
||||
|
|
@ -255,7 +255,7 @@ static bool wasm_dylink_info__parse(
|
|||
* Native callbacks exposed to Wasm modules
|
||||
*******************************************/
|
||||
|
||||
static wasm_trap_t *callback__abort(
|
||||
static wasm_trap_t *callback__abort(
|
||||
void *env,
|
||||
wasmtime_caller_t* caller,
|
||||
wasmtime_val_raw_t *args_and_results,
|
||||
|
|
@ -360,7 +360,7 @@ static wasm_trap_t *callback__lexer_eof(
|
|||
}
|
||||
|
||||
typedef struct {
|
||||
uint32_t *storage_location;
|
||||
void *storage_location;
|
||||
wasmtime_func_unchecked_callback_t callback;
|
||||
wasm_functype_t *type;
|
||||
} FunctionDefinition;
|
||||
|
|
@ -476,15 +476,11 @@ void language_id_delete(WasmLanguageId *self) {
|
|||
}
|
||||
|
||||
static wasmtime_extern_t get_builtin_extern(
|
||||
wasmtime_table_t *table,
|
||||
unsigned index
|
||||
wasmtime_func_t *func
|
||||
) {
|
||||
return (wasmtime_extern_t) {
|
||||
.kind = WASMTIME_EXTERN_FUNC,
|
||||
.of.func = (wasmtime_func_t) {
|
||||
.store_id = table->store_id,
|
||||
.__private = index
|
||||
}
|
||||
.of.func = *func
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -519,21 +515,21 @@ static bool ts_wasm_store__provide_builtin_import(
|
|||
|
||||
// Builtin functions
|
||||
else if (name_eq(import_name, "__assert_fail")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.assert_fail);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.assert_fail);
|
||||
} else if (name_eq(import_name, "__cxa_atexit")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.at_exit);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.at_exit);
|
||||
} else if (name_eq(import_name, "args_get")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_get);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.args_get);
|
||||
} else if (name_eq(import_name, "args_sizes_get")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_sizes_get);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.args_sizes_get);
|
||||
} else if (name_eq(import_name, "abort")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.abort);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.abort);
|
||||
} else if (name_eq(import_name, "proc_exit")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.proc_exit);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.proc_exit);
|
||||
} else if (name_eq(import_name, "emscripten_notify_memory_growth")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.notify_memory_growth);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.notify_memory_growth);
|
||||
} else if (name_eq(import_name, "tree_sitter_debug_message")) {
|
||||
*import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.debug_message);
|
||||
*import = get_builtin_extern(&self->builtin_fn_indices.debug_message);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -575,6 +571,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
wasmtime_module_t *stdlib_module = NULL;
|
||||
wasm_memorytype_t *memory_type = NULL;
|
||||
wasm_tabletype_t *table_type = NULL;
|
||||
wasmtime_func_t *lexer_funcs = NULL;
|
||||
|
||||
// Define functions called by scanners via function pointers on the lexer.
|
||||
LexerInWasmMemory lexer = {
|
||||
|
|
@ -583,34 +580,34 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
};
|
||||
FunctionDefinition lexer_definitions[] = {
|
||||
{
|
||||
(uint32_t *)&lexer.advance,
|
||||
&lexer.advance,
|
||||
callback__lexer_advance,
|
||||
wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.mark_end,
|
||||
&lexer.mark_end,
|
||||
callback__lexer_mark_end,
|
||||
wasm_functype_new_1_0(wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.get_column,
|
||||
&lexer.get_column,
|
||||
callback__lexer_get_column,
|
||||
wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.is_at_included_range_start,
|
||||
&lexer.is_at_included_range_start,
|
||||
callback__lexer_is_at_included_range_start,
|
||||
wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
{
|
||||
(uint32_t *)&lexer.eof,
|
||||
&lexer.eof,
|
||||
callback__lexer_eof,
|
||||
wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32())
|
||||
},
|
||||
};
|
||||
|
||||
// Define builtin functions that can be imported by scanners.
|
||||
BuiltinFunctionIndices builtin_fn_indices;
|
||||
BuiltinFunctionIndices builtin_fn_indices = {0};
|
||||
FunctionDefinition builtin_definitions[] = {
|
||||
{
|
||||
&builtin_fn_indices.proc_exit,
|
||||
|
|
@ -657,18 +654,16 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
// Create all of the Wasm functions.
|
||||
unsigned builtin_definitions_len = array_len(builtin_definitions);
|
||||
unsigned lexer_definitions_len = array_len(lexer_definitions);
|
||||
lexer_funcs = ts_calloc(lexer_definitions_len, sizeof(wasmtime_func_t));
|
||||
for (unsigned i = 0; i < builtin_definitions_len; i++) {
|
||||
FunctionDefinition *definition = &builtin_definitions[i];
|
||||
wasmtime_func_t func;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
|
||||
*definition->storage_location = func.__private;
|
||||
wasmtime_func_t *func = (wasmtime_func_t *)definition->storage_location;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, func);
|
||||
wasm_functype_delete(definition->type);
|
||||
}
|
||||
for (unsigned i = 0; i < lexer_definitions_len; i++) {
|
||||
FunctionDefinition *definition = &lexer_definitions[i];
|
||||
wasmtime_func_t func;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func);
|
||||
*definition->storage_location = func.__private;
|
||||
wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &lexer_funcs[i]);
|
||||
wasm_functype_delete(definition->type);
|
||||
}
|
||||
|
||||
|
|
@ -763,7 +758,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
.memory = memory,
|
||||
.function_table = function_table,
|
||||
.language_instances = array_new(),
|
||||
.stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(uint32_t)),
|
||||
.stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(wasmtime_func_t)),
|
||||
.builtin_fn_indices = builtin_fn_indices,
|
||||
.stack_pointer_global = stack_pointer_global,
|
||||
.current_memory_offset = 0,
|
||||
|
|
@ -816,7 +811,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
|
||||
// Process the stdlib module's exports.
|
||||
for (unsigned i = 0; i < stdlib_symbols_len; i++) {
|
||||
self->stdlib_fn_indices[i] = UINT32_MAX;
|
||||
self->stdlib_fn_indices[i] = (wasmtime_func_t){.store_id = 0};
|
||||
}
|
||||
wasmtime_module_exports(stdlib_module, &export_types);
|
||||
for (unsigned i = 0; i < export_types.size; i++) {
|
||||
|
|
@ -851,20 +846,20 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
}
|
||||
|
||||
if (name_eq(name, "reset_heap")) {
|
||||
self->builtin_fn_indices.reset_heap = export.of.func.__private;
|
||||
self->builtin_fn_indices.reset_heap = export.of.func;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (unsigned j = 0; j < stdlib_symbols_len; j++) {
|
||||
if (name_eq(name, STDLIB_SYMBOLS[j])) {
|
||||
self->stdlib_fn_indices[j] = export.of.func.__private;
|
||||
self->stdlib_fn_indices[j] = export.of.func;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (self->builtin_fn_indices.reset_heap == UINT32_MAX) {
|
||||
if (self->builtin_fn_indices.reset_heap.store_id == 0) {
|
||||
wasm_error->kind = TSWasmErrorKindInstantiate;
|
||||
format(
|
||||
&wasm_error->message,
|
||||
|
|
@ -874,7 +869,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
}
|
||||
|
||||
for (unsigned i = 0; i < stdlib_symbols_len; i++) {
|
||||
if (self->stdlib_fn_indices[i] == UINT32_MAX) {
|
||||
if (self->stdlib_fn_indices[i].store_id == 0) {
|
||||
wasm_error->kind = TSWasmErrorKindInstantiate;
|
||||
format(
|
||||
&wasm_error->message,
|
||||
|
|
@ -904,13 +899,13 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
|
|||
}
|
||||
for (unsigned i = 0; i < lexer_definitions_len; i++) {
|
||||
FunctionDefinition *definition = &lexer_definitions[i];
|
||||
wasmtime_func_t func = {function_table.store_id, *definition->storage_location};
|
||||
wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = func};
|
||||
wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = lexer_funcs[i]};
|
||||
error = wasmtime_table_set(context, &function_table, table_index, &func_val);
|
||||
ts_assert(!error);
|
||||
*(int32_t *)(definition->storage_location) = table_index;
|
||||
table_index++;
|
||||
}
|
||||
ts_free(lexer_funcs);
|
||||
|
||||
self->current_function_table_offset = table_index;
|
||||
self->lexer_address = initial_memory_pages * MEMORY_PAGE_SIZE;
|
||||
|
|
@ -937,6 +932,7 @@ error:
|
|||
if (message.size) wasm_byte_vec_delete(&message);
|
||||
if (export_types.size) wasm_exporttype_vec_delete(&export_types);
|
||||
if (imports) ts_free(imports);
|
||||
ts_free(lexer_funcs);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -1016,8 +1012,6 @@ static bool ts_wasm_store__instantiate(
|
|||
// Construct the language function name as string.
|
||||
format(&language_function_name, "tree_sitter_%s", language_name);
|
||||
|
||||
const uint64_t store_id = self->function_table.store_id;
|
||||
|
||||
// Build the imports list for the module.
|
||||
wasm_importtype_vec_t import_types = WASM_EMPTY_VEC;
|
||||
wasmtime_module_imports(module, &import_types);
|
||||
|
|
@ -1038,8 +1032,7 @@ static bool ts_wasm_store__instantiate(
|
|||
bool defined_in_stdlib = false;
|
||||
for (unsigned j = 0; j < array_len(STDLIB_SYMBOLS); j++) {
|
||||
if (name_eq(import_name, STDLIB_SYMBOLS[j])) {
|
||||
uint16_t address = self->stdlib_fn_indices[j];
|
||||
imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = {store_id, address}};
|
||||
imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = self->stdlib_fn_indices[j]};
|
||||
defined_in_stdlib = true;
|
||||
break;
|
||||
}
|
||||
|
|
@ -1546,16 +1539,13 @@ bool ts_wasm_store_add_language(
|
|||
|
||||
void ts_wasm_store_reset_heap(TSWasmStore *self) {
|
||||
wasmtime_context_t *context = wasmtime_store_context(self->store);
|
||||
wasmtime_func_t func = {
|
||||
self->function_table.store_id,
|
||||
self->builtin_fn_indices.reset_heap
|
||||
};
|
||||
wasmtime_func_t *func = &self->builtin_fn_indices.reset_heap;
|
||||
wasm_trap_t *trap = NULL;
|
||||
wasmtime_val_t args[1] = {
|
||||
{.of.i32 = ts_wasm_store__heap_address(self), .kind = WASMTIME_I32},
|
||||
};
|
||||
|
||||
wasmtime_error_t *error = wasmtime_func_call(context, &func, args, 1, NULL, 0, &trap);
|
||||
wasmtime_error_t *error = wasmtime_func_call(context, func, args, 1, NULL, 0, &trap);
|
||||
ts_assert(!error);
|
||||
ts_assert(!trap);
|
||||
}
|
||||
|
|
|
|||
32
test/fixtures/fixtures.json
vendored
32
test/fixtures/fixtures.json
vendored
|
|
@ -1,17 +1,17 @@
|
|||
[
|
||||
["bash","v0.25.0"],
|
||||
["c","v0.24.1"],
|
||||
["cpp","v0.23.4"],
|
||||
["embedded-template","v0.25.0"],
|
||||
["go","v0.25.0"],
|
||||
["html","v0.23.2"],
|
||||
["java","v0.23.5"],
|
||||
["javascript","v0.25.0"],
|
||||
["jsdoc","v0.23.2"],
|
||||
["json","v0.24.8"],
|
||||
["php","v0.24.2"],
|
||||
["python","v0.23.6"],
|
||||
["ruby","v0.23.1"],
|
||||
["rust","v0.24.0"],
|
||||
["typescript","v0.23.2"]
|
||||
]
|
||||
["bash","v0.25.0", null],
|
||||
["c","v0.24.1", null],
|
||||
["cpp","v0.23.4", null],
|
||||
["embedded-template","v0.25.0", null],
|
||||
["go","v0.25.0", null],
|
||||
["html","v0.23.2", null],
|
||||
["java","v0.23.5", null],
|
||||
["javascript","v0.25.0", null],
|
||||
["jsdoc","v0.23.2", null],
|
||||
["json","v0.24.8", null],
|
||||
["php","v0.24.2", "upstream_test_fixture"],
|
||||
["python","v0.23.6", null],
|
||||
["ruby","v0.23.1", null],
|
||||
["rust","v0.24.0", null],
|
||||
["typescript","v0.23.2", null]
|
||||
]
|
||||
|
|
|
|||
1
test/fixtures/test_grammars/terminal_supertype/expected_error.txt
vendored
Normal file
1
test/fixtures/test_grammars/terminal_supertype/expected_error.txt
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
Terminal rule 'A' cannot be used as a supertype
|
||||
18
test/fixtures/test_grammars/terminal_supertype/grammar.js
vendored
Normal file
18
test/fixtures/test_grammars/terminal_supertype/grammar.js
vendored
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
module.exports = grammar({
|
||||
name: 'terminal_supertype',
|
||||
|
||||
supertypes: $ => [$.A],
|
||||
extras: $ => [$.lineComment, $.lineComment2],
|
||||
|
||||
rules: {
|
||||
source_file: $ => choice(
|
||||
$.B,
|
||||
$.A,
|
||||
),
|
||||
lineComment: $ => /;;.*/,
|
||||
lineComment2: $ => /##.*/,
|
||||
A: $ => /xyz/,
|
||||
C: $ => "xyz",
|
||||
B: $ => seq('abc', $.C),
|
||||
}
|
||||
});
|
||||
9
test/fixtures/test_grammars/wasm_realloc_clobber_region/corpus.txt
vendored
Normal file
9
test/fixtures/test_grammars/wasm_realloc_clobber_region/corpus.txt
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
==========================
|
||||
A leading zero-width token
|
||||
==========================
|
||||
|
||||
hello
|
||||
|
||||
---
|
||||
|
||||
(document (zero_width))
|
||||
10
test/fixtures/test_grammars/wasm_realloc_clobber_region/grammar.js
vendored
Normal file
10
test/fixtures/test_grammars/wasm_realloc_clobber_region/grammar.js
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
export default grammar({
|
||||
name: 'wasm_realloc_clobber_region',
|
||||
|
||||
extras: _ => [/\s/],
|
||||
externals: $ => [$.zero_width],
|
||||
|
||||
rules: {
|
||||
document: $ => seq($.zero_width, 'hello'),
|
||||
}
|
||||
});
|
||||
68
test/fixtures/test_grammars/wasm_realloc_clobber_region/scanner.c
vendored
Normal file
68
test/fixtures/test_grammars/wasm_realloc_clobber_region/scanner.c
vendored
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
#include "tree_sitter/parser.h"
|
||||
#include <string.h>
|
||||
|
||||
#ifdef __wasm__
|
||||
#define panic(x) __builtin_trap()
|
||||
#else
|
||||
#include <assert.h>
|
||||
#define panic(x) assert(0)
|
||||
#endif
|
||||
|
||||
enum TokenType {
|
||||
ZERO_WIDTH_TOKEN
|
||||
};
|
||||
|
||||
void *tree_sitter_wasm_realloc_clobber_region_external_scanner_create(void) {
|
||||
size_t a_count = 32;
|
||||
void *a = malloc(a_count);
|
||||
memset(a, 0x41, a_count); // fill `a` with 'A's
|
||||
for (int i = 0; i < 32; i++) {
|
||||
// This condition is unreachable, but it prevents wasi-sdk clang from optimizing
|
||||
// out the allocation to `a`.
|
||||
if (((char *)a)[i] != 0x41) panic();
|
||||
}
|
||||
|
||||
size_t b_count = 16;
|
||||
void *b = malloc(b_count);
|
||||
memset(b, 0x42, b_count); // fill `b` with 'B's
|
||||
|
||||
// `a` is now first on the free list
|
||||
free(a);
|
||||
// We're re-allocating the last region, so `realloc`'s optimization to re-use the region
|
||||
// by altering its metadata's size (thus avoiding a memcpy) should be used.
|
||||
void *c = realloc(b, 24); // realloc `b` should still contain 'B's
|
||||
|
||||
// ensure the contents of `b` were not clobbered. A previous bug caused `a`s region
|
||||
// to be used for `c`.
|
||||
char *p = (char *)c;
|
||||
for (int i = 0; i < (int)b_count; i++) {
|
||||
if (p[i] != 0x42) panic();
|
||||
}
|
||||
|
||||
free(c);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool tree_sitter_wasm_realloc_clobber_region_external_scanner_scan(
|
||||
void *payload,
|
||||
TSLexer *lexer,
|
||||
const bool *valid_symbols
|
||||
) {
|
||||
lexer->result_symbol = ZERO_WIDTH_TOKEN;
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned tree_sitter_wasm_realloc_clobber_region_external_scanner_serialize(
|
||||
void *payload,
|
||||
char *buffer
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tree_sitter_wasm_realloc_clobber_region_external_scanner_deserialize(
|
||||
void *payload,
|
||||
const char *buffer,
|
||||
unsigned length
|
||||
) {}
|
||||
|
||||
void tree_sitter_wasm_realloc_clobber_region_external_scanner_destroy(void *payload) {}
|
||||
9
test/fixtures/test_grammars/wasm_realloc_overflow_heap/corpus.txt
vendored
Normal file
9
test/fixtures/test_grammars/wasm_realloc_overflow_heap/corpus.txt
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
==========================
|
||||
A leading zero-width token
|
||||
==========================
|
||||
|
||||
hello
|
||||
|
||||
---
|
||||
|
||||
(document (zero_width))
|
||||
10
test/fixtures/test_grammars/wasm_realloc_overflow_heap/grammar.js
vendored
Normal file
10
test/fixtures/test_grammars/wasm_realloc_overflow_heap/grammar.js
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
export default grammar({
|
||||
name: 'wasm_realloc_overflow_heap',
|
||||
|
||||
extras: _ => [/\s/],
|
||||
externals: $ => [$.zero_width],
|
||||
|
||||
rules: {
|
||||
document: $ => seq($.zero_width, 'hello'),
|
||||
}
|
||||
});
|
||||
43
test/fixtures/test_grammars/wasm_realloc_overflow_heap/scanner.c
vendored
Normal file
43
test/fixtures/test_grammars/wasm_realloc_overflow_heap/scanner.c
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#include "tree_sitter/parser.h"
|
||||
|
||||
// Constant copied from `crates/language/wasm/src/stdlib.c`,
|
||||
// Must be kept in sync for a reliable repro.
|
||||
#define MAX_HEAP_SIZE (4 * 1024 * 1024)
|
||||
|
||||
enum TokenType {
|
||||
ZERO_WIDTH_TOKEN
|
||||
};
|
||||
|
||||
void *tree_sitter_wasm_realloc_overflow_heap_external_scanner_create(void) {
|
||||
size_t large_size = MAX_HEAP_SIZE - 64;
|
||||
void *p = malloc(large_size);
|
||||
void *q = malloc(4);
|
||||
p = realloc(p, 4);
|
||||
free(p);
|
||||
free(q);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool tree_sitter_wasm_realloc_overflow_heap_external_scanner_scan(
|
||||
void *payload,
|
||||
TSLexer *lexer,
|
||||
const bool *valid_symbols
|
||||
) {
|
||||
lexer->result_symbol = ZERO_WIDTH_TOKEN;
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned tree_sitter_wasm_realloc_overflow_heap_external_scanner_serialize(
|
||||
void *payload,
|
||||
char *buffer
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tree_sitter_wasm_realloc_overflow_heap_external_scanner_deserialize(
|
||||
void *payload,
|
||||
const char *buffer,
|
||||
unsigned length
|
||||
) {}
|
||||
|
||||
void tree_sitter_wasm_realloc_overflow_heap_external_scanner_destroy(void *payload) {}
|
||||
Loading…
Reference in a new issue