libs-code/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:26:23 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Upgrade to latest wasmtime, improve robustness and perf of wasm-based parsing (#5847)

Browse source 
* build(deps): upgrade wasmtime C API to 48.0.0

Upgrade the Rust and Zig Wasmtime dependencies and enable reference values with the null GC collector.

Wasmtime 48 requires a newer Rust toolchain, whose Clippy version identifies three item helpers that can be const. Mark them const so the workspace continues to pass Clippy with warnings denied.

* feat(benchmark): support Wasm grammars

* perf(wasm): cache language function handles

* fix(wasm): improve validation and failure cleanup

Bounds-check dylink metadata parsing, require exact import and export names, and restore memory and function-table allocation offsets when language loading fails.

* fix(wasm): copy the complete supertype map
This commit is contained in:
Max Brunsfeld 2026-08-20 14:02:11 -07:00 committed by GitHub
parent fe3fe327e2
commit 43623ec9bf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 225 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
Cargo.lock generated
View file

Binary file not shown.

64
build.zig.zon
View file

@ -13,83 +13,83 @@
},
.dependencies = .{
.wasmtime_c_api_aarch64_android = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-aarch64-android-c-api.tar.xz",
.hash = "N-V-__8AAL8HGQRf6Aok4jexDn9MNYgTKGXu2q4gKbNPxTCN",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-aarch64-android-c-api.tar.xz",
.hash = "N-V-__8AAMDnmAUOEZqO-z-5HRo1SH_NIochKrux5zbDB0X5",
.lazy = true,
},
.wasmtime_c_api_aarch64_linux = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-aarch64-linux-c-api.tar.xz",
.hash = "N-V-__8AAFGaGQQ3nQ-ewhE9f17nT7vSSTeZXQokyIPBQ2sm",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-aarch64-linux-c-api.tar.xz",
.hash = "N-V-__8AAJpPsgVbL7LywSrreSWC5MNHxgLj3X9iNDuFa46C",
.lazy = true,
},
.wasmtime_c_api_aarch64_macos = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-aarch64-macos-c-api.tar.xz",
.hash = "N-V-__8AAB-23gIBn8QomBEoJWmRkrzxYEIJbTFJ4WO5aMUC",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-aarch64-macos-c-api.tar.xz",
.hash = "N-V-__8AAL7yOgSAHCJpyjWudbWhYxe_GKBuB6L985obibtT",
.lazy = true,
},
.wasmtime_c_api_aarch64_musl = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-aarch64-musl-c-api.tar.xz",
.hash = "N-V-__8AAOMkGATqiWukTMhp4ND7_-Q1Jcd_ItGWAHEo9cCI",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-aarch64-musl-c-api.tar.xz",
.hash = "N-V-__8AAMJ1zQUQdXeCuWPSqTsObcnX109Se-ofXuw23Br5",
.lazy = true,
},
.wasmtime_c_api_aarch64_windows = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-aarch64-windows-c-api.zip",
.hash = "N-V-__8AAFGzoQTunNKTpoAb_bu3PAgonbVyfAlvef4y33mj",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-aarch64-windows-c-api.zip",
.hash = "N-V-__8AALQQtQVnU00MdUGpMv72sqycvIzmNMADhTKjFoW4",
.lazy = true,
},
.wasmtime_c_api_armv7_linux = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-armv7-linux-c-api.tar.xz",
.hash = "N-V-__8AAIPdaQOBVVKNRPvYPcLMsz-trKZpJIgDRmJ6W8DX",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-armv7-linux-c-api.tar.xz",
.hash = "N-V-__8AAFaG6gRszx9F_KJE7cNKJKDqpcpycIjYCP5pBizT",
.lazy = true,
},
.wasmtime_c_api_i686_linux = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-i686-linux-c-api.tar.xz",
.hash = "N-V-__8AALn8pgP1tXeKwG1OqyiCMMd6RsqXQPsqdKdRFWiN",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-i686-linux-c-api.tar.xz",
.hash = "N-V-__8AAEAYMgW_ag-7_r7cFcDL89ReVYZV0qTszxcFlIYh",
.lazy = true,
},
.wasmtime_c_api_i686_windows = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-i686-windows-c-api.zip",
.hash = "N-V-__8AAAWUhgTV4gLBAlWLf96a4y_Gg5wlTdSpA6dQc51t",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-i686-windows-c-api.zip",
.hash = "N-V-__8AAC72gQVYZwTcwxsw3akAASRaqmQXb51kYvnhS1CT",
.lazy = true,
},
.wasmtime_c_api_riscv64gc_linux = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-riscv64gc-linux-c-api.tar.xz",
.hash = "N-V-__8AACk-UwUha3DFvTQT4W3p26jlQVro4GYSM5G37KlO",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-riscv64gc-linux-c-api.tar.xz",
.hash = "N-V-__8AAIjFCAciMIMsvMJRdbOhdnL5_lyiENHJKdGFnNaL",
.lazy = true,
},
.wasmtime_c_api_s390x_linux = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-s390x-linux-c-api.tar.xz",
.hash = "N-V-__8AANFzVATF92YwQrAzIfiEUGVlgV7uLZQqYIPHByLj",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-s390x-linux-c-api.tar.xz",
.hash = "N-V-__8AAADFBgYJRUzgXgyIvU4Rs4QJSMYM37FfOzsfTpT-",
.lazy = true,
},
.wasmtime_c_api_x86_64_android = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-x86_64-android-c-api.tar.xz",
.hash = "N-V-__8AAHujmQSXQZe1sJZL9Mw_DnfK_4csIgup51QLMR2K",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-x86_64-android-c-api.tar.xz",
.hash = "N-V-__8AAF7CIQYBFoplpm67_5LRgeP0FBbJWLS-9tH5Y_q2",
.lazy = true,
},
.wasmtime_c_api_x86_64_linux = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-x86_64-linux-c-api.tar.xz",
.hash = "N-V-__8AADcQmQQPW--E-rUJ2K6MPNiagGUY29Vi2g2HJWjK",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-x86_64-linux-c-api.tar.xz",
.hash = "N-V-__8AAOQtZwZ3ZxTdP71j5BUudeuEL6sNETbcOILg3fGt",
.lazy = true,
},
.wasmtime_c_api_x86_64_macos = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-x86_64-macos-c-api.tar.xz",
.hash = "N-V-__8AACdCVQPY11zv6CURvFTdeEhzAe3oO_Qjyn7886u7",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-x86_64-macos-c-api.tar.xz",
.hash = "N-V-__8AAAKFlgRutyDpY7b-EE3VkrsRwoXvtlwSCKq-oEjg",
.lazy = true,
},
.wasmtime_c_api_x86_64_mingw = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-x86_64-mingw-c-api.zip",
.hash = "N-V-__8AAF6L-QQapsAEpWJbYyHc-4BR_m75DMiqGLPFS2zt",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-x86_64-mingw-c-api.zip",
.hash = "N-V-__8AALXQxAbkf7LIM3hheOLmRWKMNeOMyVCg_mWngb2u",
.lazy = true,
},
.wasmtime_c_api_x86_64_musl = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-x86_64-musl-c-api.tar.xz",
.hash = "N-V-__8AAFk0nARolCrPj8pCSUuC9N8nwzZX81E8LJJN4y3A",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-x86_64-musl-c-api.tar.xz",
.hash = "N-V-__8AAPRwWgYSkdaLB8rJjus1EEqlnlMmpOR4Zapffazi",
.lazy = true,
},
.wasmtime_c_api_x86_64_windows = .{
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v36.0.13/wasmtime-v36.0.13-x86_64-windows-c-api.zip",
.hash = "N-V-__8AAD9IXgXlIgpLCouQK6XwGR9WnJU3j_GarV0PP_h0",
.url = "https://github.com/bytecodealliance/wasmtime/releases/download/v48.0.0/wasmtime-v48.0.0-x86_64-windows-c-api.zip",
.hash = "N-V-__8AANq6kQYVl4GwVqu4rDBPB4WAqQEgZcOCQ3sb6Lmu",
.lazy = true,
},
},

40
crates/cli/benches/benchmark.rs
View file

@ -9,6 +9,8 @@ use std::{
use anyhow::Context;
use log::info;
use tree_sitter::{Language, Parser, Query};
#[cfg(feature = "wasm")]
use tree_sitter::{WasmStore, wasmtime};
use tree_sitter_loader::{CompileConfig, Loader};
include!("../src/tests/helpers/dirs.rs");
@ -20,8 +22,11 @@ static EXAMPLE_FILTER: LazyLock<Option<String>> =
static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT").map_or(5, |s| s.parse::<usize>().unwrap())
});
static WASM: LazyLock<bool> = LazyLock::new(|| env::var_os("TREE_SITTER_BENCHMARK_WASM").is_some());
static TEST_LOADER: LazyLock<Loader> =
LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
#[cfg(feature = "wasm")]
static WASM_ENGINE: LazyLock<wasmtime::Engine> = LazyLock::new(Default::default);
#[expect(
clippy::type_complexity,
@ -94,7 +99,11 @@ fn main() {
}
info!("\nLanguage: {language_name}");
let language = get_language(language_path);
let language = if *WASM {
get_wasm_language(language_name, &mut parser)
} else {
get_language(language_path)
};
parser.set_language(&language).unwrap();
info!(" Constructing Queries");
@ -214,3 +223,32 @@ fn get_language(path: &Path) -> Language {
.with_context(|| format!("Failed to load language at path {}", src_path.display()))
.unwrap()
}
#[cfg(feature = "wasm")]
fn get_wasm_language(language_name: &str, parser: &mut Parser) -> Language {
let wasm_language_name = language_name.replace('-', "_");
let wasm_path = ROOT_DIR
.join("target")
.join("release")
.join(format!("tree-sitter-{language_name}.wasm"));
let wasm = fs::read(&wasm_path)
.with_context(|| {
format!(
"Failed to read {}. Generate Wasm fixtures with `cargo xtask generate-fixtures --wasm`",
wasm_path.display()
)
})
.unwrap();
let mut store = WasmStore::new(&WASM_ENGINE).expect("Failed to create Wasm store");
let language = store
.load_language(&wasm_language_name, &wasm)
.with_context(|| format!("Failed to load Wasm language at {}", wasm_path.display()))
.unwrap();
parser.set_wasm_store(store).unwrap();
language
}
#[cfg(not(feature = "wasm"))]
fn get_wasm_language(_language_name: &str, _parser: &mut Parser) -> Language {
panic!("Wasm benchmarking requires the `wasm` feature");
}

6
crates/generate/src/build_tables/item.rs
View file

@ -139,7 +139,7 @@ struct ItemContent<'a> {
impl Eq for ItemContent<'_> {}
impl ItemContent<'_> {
fn prec(&self) -> Precedence {
const fn prec(&self) -> Precedence {
if self.dot > 0 {
self.production.steps[self.dot - 1].precedence()
} else {
@ -147,7 +147,7 @@ impl ItemContent<'_> {
}
}
fn assoc(&self) -> Option<Associativity> {
const fn assoc(&self) -> Option<Associativity> {
if self.dot > 0 {
self.production.steps[self.dot - 1].associativity()
} else {
@ -486,7 +486,7 @@ impl<'a> ParseItem<'a> {
/// This item's identity keys at the current dot.
#[must_use]
fn dot_keys(&self) -> DotKeys {
const fn dot_keys(&self) -> DotKeys {
self.keys[self.step_index as usize]
}
}

16
crates/xtask/src/benchmark.rs
View file

@ -3,6 +3,10 @@ use anyhow::Result;
use crate::{Benchmark, bail_on_err};
pub fn run(args: &Benchmark) -> Result<()> {
if args.wasm {
unsafe { std::env::set_var("TREE_SITTER_BENCHMARK_WASM", "1") };
}
if let Some(ref example) = args.example_file_name {
unsafe { std::env::set_var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER", example) };
}
@ -26,6 +30,12 @@ pub fn run(args: &Benchmark) -> Result<()> {
.arg("benchmark")
.arg("-p")
.arg("tree-sitter-cli")
.args(
args.wasm
.then_some(["--features", "wasm"])
.into_iter()
.flatten(),
)
.arg("--no-run")
.arg("--message-format=json")
.spawn()?
@ -66,6 +76,12 @@ pub fn run(args: &Benchmark) -> Result<()> {
.arg("benchmark")
.arg("-p")
.arg("tree-sitter-cli")
.args(
args.wasm
.then_some(["--features", "wasm"])
.into_iter()
.flatten(),
)
.status()?;
if !status.success() {

3
crates/xtask/src/main.rs
View file

@ -71,6 +71,9 @@ struct Benchmark {
/// Whether to run the benchmarks in debug mode.
#[arg(long, short = 'g')]
debug: bool,
/// Benchmark Wasm grammars instead of native grammars.
#[arg(long)]
wasm: bool,
}
#[derive(Args)]

4
lib/Cargo.toml
View file

@ -47,9 +47,9 @@ streaming-iterator = "0.1.9"
tree-sitter-language.workspace = true
[dependencies.wasmtime-c-api]
version = "36.0.13"
version = "48.0.0"
default-features = false
features = [ "cranelift", "gc-drc" ]
features = [ "cranelift", "gc", "gc-null" ]
optional = true
package = "wasmtime-c-api-impl"

211
lib/src/wasm_store.c
View file

@ -75,13 +75,13 @@ typedef struct {
WasmLanguageId *language_id;
wasmtime_instance_t instance;
int32_t external_states_address;
int32_t lex_main_fn_index;
int32_t lex_keyword_fn_index;
int32_t scanner_create_fn_index;
int32_t scanner_destroy_fn_index;
int32_t scanner_serialize_fn_index;
int32_t scanner_deserialize_fn_index;
int32_t scanner_scan_fn_index;
wasmtime_func_t lex_main_fn;
wasmtime_func_t lex_keyword_fn;
wasmtime_func_t scanner_create_fn;
wasmtime_func_t scanner_destroy_fn;
wasmtime_func_t scanner_serialize_fn;
wasmtime_func_t scanner_deserialize_fn;
wasmtime_func_t scanner_scan_fn;
} LanguageWasmInstance;
typedef struct {
@ -188,19 +188,31 @@ typedef struct {
* WasmDylinkMemoryInfo
***********************/
static uint8_t read_u8(const uint8_t **p) {
return *(*p)++;
typedef struct {
const uint8_t *data;
size_t offset;
size_t size;
} WasmReader;
static bool wasm_reader__read_u8(WasmReader *reader, uint8_t *result) {
if (reader->offset >= reader->size) return false;
*result = reader->data[reader->offset++];
return true;
}
static inline uint64_t read_uleb128(const uint8_t **p, const uint8_t *end) {
uint64_t value = 0;
unsigned shift = 0;
do {
if (*p == end) return UINT64_MAX;
value += (uint64_t)(**p & 0x7f) << shift;
shift += 7;
} while (*((*p)++) >= 128);
return value;
static bool wasm_reader__read_uleb128(WasmReader *reader, uint32_t *result) {
uint32_t value = 0;
for (unsigned shift = 0; shift < 32; shift += 7) {
uint8_t byte;
if (!wasm_reader__read_u8(reader, &byte)) return false;
if (shift == 28 && (byte & 0xf0) != 0) return false;
value |= (uint32_t)(byte & 0x7f) << shift;
if ((byte & 0x80) == 0) {
*result = value;
return true;
}
}
return false;
}
static bool wasm_dylink_info__parse(
@ -213,45 +225,64 @@ static bool wasm_dylink_info__parse(
const uint8_t WASM_CUSTOM_SECTION = 0x0;
const uint8_t WASM_DYLINK_MEM_INFO = 0x1;
const uint8_t *p = bytes;
const uint8_t *end = bytes + length;
if (length < 8) return false;
if (memcmp(p, WASM_MAGIC_NUMBER, 4) != 0) return false;
p += 4;
if (memcmp(p, WASM_VERSION, 4) != 0) return false;
p += 4;
if (memcmp(bytes, WASM_MAGIC_NUMBER, 4) != 0) return false;
if (memcmp(bytes + 4, WASM_VERSION, 4) != 0) return false;
while (p < end) {
uint8_t section_id = read_u8(&p);
uint32_t section_length = read_uleb128(&p, end);
const uint8_t *section_end = p + section_length;
if (section_end > end) return false;
WasmReader reader = {
.data = bytes,
.offset = 8,
.size = length,
};
while (reader.offset < reader.size) {
uint8_t section_id;
uint32_t section_length;
if (
!wasm_reader__read_u8(&reader, &section_id) ||
!wasm_reader__read_uleb128(&reader, &section_length) ||
section_length > reader.size - reader.offset
) return false;
size_t section_end = reader.offset + section_length;
if (section_id == WASM_CUSTOM_SECTION) {
uint32_t name_length = read_uleb128(&p, section_end);
const uint8_t *name_end = p + name_length;
if (name_end > section_end) return false;
size_t previous_size = reader.size;
reader.size = section_end;
uint32_t name_length;
if (
!wasm_reader__read_uleb128(&reader, &name_length) ||
name_length > reader.size - reader.offset
) return false;
size_t name_end = reader.offset + name_length;
if (name_length == 8 && memcmp(p, "dylink.0", 8) == 0) {
p = name_end;
while (p < section_end) {
uint8_t subsection_type = read_u8(&p);
uint32_t subsection_size = read_uleb128(&p, section_end);
const uint8_t *subsection_end = p + subsection_size;
if (subsection_end > section_end) return false;
if (name_length == 8 && memcmp(&reader.data[reader.offset], "dylink.0", 8) == 0) {
reader.offset = name_end;
while (reader.offset < section_end) {
uint8_t subsection_type;
uint32_t subsection_size;
if (
!wasm_reader__read_u8(&reader, &subsection_type) ||
!wasm_reader__read_uleb128(&reader, &subsection_size) ||
subsection_size > section_end - reader.offset
) return false;
size_t subsection_end = reader.offset + subsection_size;
if (subsection_type == WASM_DYLINK_MEM_INFO) {
info->memory_size = read_uleb128(&p, subsection_end);
info->memory_align = read_uleb128(&p, subsection_end);
info->table_size = read_uleb128(&p, subsection_end);
info->table_align = read_uleb128(&p, subsection_end);
reader.size = subsection_end;
if (
!wasm_reader__read_uleb128(&reader, &info->memory_size) ||
!wasm_reader__read_uleb128(&reader, &info->memory_align) ||
!wasm_reader__read_uleb128(&reader, &info->table_size) ||
!wasm_reader__read_uleb128(&reader, &info->table_align) ||
reader.offset != subsection_end
) return false;
return true;
}
p = subsection_end;
reader.offset = subsection_end;
}
}
reader.size = previous_size;
}
p = section_end;
reader.offset = section_end;
}
return false;
}
@ -540,7 +571,8 @@ static void delete_partially_loaded_language(
}
static bool name_eq(const wasm_name_t *name, const char *string) {
return strncmp(string, name->data, name->size) == 0;
size_t length = strlen(string);
return name->size == length && memcmp(name->data, string, length) == 0;
}
static inline wasm_functype_t* wasm_functype_new_4_0(
@ -1075,6 +1107,18 @@ static uint32_t ts_wasm_store__serialization_buffer_address(TSWasmStore *self) {
return self->current_memory_offset;
}
static wasmtime_func_t ts_wasm_store__get_function(
TSWasmStore *self,
int32_t function_index
) {
wasmtime_context_t *context = wasmtime_store_context(self->store);
wasmtime_val_t value;
bool succeeded = wasmtime_table_get(context, &self->function_table, function_index, &value);
ts_assert(succeeded);
ts_assert(value.kind == WASMTIME_FUNCREF);
return value.of.funcref;
}
static bool ts_wasm_store__instantiate(
TSWasmStore *self,
wasmtime_module_t *module,
@ -1090,6 +1134,8 @@ static bool ts_wasm_store__instantiate(
char *language_function_name = NULL;
wasmtime_extern_t *imports = NULL;
wasmtime_context_t *context = wasmtime_store_context(self->store);
uint32_t initial_memory_offset = self->current_memory_offset;
uint32_t initial_function_table_offset = self->current_function_table_offset;
// Grow the function table to make room for the new functions.
wasmtime_val_t initializer = {.kind = WASMTIME_FUNCREF};
@ -1256,6 +1302,8 @@ static bool ts_wasm_store__instantiate(
return true;
error:
self->current_memory_offset = initial_memory_offset;
self->current_function_table_offset = initial_function_table_offset;
if (language_function_name) ts_free(language_function_name);
if (message.size) wasm_byte_vec_delete(&message);
if (error) wasmtime_error_delete(error);
@ -1282,6 +1330,8 @@ const TSLanguage *ts_wasm_store_load_language(
TSLanguage *language = NULL;
StringData symbol_name_buffer = array_new();
StringData field_name_buffer = array_new();
uint32_t initial_memory_offset = self->current_memory_offset;
uint32_t initial_function_table_offset = self->current_function_table_offset;
wasm_error->kind = TSWasmErrorKindNone;
if (!wasm_dylink_info__parse((const unsigned char *)wasm, wasm_len, &dylink_info)) {
@ -1508,14 +1558,13 @@ const TSLanguage *ts_wasm_store_load_language(
);
if (!valid_wasm_memory) goto invalid_language_memory;
TSSymbol last_supertype = language->supertype_symbols[language->supertype_count - 1];
TSMapSlice last_slice = language->supertype_map_slices[last_supertype];
TSMapSlice last_slice = language->supertype_map_slices[largest_supertype];
uint32_t supertype_map_entry_count = last_slice.index + last_slice.length;
language->supertype_map_entries = copy(
&wasm_memory,
wasm_language.supertype_map_entries,
supertype_map_entry_count * sizeof(char *),
supertype_map_entry_count * sizeof(TSSymbol),
&valid_wasm_memory
);
if (!valid_wasm_memory) goto invalid_language_memory;
@ -1651,13 +1700,13 @@ const TSLanguage *ts_wasm_store_load_language(
.language_id = language_id_clone(result->language_id),
.instance = instance,
.external_states_address = wasm_language.external_scanner.states,
.lex_main_fn_index = wasm_language.lex_fn,
.lex_keyword_fn_index = wasm_language.keyword_lex_fn,
.scanner_create_fn_index = wasm_language.external_scanner.create,
.scanner_destroy_fn_index = wasm_language.external_scanner.destroy,
.scanner_serialize_fn_index = wasm_language.external_scanner.serialize,
.scanner_deserialize_fn_index = wasm_language.external_scanner.deserialize,
.scanner_scan_fn_index = wasm_language.external_scanner.scan,
.lex_main_fn = ts_wasm_store__get_function(self, wasm_language.lex_fn),
.lex_keyword_fn = ts_wasm_store__get_function(self, wasm_language.keyword_lex_fn),
.scanner_create_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.create),
.scanner_destroy_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.destroy),
.scanner_serialize_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.serialize),
.scanner_deserialize_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.deserialize),
.scanner_scan_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.scan),
}));
return language;
@ -1668,6 +1717,8 @@ invalid_language_memory:
goto error;
error:
self->current_memory_offset = initial_memory_offset;
self->current_function_table_offset = initial_function_table_offset;
delete_partially_loaded_language(result, &symbol_name_buffer, &field_name_buffer);
if (module) wasmtime_module_delete(module);
return NULL;
@ -1699,6 +1750,8 @@ bool ts_wasm_store_add_language(
// If the language module has not been instantiated in this store, then add
// it to this store.
if (!exists) {
uint32_t initial_memory_offset = self->current_memory_offset;
uint32_t initial_function_table_offset = self->current_function_table_offset;
*index = self->language_instances.size;
char *message;
wasmtime_instance_t instance;
@ -1723,19 +1776,21 @@ bool ts_wasm_store_add_language(
.size = wasmtime_memory_data_size(context, &self->memory),
};
if (!wasm_memory__read(&wasm_memory, language_address, &wasm_language, sizeof(LanguageInWasmMemory))) {
self->current_memory_offset = initial_memory_offset;
self->current_function_table_offset = initial_function_table_offset;
return false;
}
array_push(&self->language_instances, ((LanguageWasmInstance) {
.language_id = language_id_clone(language_data->language_id),
.instance = instance,
.external_states_address = wasm_language.external_scanner.states,
.lex_main_fn_index = wasm_language.lex_fn,
.lex_keyword_fn_index = wasm_language.keyword_lex_fn,
.scanner_create_fn_index = wasm_language.external_scanner.create,
.scanner_destroy_fn_index = wasm_language.external_scanner.destroy,
.scanner_serialize_fn_index = wasm_language.external_scanner.serialize,
.scanner_deserialize_fn_index = wasm_language.external_scanner.deserialize,
.scanner_scan_fn_index = wasm_language.external_scanner.scan,
.lex_main_fn = ts_wasm_store__get_function(self, wasm_language.lex_fn),
.lex_keyword_fn = ts_wasm_store__get_function(self, wasm_language.keyword_lex_fn),
.scanner_create_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.create),
.scanner_destroy_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.destroy),
.scanner_serialize_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.serialize),
.scanner_deserialize_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.deserialize),
.scanner_scan_fn = ts_wasm_store__get_function(self, wasm_language.external_scanner.scan),
}));
}
@ -1774,19 +1829,13 @@ void ts_wasm_store_reset(TSWasmStore *self) {
static void ts_wasm_store__call(
TSWasmStore *self,
int32_t function_index,
wasmtime_func_t *func,
wasmtime_val_raw_t *args_and_results,
size_t args_and_results_len
) {
wasmtime_context_t *context = wasmtime_store_context(self->store);
wasmtime_val_t value;
bool succeeded = wasmtime_table_get(context, &self->function_table, function_index, &value);
ts_assert(succeeded);
ts_assert(value.kind == WASMTIME_FUNCREF);
wasmtime_func_t func = value.of.funcref;
wasm_trap_t *trap = NULL;
wasmtime_error_t *error = wasmtime_func_call_unchecked(context, &func, args_and_results, args_and_results_len, &trap);
wasmtime_error_t *error = wasmtime_func_call_unchecked(context, func, args_and_results, args_and_results_len, &trap);
if (error) {
// wasm_message_t message;
// wasmtime_error_message(error, &message);
@ -1819,7 +1868,7 @@ typedef struct {
TSSymbol result_symbol;
} TSLexerDataPrefix;
static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned function_index, TSStateId state) {
static bool ts_wasm_store__call_lex_function(TSWasmStore *self, wasmtime_func_t *func, TSStateId state) {
wasmtime_context_t *context = wasmtime_store_context(self->store);
uint8_t *memory_data = wasmtime_memory_data(context, &self->memory);
memcpy(
@ -1832,7 +1881,7 @@ static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned functio
{.i32 = self->lexer_address},
{.i32 = state},
};
ts_wasm_store__call(self, function_index, args, 2);
ts_wasm_store__call(self, func, args, 2);
if (self->has_error) return false;
bool result = args[0].i32;
@ -1847,7 +1896,7 @@ static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned functio
bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state) {
return ts_wasm_store__call_lex_function(
self,
self->current_instance->lex_main_fn_index,
&self->current_instance->lex_main_fn,
state
);
}
@ -1855,14 +1904,14 @@ bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state) {
bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state) {
return ts_wasm_store__call_lex_function(
self,
self->current_instance->lex_keyword_fn_index,
&self->current_instance->lex_keyword_fn,
state
);
}
uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self) {
wasmtime_val_raw_t args[1] = {{.i32 = 0}};
ts_wasm_store__call(self, self->current_instance->scanner_create_fn_index, args, 1);
ts_wasm_store__call(self, &self->current_instance->scanner_create_fn, args, 1);
if (self->has_error) return 0;
return args[0].i32;
}
@ -1870,7 +1919,7 @@ uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self) {
void ts_wasm_store_call_scanner_destroy(TSWasmStore *self, uint32_t scanner_address) {
if (self->current_instance) {
wasmtime_val_raw_t args[1] = {{.i32 = scanner_address}};
ts_wasm_store__call(self, self->current_instance->scanner_destroy_fn_index, args, 1);
ts_wasm_store__call(self, &self->current_instance->scanner_destroy_fn, args, 1);
}
}
@ -1896,7 +1945,7 @@ bool ts_wasm_store_call_scanner_scan(
{.i32 = self->lexer_address},
{.i32 = valid_tokens_address}
};
ts_wasm_store__call(self, self->current_instance->scanner_scan_fn_index, args, 3);
ts_wasm_store__call(self, &self->current_instance->scanner_scan_fn, args, 3);
if (self->has_error) return false;
memcpy(
@ -1920,7 +1969,7 @@ uint32_t ts_wasm_store_call_scanner_serialize(
{.i32 = scanner_address},
{.i32 = serialization_buffer_address},
};
ts_wasm_store__call(self, self->current_instance->scanner_serialize_fn_index, args, 2);
ts_wasm_store__call(self, &self->current_instance->scanner_serialize_fn, args, 2);
if (self->has_error) return 0;
uint32_t length = args[0].i32;
@ -1962,7 +2011,7 @@ void ts_wasm_store_call_scanner_deserialize(
{.i32 = serialization_buffer_address},
{.i32 = length},
};
ts_wasm_store__call(self, self->current_instance->scanner_deserialize_fn_index, args, 3);
ts_wasm_store__call(self, &self->current_instance->scanner_deserialize_fn, args, 3);
}
bool ts_wasm_store_has_error(const TSWasmStore *self) {