mirror of
https://github.com/tree-sitter/tree-sitter.git
synced 2026-09-09 23:26:20 -04:00
fix(wasm): align scanner stdlib exports
This commit is contained in:
Max Brunsfeld
parent
8933cb7b40
commit
fe5501ced9
3
.github/scripts/wasm_stdlib.js
vendored
3
.github/scripts/wasm_stdlib.js
vendored
|
|
@ -17,8 +17,7 @@ module.exports = async ({ github, context, core }) => {
|
|||
const wasmStdLibSources = [
|
||||
'lib/src/wasm-stdlib/external_scanner_allocator.c',
|
||||
'lib/src/wasm-stdlib/imports.txt',
|
||||
'lib/src/wasm-stdlib/libc.c',
|
||||
'lib/src/wasm-stdlib/stdio.c'
|
||||
'lib/src/wasm-stdlib/libc.c'
|
||||
];
|
||||
const dirChanged = changedFiles.some(file =>
|
||||
wasmStdLibSources.includes(file) ||
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@ pub fn compile_language_to_wasm(
|
|||
"__memory_base",
|
||||
"__stack_pointer",
|
||||
"__table_base",
|
||||
"__table_base",
|
||||
"memory",
|
||||
];
|
||||
let builtin_symbols = [
|
||||
|
|
|
|||
|
|
@ -28,8 +28,6 @@ TS_WASM_EXPORT("memmove") void *memmove(void *dst, const void *src, size_t count
|
|||
|
||||
TS_WASM_EXPORT("memset") void *memset(void *dst, int value, size_t count);
|
||||
|
||||
TS_WASM_EXPORT("strchr") char *strchr(const char *str, int c);
|
||||
|
||||
TS_WASM_EXPORT("strcmp") int strcmp(const char *left, const char *right);
|
||||
|
||||
TS_WASM_EXPORT("strlen") size_t strlen(const char *str);
|
||||
|
|
@ -40,8 +38,6 @@ TS_WASM_EXPORT("strncmp") int strncmp(const char *left, const char *right, size_
|
|||
|
||||
char *__stpncpy(char *restrict dest, const char *restrict src, size_t count);
|
||||
|
||||
char *__strchrnul(const char *str, int c);
|
||||
|
||||
char *stpncpy(char *restrict dest, const char *restrict src, size_t count);
|
||||
|
||||
TS_WASM_EXPORT("strncpy") char *strncpy(char *restrict dest, const char *restrict src, size_t count);
|
||||
|
|
|
|||
|
|
@ -76,8 +76,6 @@ const WASI_LIBC_FILES: &[&str] = &[
|
|||
"string/memcpy.c",
|
||||
"string/memmove.c",
|
||||
"string/memset.c",
|
||||
"string/strchr.c",
|
||||
"string/strchrnul.c",
|
||||
"string/strcmp.c",
|
||||
"string/strlen.c",
|
||||
"string/stpncpy.c",
|
||||
|
|
@ -734,7 +732,6 @@ pub fn run_wasm_stdlib() -> Result<()> {
|
|||
.args(&export_flags)
|
||||
.arg("-Icrates/language/wasm/include")
|
||||
.arg("lib/src/wasm-stdlib/libc.c")
|
||||
.arg("lib/src/wasm-stdlib/stdio.c")
|
||||
.arg("lib/src/wasm-stdlib/external_scanner_allocator.c")
|
||||
.output()?;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,18 +9,16 @@ subset:
|
|||
|
||||
- `libc/` contains sources vendored from the `wasi-libc` revision used by the
|
||||
repository's pinned WASI SDK.
|
||||
- `stdio.c` is Tree-sitter's scanner-oriented stdio implementation. It provides
|
||||
in-memory formatting and intentionally implements stream operations as
|
||||
no-ops.
|
||||
- `external_scanner_allocator.c` is the resettable allocator used for isolated
|
||||
Wasm language modules.
|
||||
- `external_scanner_stdlib.h` is a generated Wasm module containing the
|
||||
vendored libc subset, `stdio.c`, and the resettable allocator.
|
||||
libc functions listed in `imports.txt` and the resettable allocator.
|
||||
|
||||
When the Tree-sitter Rust library is compiled for `wasm32-unknown-unknown`, the
|
||||
same vendored libc sources and `stdio.c` are linked directly into the
|
||||
application. In that environment, allocation is instead provided by Rust's
|
||||
application-selected global allocator.
|
||||
same vendored libc sources are linked directly into the application. `stdio.c`
|
||||
provides the additional internal functions needed by the Tree-sitter core;
|
||||
these functions are not part of the external-scanner API. Allocation is
|
||||
provided by Rust's application-selected global allocator.
|
||||
|
||||
To refresh the vendored sources and regenerate the embedded module, run:
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ unsigned char STDLIB_WASM[] = {
|
|||
0x02, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x00, 0x00, 0x60, 0x01, 0x7f, 0x00,
|
||||
0x02, 0x0f, 0x01, 0x03, 0x65, 0x6e, 0x76, 0x06, 0x6d, 0x65, 0x6d, 0x6f,
|
||||
0x72, 0x79, 0x02, 0x00, 0x02, 0x03, 0x1e, 0x1d, 0x03, 0x03, 0x01, 0x01,
|
||||
0x01, 0x01, 0x01, 0x00, 0x02, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00,
|
||||
0x01, 0x01, 0x01, 0x02, 0x00, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x02,
|
||||
0x02, 0x06, 0x08, 0x01, 0x7f, 0x01, 0x41, 0x80, 0x80, 0x04, 0x0b, 0x07,
|
||||
0xaf, 0x02, 0x1c, 0x11, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x6d, 0x5f, 0x63,
|
||||
|
|
@ -14,8 +14,8 @@ unsigned char STDLIB_WASM[] = {
|
|||
0x00, 0x02, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x6d, 0x70, 0x00, 0x03, 0x06,
|
||||
0x6d, 0x65, 0x6d, 0x63, 0x70, 0x79, 0x00, 0x04, 0x07, 0x6d, 0x65, 0x6d,
|
||||
0x6d, 0x6f, 0x76, 0x65, 0x00, 0x05, 0x06, 0x6d, 0x65, 0x6d, 0x73, 0x65,
|
||||
0x74, 0x00, 0x06, 0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, 0x00, 0x07,
|
||||
0x06, 0x73, 0x74, 0x72, 0x63, 0x6d, 0x70, 0x00, 0x08, 0x07, 0x73, 0x74,
|
||||
0x74, 0x00, 0x06, 0x06, 0x73, 0x74, 0x72, 0x63, 0x6d, 0x70, 0x00, 0x07,
|
||||
0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, 0x00, 0x08, 0x07, 0x73, 0x74,
|
||||
0x72, 0x6e, 0x63, 0x61, 0x74, 0x00, 0x09, 0x07, 0x73, 0x74, 0x72, 0x6e,
|
||||
0x63, 0x6d, 0x70, 0x00, 0x0a, 0x07, 0x73, 0x74, 0x72, 0x6e, 0x63, 0x70,
|
||||
0x79, 0x00, 0x0b, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x6e, 0x75, 0x6d,
|
||||
|
|
@ -200,30 +200,30 @@ unsigned char STDLIB_WASM[] = {
|
|||
0x03, 0x00, 0x20, 0x02, 0x41, 0x10, 0x6a, 0x20, 0x06, 0x37, 0x03, 0x00,
|
||||
0x20, 0x02, 0x41, 0x08, 0x6a, 0x20, 0x06, 0x37, 0x03, 0x00, 0x20, 0x02,
|
||||
0x41, 0x20, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x20, 0x6b, 0x22, 0x01,
|
||||
0x41, 0x1f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0b, 0xaa, 0x01,
|
||||
0x01, 0x03, 0x7f, 0x20, 0x00, 0x21, 0x01, 0x02, 0x40, 0x02, 0x40, 0x20,
|
||||
0x00, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x00,
|
||||
0x45, 0x04, 0x40, 0x20, 0x00, 0x21, 0x02, 0x0c, 0x02, 0x0b, 0x20, 0x00,
|
||||
0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00, 0x21, 0x02, 0x03, 0x40, 0x20,
|
||||
0x02, 0x41, 0x01, 0x6a, 0x22, 0x02, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x01,
|
||||
0x20, 0x01, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01,
|
||||
0x0d, 0x00, 0x0b, 0x0c, 0x01, 0x0b, 0x20, 0x01, 0x41, 0x04, 0x6b, 0x21,
|
||||
0x03, 0x20, 0x01, 0x41, 0x05, 0x6b, 0x21, 0x02, 0x03, 0x40, 0x20, 0x02,
|
||||
0x41, 0x04, 0x6a, 0x21, 0x02, 0x41, 0x80, 0x82, 0x84, 0x08, 0x20, 0x03,
|
||||
0x41, 0x04, 0x6a, 0x22, 0x03, 0x28, 0x02, 0x00, 0x22, 0x01, 0x6b, 0x20,
|
||||
0x01, 0x72, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x41, 0x80, 0x81,
|
||||
0x82, 0x84, 0x78, 0x46, 0x0d, 0x00, 0x0b, 0x03, 0x40, 0x20, 0x02, 0x41,
|
||||
0x01, 0x6a, 0x21, 0x02, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x20, 0x03, 0x41,
|
||||
0x01, 0x6a, 0x21, 0x03, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x20, 0x00,
|
||||
0x6b, 0x0b, 0x58, 0x01, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x2d, 0x00,
|
||||
0x00, 0x22, 0x02, 0x45, 0x20, 0x02, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22,
|
||||
0x03, 0x47, 0x72, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00,
|
||||
0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x2d,
|
||||
0x00, 0x00, 0x21, 0x03, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45,
|
||||
0x0d, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41,
|
||||
0x01, 0x6a, 0x21, 0x01, 0x20, 0x02, 0x20, 0x03, 0x46, 0x0d, 0x00, 0x0b,
|
||||
0x0b, 0x20, 0x02, 0x20, 0x03, 0x6b, 0x0b, 0x49, 0x01, 0x02, 0x7f, 0x20,
|
||||
0x00, 0x10, 0x07, 0x20, 0x00, 0x6a, 0x21, 0x03, 0x02, 0x40, 0x20, 0x02,
|
||||
0x41, 0x1f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0b, 0x58, 0x01,
|
||||
0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45,
|
||||
0x20, 0x02, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x03, 0x47, 0x72, 0x0d,
|
||||
0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41, 0x01,
|
||||
0x6a, 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x21, 0x03,
|
||||
0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45, 0x0d, 0x01, 0x20, 0x00,
|
||||
0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01,
|
||||
0x20, 0x02, 0x20, 0x03, 0x46, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x20,
|
||||
0x03, 0x6b, 0x0b, 0xaa, 0x01, 0x01, 0x03, 0x7f, 0x20, 0x00, 0x21, 0x01,
|
||||
0x02, 0x40, 0x02, 0x40, 0x20, 0x00, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00,
|
||||
0x20, 0x00, 0x2d, 0x00, 0x00, 0x45, 0x04, 0x40, 0x20, 0x00, 0x21, 0x02,
|
||||
0x0c, 0x02, 0x0b, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00,
|
||||
0x21, 0x02, 0x03, 0x40, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x22, 0x02, 0x41,
|
||||
0x03, 0x71, 0x45, 0x0d, 0x01, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x20, 0x01,
|
||||
0x41, 0x01, 0x6a, 0x21, 0x01, 0x0d, 0x00, 0x0b, 0x0c, 0x01, 0x0b, 0x20,
|
||||
0x01, 0x41, 0x04, 0x6b, 0x21, 0x03, 0x20, 0x01, 0x41, 0x05, 0x6b, 0x21,
|
||||
0x02, 0x03, 0x40, 0x20, 0x02, 0x41, 0x04, 0x6a, 0x21, 0x02, 0x41, 0x80,
|
||||
0x82, 0x84, 0x08, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x22, 0x03, 0x28, 0x02,
|
||||
0x00, 0x22, 0x01, 0x6b, 0x20, 0x01, 0x72, 0x41, 0x80, 0x81, 0x82, 0x84,
|
||||
0x78, 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x46, 0x0d, 0x00, 0x0b,
|
||||
0x03, 0x40, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x21, 0x02, 0x20, 0x03, 0x2d,
|
||||
0x00, 0x00, 0x20, 0x03, 0x41, 0x01, 0x6a, 0x21, 0x03, 0x0d, 0x00, 0x0b,
|
||||
0x0b, 0x20, 0x02, 0x20, 0x00, 0x6b, 0x0b, 0x49, 0x01, 0x02, 0x7f, 0x20,
|
||||
0x00, 0x10, 0x08, 0x20, 0x00, 0x6a, 0x21, 0x03, 0x02, 0x40, 0x20, 0x02,
|
||||
0x45, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x04,
|
||||
0x45, 0x0d, 0x01, 0x20, 0x03, 0x20, 0x04, 0x3a, 0x00, 0x00, 0x20, 0x03,
|
||||
0x41, 0x01, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01,
|
||||
|
|
|
|||
|
|
@ -18,13 +18,6 @@
|
|||
#undef WS
|
||||
|
||||
#include "./libc/string/memset.c"
|
||||
#include "./libc/string/strchrnul.c"
|
||||
#undef ALIGN
|
||||
#undef HASZERO
|
||||
#undef HIGHS
|
||||
#undef ONES
|
||||
|
||||
#include "./libc/string/strchr.c"
|
||||
#include "./libc/string/strcmp.c"
|
||||
#include "./libc/string/strlen.c"
|
||||
#undef ALIGN
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
#include <string.h>
|
||||
|
||||
char *strchr(const char *s, int c)
|
||||
{
|
||||
char *r = __strchrnul(s, c);
|
||||
return *(unsigned char *)r == (unsigned char)c ? r : 0;
|
||||
}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <limits.h>
|
||||
|
||||
#ifdef __wasm_simd128__
|
||||
#include <wasm_simd128.h>
|
||||
#endif
|
||||
|
||||
#define ALIGN (sizeof(size_t))
|
||||
#define ONES ((size_t)-1/UCHAR_MAX)
|
||||
#define HIGHS (ONES * (UCHAR_MAX/2+1))
|
||||
#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
|
||||
|
||||
char *__strchrnul(const char *s, int c)
|
||||
{
|
||||
c = (unsigned char)c;
|
||||
if (!c) return (char *)s + strlen(s);
|
||||
|
||||
#if defined(__wasm_simd128__) && defined(__wasilibc_simd_string)
|
||||
// Skip Clang 19 and Clang 20 which have a bug (llvm/llvm-project#146574)
|
||||
// which results in an ICE when inline assembly is used with a vector result.
|
||||
#if __clang_major__ != 19 && __clang_major__ != 20
|
||||
// Note that reading before/after the allocation of a pointer is UB in
|
||||
// C, so inline assembly is used to generate the exact machine
|
||||
// instruction we want with opaque semantics to the compiler to avoid
|
||||
// the UB.
|
||||
uintptr_t align = (uintptr_t)s % sizeof(v128_t);
|
||||
uintptr_t addr = (uintptr_t)s - align;
|
||||
v128_t vc = wasm_i8x16_splat(c);
|
||||
|
||||
for (;;) {
|
||||
v128_t v;
|
||||
__asm__ (
|
||||
"local.get %1\n"
|
||||
"v128.load 0\n"
|
||||
"local.set %0\n"
|
||||
: "=r"(v)
|
||||
: "r"(addr)
|
||||
: "memory");
|
||||
const v128_t cmp = wasm_i8x16_eq(v, (v128_t){}) | wasm_i8x16_eq(v, vc);
|
||||
// Bitmask is slow on AArch64, any_true is much faster.
|
||||
if (wasm_v128_any_true(cmp)) {
|
||||
// Clear the bits corresponding to align (little-endian)
|
||||
// so we can count trailing zeros.
|
||||
int mask = wasm_i8x16_bitmask(cmp) >> align << align;
|
||||
// At least one bit will be set, unless align cleared them.
|
||||
// Knowing this helps the compiler if it unrolls the loop.
|
||||
__builtin_assume(mask || align);
|
||||
// If the mask became zero because of align,
|
||||
// it's as if we didn't find anything.
|
||||
if (mask) {
|
||||
// Find the offset of the first one bit (little-endian).
|
||||
return (char *)s + (addr - (uintptr_t)s + __builtin_ctz(mask));
|
||||
}
|
||||
}
|
||||
align = 0;
|
||||
addr += sizeof(v128_t);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef __GNUC__
|
||||
typedef size_t __attribute__((__may_alias__)) word;
|
||||
const word *w;
|
||||
for (; (uintptr_t)s % ALIGN; s++)
|
||||
if (!*s || *(unsigned char *)s == c) return (char *)s;
|
||||
size_t k = ONES * c;
|
||||
for (w = (void *)s; !HASZERO(*w) && !HASZERO(*w^k); w++);
|
||||
s = (void *)w;
|
||||
#endif
|
||||
for (; *s && *(unsigned char *)s != c; s++);
|
||||
return (char *)s;
|
||||
}
|
||||
|
||||
weak_alias(__strchrnul, strchrnul);
|
||||
Loading…
Reference in a new issue