fix(wasm): align scanner stdlib exports

.github/scripts/wasm_stdlib.js

@@ -17,8 +17,7 @@ module.exports = async ({ github, context, core }) => {
   const wasmStdLibSources = [
     'lib/src/wasm-stdlib/external_scanner_allocator.c',
     'lib/src/wasm-stdlib/imports.txt',
-    'lib/src/wasm-stdlib/libc.c',
+    'lib/src/wasm-stdlib/libc.c'
-    'lib/src/wasm-stdlib/stdio.c'
   ];
   const dirChanged = changedFiles.some(file =>
     wasmStdLibSources.includes(file) ||

crates/cli/src/wasm.rs

@@ -67,7 +67,6 @@ pub fn compile_language_to_wasm(
         "__memory_base",
         "__stack_pointer",
         "__table_base",
-        "__table_base",
         "memory",
     ];
     let builtin_symbols = [

crates/language/wasm/include/string.h

@@ -28,8 +28,6 @@ TS_WASM_EXPORT("memmove") void *memmove(void *dst, const void *src, size_t count
 
 TS_WASM_EXPORT("memset") void *memset(void *dst, int value, size_t count);
 
-TS_WASM_EXPORT("strchr") char *strchr(const char *str, int c);
-
 TS_WASM_EXPORT("strcmp") int strcmp(const char *left, const char *right);
 
 TS_WASM_EXPORT("strlen") size_t strlen(const char *str);
@@ -40,8 +38,6 @@ TS_WASM_EXPORT("strncmp") int strncmp(const char *left, const char *right, size_
 
 char *__stpncpy(char *restrict dest, const char *restrict src, size_t count);
 
-char *__strchrnul(const char *str, int c);
-
 char *stpncpy(char *restrict dest, const char *restrict src, size_t count);
 
 TS_WASM_EXPORT("strncpy") char *strncpy(char *restrict dest, const char *restrict src, size_t count);

crates/xtask/src/build_wasm.rs

@@ -76,8 +76,6 @@ const WASI_LIBC_FILES: &[&str] = &[
     "string/memcpy.c",
     "string/memmove.c",
     "string/memset.c",
-    "string/strchr.c",
-    "string/strchrnul.c",
     "string/strcmp.c",
     "string/strlen.c",
     "string/stpncpy.c",
@@ -734,7 +732,6 @@ pub fn run_wasm_stdlib() -> Result<()> {
         .args(&export_flags)
         .arg("-Icrates/language/wasm/include")
         .arg("lib/src/wasm-stdlib/libc.c")
-        .arg("lib/src/wasm-stdlib/stdio.c")
         .arg("lib/src/wasm-stdlib/external_scanner_allocator.c")
         .output()?;
 

lib/src/wasm-stdlib/README.md

@@ -9,18 +9,16 @@ subset:
 
 - `libc/` contains sources vendored from the `wasi-libc` revision used by the
   repository's pinned WASI SDK.
-- `stdio.c` is Tree-sitter's scanner-oriented stdio implementation. It provides
-  in-memory formatting and intentionally implements stream operations as
-  no-ops.
 - `external_scanner_allocator.c` is the resettable allocator used for isolated
   Wasm language modules.
 - `external_scanner_stdlib.h` is a generated Wasm module containing the
-  vendored libc subset, `stdio.c`, and the resettable allocator.
+  libc functions listed in `imports.txt` and the resettable allocator.
 
 When the Tree-sitter Rust library is compiled for `wasm32-unknown-unknown`, the
-same vendored libc sources and `stdio.c` are linked directly into the
+same vendored libc sources are linked directly into the application. `stdio.c`
-application. In that environment, allocation is instead provided by Rust's
+provides the additional internal functions needed by the Tree-sitter core;
-application-selected global allocator.
+these functions are not part of the external-scanner API. Allocation is
+provided by Rust's application-selected global allocator.
 
 To refresh the vendored sources and regenerate the embedded module, run:
 

lib/src/wasm-stdlib/external_scanner_stdlib.h

@@ -4,7 +4,7 @@ unsigned char STDLIB_WASM[] = {
   0x02, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x00, 0x00, 0x60, 0x01, 0x7f, 0x00,
   0x02, 0x0f, 0x01, 0x03, 0x65, 0x6e, 0x76, 0x06, 0x6d, 0x65, 0x6d, 0x6f,
   0x72, 0x79, 0x02, 0x00, 0x02, 0x03, 0x1e, 0x1d, 0x03, 0x03, 0x01, 0x01,
-  0x01, 0x01, 0x01, 0x00, 0x02, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00,
+  0x01, 0x01, 0x01, 0x02, 0x00, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00,
   0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x02,
   0x02, 0x06, 0x08, 0x01, 0x7f, 0x01, 0x41, 0x80, 0x80, 0x04, 0x0b, 0x07,
   0xaf, 0x02, 0x1c, 0x11, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x6d, 0x5f, 0x63,
@@ -14,8 +14,8 @@ unsigned char STDLIB_WASM[] = {
   0x00, 0x02, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x6d, 0x70, 0x00, 0x03, 0x06,
   0x6d, 0x65, 0x6d, 0x63, 0x70, 0x79, 0x00, 0x04, 0x07, 0x6d, 0x65, 0x6d,
   0x6d, 0x6f, 0x76, 0x65, 0x00, 0x05, 0x06, 0x6d, 0x65, 0x6d, 0x73, 0x65,
-  0x74, 0x00, 0x06, 0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, 0x00, 0x07,
+  0x74, 0x00, 0x06, 0x06, 0x73, 0x74, 0x72, 0x63, 0x6d, 0x70, 0x00, 0x07,
-  0x06, 0x73, 0x74, 0x72, 0x63, 0x6d, 0x70, 0x00, 0x08, 0x07, 0x73, 0x74,
+  0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, 0x00, 0x08, 0x07, 0x73, 0x74,
   0x72, 0x6e, 0x63, 0x61, 0x74, 0x00, 0x09, 0x07, 0x73, 0x74, 0x72, 0x6e,
   0x63, 0x6d, 0x70, 0x00, 0x0a, 0x07, 0x73, 0x74, 0x72, 0x6e, 0x63, 0x70,
   0x79, 0x00, 0x0b, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x6e, 0x75, 0x6d,
@@ -200,30 +200,30 @@ unsigned char STDLIB_WASM[] = {
   0x03, 0x00, 0x20, 0x02, 0x41, 0x10, 0x6a, 0x20, 0x06, 0x37, 0x03, 0x00,
   0x20, 0x02, 0x41, 0x08, 0x6a, 0x20, 0x06, 0x37, 0x03, 0x00, 0x20, 0x02,
   0x41, 0x20, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x20, 0x6b, 0x22, 0x01,
-  0x41, 0x1f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0b, 0xaa, 0x01,
+  0x41, 0x1f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0b, 0x58, 0x01,
-  0x01, 0x03, 0x7f, 0x20, 0x00, 0x21, 0x01, 0x02, 0x40, 0x02, 0x40, 0x20,
+  0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45,
-  0x00, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x00,
+  0x20, 0x02, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x03, 0x47, 0x72, 0x0d,
-  0x45, 0x04, 0x40, 0x20, 0x00, 0x21, 0x02, 0x0c, 0x02, 0x0b, 0x20, 0x00,
+  0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41, 0x01,
-  0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00, 0x21, 0x02, 0x03, 0x40, 0x20,
+  0x6a, 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x21, 0x03,
-  0x02, 0x41, 0x01, 0x6a, 0x22, 0x02, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x01,
+  0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45, 0x0d, 0x01, 0x20, 0x00,
-  0x20, 0x01, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01,
+  0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01,
-  0x0d, 0x00, 0x0b, 0x0c, 0x01, 0x0b, 0x20, 0x01, 0x41, 0x04, 0x6b, 0x21,
+  0x20, 0x02, 0x20, 0x03, 0x46, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x20,
-  0x03, 0x20, 0x01, 0x41, 0x05, 0x6b, 0x21, 0x02, 0x03, 0x40, 0x20, 0x02,
+  0x03, 0x6b, 0x0b, 0xaa, 0x01, 0x01, 0x03, 0x7f, 0x20, 0x00, 0x21, 0x01,
-  0x41, 0x04, 0x6a, 0x21, 0x02, 0x41, 0x80, 0x82, 0x84, 0x08, 0x20, 0x03,
+  0x02, 0x40, 0x02, 0x40, 0x20, 0x00, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00,
-  0x41, 0x04, 0x6a, 0x22, 0x03, 0x28, 0x02, 0x00, 0x22, 0x01, 0x6b, 0x20,
+  0x20, 0x00, 0x2d, 0x00, 0x00, 0x45, 0x04, 0x40, 0x20, 0x00, 0x21, 0x02,
-  0x01, 0x72, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x41, 0x80, 0x81,
+  0x0c, 0x02, 0x0b, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00,
-  0x82, 0x84, 0x78, 0x46, 0x0d, 0x00, 0x0b, 0x03, 0x40, 0x20, 0x02, 0x41,
+  0x21, 0x02, 0x03, 0x40, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x22, 0x02, 0x41,
-  0x01, 0x6a, 0x21, 0x02, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x20, 0x03, 0x41,
+  0x03, 0x71, 0x45, 0x0d, 0x01, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x20, 0x01,
-  0x01, 0x6a, 0x21, 0x03, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x20, 0x00,
+  0x41, 0x01, 0x6a, 0x21, 0x01, 0x0d, 0x00, 0x0b, 0x0c, 0x01, 0x0b, 0x20,
-  0x6b, 0x0b, 0x58, 0x01, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x2d, 0x00,
+  0x01, 0x41, 0x04, 0x6b, 0x21, 0x03, 0x20, 0x01, 0x41, 0x05, 0x6b, 0x21,
-  0x00, 0x22, 0x02, 0x45, 0x20, 0x02, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22,
+  0x02, 0x03, 0x40, 0x20, 0x02, 0x41, 0x04, 0x6a, 0x21, 0x02, 0x41, 0x80,
-  0x03, 0x47, 0x72, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00,
+  0x82, 0x84, 0x08, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x22, 0x03, 0x28, 0x02,
-  0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x2d,
+  0x00, 0x22, 0x01, 0x6b, 0x20, 0x01, 0x72, 0x41, 0x80, 0x81, 0x82, 0x84,
-  0x00, 0x00, 0x21, 0x03, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45,
+  0x78, 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x46, 0x0d, 0x00, 0x0b,
-  0x0d, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41,
+  0x03, 0x40, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x21, 0x02, 0x20, 0x03, 0x2d,
-  0x01, 0x6a, 0x21, 0x01, 0x20, 0x02, 0x20, 0x03, 0x46, 0x0d, 0x00, 0x0b,
+  0x00, 0x00, 0x20, 0x03, 0x41, 0x01, 0x6a, 0x21, 0x03, 0x0d, 0x00, 0x0b,
-  0x0b, 0x20, 0x02, 0x20, 0x03, 0x6b, 0x0b, 0x49, 0x01, 0x02, 0x7f, 0x20,
+  0x0b, 0x20, 0x02, 0x20, 0x00, 0x6b, 0x0b, 0x49, 0x01, 0x02, 0x7f, 0x20,
-  0x00, 0x10, 0x07, 0x20, 0x00, 0x6a, 0x21, 0x03, 0x02, 0x40, 0x20, 0x02,
+  0x00, 0x10, 0x08, 0x20, 0x00, 0x6a, 0x21, 0x03, 0x02, 0x40, 0x20, 0x02,
   0x45, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x04,
   0x45, 0x0d, 0x01, 0x20, 0x03, 0x20, 0x04, 0x3a, 0x00, 0x00, 0x20, 0x03,
   0x41, 0x01, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01,

lib/src/wasm-stdlib/libc.c

@@ -18,13 +18,6 @@
 #undef WS
 
 #include "./libc/string/memset.c"
-#include "./libc/string/strchrnul.c"
-#undef ALIGN
-#undef HASZERO
-#undef HIGHS
-#undef ONES
-
-#include "./libc/string/strchr.c"
 #include "./libc/string/strcmp.c"
 #include "./libc/string/strlen.c"
 #undef ALIGN

lib/src/wasm-stdlib/libc/string/strchr.c

@@ -1,7 +0,0 @@
-#include <string.h>
-
-char *strchr(const char *s, int c)
-{
-	char *r = __strchrnul(s, c);
-	return *(unsigned char *)r == (unsigned char)c ? r : 0;
-}

lib/src/wasm-stdlib/libc/string/strchrnul.c

@@ -1,75 +0,0 @@
-#include <string.h>
-#include <stdint.h>
-#include <limits.h>
-
-#ifdef __wasm_simd128__
-#include <wasm_simd128.h>
-#endif
-
-#define ALIGN (sizeof(size_t))
-#define ONES ((size_t)-1/UCHAR_MAX)
-#define HIGHS (ONES * (UCHAR_MAX/2+1))
-#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
-
-char *__strchrnul(const char *s, int c)
-{
-	c = (unsigned char)c;
-	if (!c) return (char *)s + strlen(s);
-
-#if defined(__wasm_simd128__) && defined(__wasilibc_simd_string)
-	// Skip Clang 19 and Clang 20 which have a bug (llvm/llvm-project#146574)
-	// which results in an ICE when inline assembly is used with a vector result.
-#if __clang_major__ != 19 && __clang_major__ != 20
-	// Note that reading before/after the allocation of a pointer is UB in
-	// C, so inline assembly is used to generate the exact machine
-	// instruction we want with opaque semantics to the compiler to avoid
-	// the UB.
-	uintptr_t align = (uintptr_t)s % sizeof(v128_t);
-	uintptr_t addr = (uintptr_t)s - align;
-	v128_t vc = wasm_i8x16_splat(c);
-
-	for (;;) {
-		v128_t v;
-		__asm__ (
-			"local.get %1\n"
-			"v128.load 0\n"
-			"local.set %0\n"
-			: "=r"(v)
-			: "r"(addr)
-			: "memory");
-		const v128_t cmp = wasm_i8x16_eq(v, (v128_t){}) | wasm_i8x16_eq(v, vc);
-		// Bitmask is slow on AArch64, any_true is much faster.
-		if (wasm_v128_any_true(cmp)) {
-			// Clear the bits corresponding to align (little-endian)
-			// so we can count trailing zeros.
-			int mask = wasm_i8x16_bitmask(cmp) >> align << align;
-			// At least one bit will be set, unless align cleared them.
-			// Knowing this helps the compiler if it unrolls the loop.
-			__builtin_assume(mask || align);
-			// If the mask became zero because of align,
-			// it's as if we didn't find anything.
-			if (mask) {
-				// Find the offset of the first one bit (little-endian).
-				return (char *)s + (addr - (uintptr_t)s + __builtin_ctz(mask));
-			}
-		}
-		align = 0;
-		addr += sizeof(v128_t);
-	}
-#endif
-#endif
-
-#ifdef __GNUC__
-	typedef size_t __attribute__((__may_alias__)) word;
-	const word *w;
-	for (; (uintptr_t)s % ALIGN; s++)
-		if (!*s || *(unsigned char *)s == c) return (char *)s;
-	size_t k = ONES * c;
-	for (w = (void *)s; !HASZERO(*w) && !HASZERO(*w^k); w++);
-	s = (void *)w;
-#endif
-	for (; *s && *(unsigned char *)s != c; s++);
-	return (char *)s;
-}
-
-weak_alias(__strchrnul, strchrnul);