Allow using Tree-sitter rust lib in multi-threaded web apps compiled to wasm32-unknown-unknown (#5851)

* feat(wasm): make syntax trees sendable

* test(wasm): transfer trees across workers

* test(wasm): use JSON grammar for tree transfer

* test(wasm): edit trees across workers

* test(wasm): share dlmalloc with tree-sitter

* test(wasm): simplify worker tree exchange

* test(wasm): drive tree exchange from Rust

* test(wasm): split sendable-tree xtask

* test(wasm): generalize Rust web fixture

* test(wasm): exercise parallel Rust tree access

* fix(wasm): use Rust global allocator for C core

* test(wasm): use default Rust allocator

* feat(wasm): support external scanners in Rust web apps

* Simplify example further, add a readme

* Regenerate wasm-stdlib

* Fix wasm_stdlib check script

* Vendor the Wasm standard library subset

* Test Unicode Ruby scanner behavior in Wasm

* Make Wasm tree languages instance-aware

* Test multi-threaded use of queries in wasm32-unknown

* Refactor reference-counted language storage

* Check ABI version compat before loading rest of language

* 🎨 Remove redundant #ifdef block

* Reject unsupported Rust Wasm builds on 0.26
This commit is contained in:
Max Brunsfeld 2026-08-14 17:05:42 -07:00 committed by GitHub
parent 21e3614b8d
commit 0e2af0d8d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
84 changed files with 4598 additions and 1678 deletions

View file

@ -21,7 +21,7 @@ runs:
'lib/src/parser.h',
'lib/src/array.h',
'lib/src/alloc.h',
'lib/src/wasm/wasm-stdlib.h',
'lib/src/wasm-stdlib/external_scanner_stdlib.h',
'crates/loader/wasi-sdk-version',
'crates/loader/binaryen-version',
'test/fixtures/grammars/*/**/src/*.c',

View file

@ -5,21 +5,31 @@ module.exports = async ({ github, context, core }) => {
const owner = context.repo.owner;
const repo = context.repo.repo;
const { data: files } = await github.rest.pulls.listFiles({
const files = await github.paginate(github.rest.pulls.listFiles, {
owner,
repo,
pull_number: prNumber
pull_number: prNumber,
per_page: 100
});
const changedFiles = files.map(file => file.filename);
const wasmStdLibSrc = 'crates/language/wasm/';
const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc));
const wasmStdLibSources = [
'lib/src/wasm-stdlib/external_scanner_allocator.c',
'lib/src/wasm-stdlib/imports.txt',
'lib/src/wasm-stdlib/libc.c',
'lib/src/wasm-stdlib/stdio.c'
];
const dirChanged = changedFiles.some(file =>
wasmStdLibSources.includes(file) ||
file.startsWith('lib/src/wasm-stdlib/libc/ctype/') ||
file.startsWith('lib/src/wasm-stdlib/libc/string/')
);
if (!dirChanged) return;
const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h';
const wasmStdLibHeader = 'lib/src/wasm-stdlib/external_scanner_stdlib.h';
const requiredChanged = changedFiles.includes(wasmStdLibHeader);
if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`);
if (!requiredChanged) core.setFailed(`Changes detected in the Wasm stdlib sources but ${wasmStdLibHeader} was not modified.`);
};

View file

@ -128,6 +128,10 @@ jobs:
with:
target: ${{ matrix.target }}
- name: Install Rust Wasm test target
if: matrix.run-wasm-test
run: rustup toolchain install nightly --profile minimal --component rust-src
- name: Install cross-compilation toolchain
if: matrix.cross
run: |
@ -295,6 +299,10 @@ jobs:
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
- name: Run Rust Wasm web test
if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && matrix.run-wasm-test
run: cargo run -p xtask --target='${{ matrix.target }}' -- test-rust-wasm-web
- name: Upload CLI artifact
if: "!inputs.run-test && !matrix.no-run"
uses: actions/upload-artifact@v7

View file

@ -134,4 +134,4 @@ tree-sitter-highlight = { path = "./crates/highlight", version = "0.27.0" }
tree-sitter-loader = { path = "./crates/loader", version = "0.27.0" }
tree-sitter-tags = { path = "./crates/tags", version = "0.27.0" }
tree-sitter-language = { path = "./crates/language", version = "0.1" }
tree-sitter-language = { path = "./crates/language", version = "0.1.8" }

View file

@ -976,18 +976,8 @@ fn update_rust_build_rs(path: &Path, language_name: &str, opts: &GenerateOpts) -
let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
};
let Ok(wasm_src) =
std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
};
c_config.include(&wasm_headers);
c_config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
]);
}
"#}
.lines()

View file

@ -11,18 +11,8 @@ fn main() {
let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
};
let Ok(wasm_src) =
std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
};
c_config.include(&wasm_headers);
c_config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
]);
}
let parser_path = src_dir.join("parser.c");

View file

@ -2,3 +2,8 @@
This crate provides a `LanguageFn` type for grammars to create `Language` instances from a parser,
without having to worry about the `tree-sitter` crate version not matching.
When targeting `wasm32-unknown-unknown`, this crate also provides the C headers
needed to compile generated parsers and external scanners. The final
`tree-sitter` Rust application supplies the corresponding libc-compatible
implementations.

View file

@ -5,7 +5,8 @@ fn main() {
{
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
let wasm_headers = std::path::Path::new(&manifest_dir).join("wasm/include");
let wasm_src = std::path::Path::new(&manifest_dir).join("wasm/src");
let wasm_src =
std::path::Path::new(&manifest_dir).join("wasm/unsupported/tree-sitter-0.26");
println!("cargo::metadata=wasm-headers={}", wasm_headers.display());
println!("cargo::metadata=wasm-src={}", wasm_src.display());

View file

@ -1,6 +1,15 @@
#ifndef TREE_SITTER_WASM_CTYPE_H_
#define TREE_SITTER_WASM_CTYPE_H_
typedef void *locale_t;
#ifndef weak_alias
#define weak_alias(old, new) \
extern __typeof(old) new __attribute__((__weak__, __alias__(#old)))
#endif
int isblank(int c);
static inline int isprint(int c) {
return c >= 0x20 && c <= 0x7E;
}

View file

@ -1,6 +1,16 @@
#ifndef TREE_SITTER_WASM_ENDIAN_H_
#define TREE_SITTER_WASM_ENDIAN_H_
#ifndef __LITTLE_ENDIAN
#define __LITTLE_ENDIAN 1234
#endif
#ifndef __BIG_ENDIAN
#define __BIG_ENDIAN 4321
#endif
#ifndef __BYTE_ORDER
#define __BYTE_ORDER __LITTLE_ENDIAN
#endif
#define be16toh(x) __builtin_bswap16(x)
#define be32toh(x) __builtin_bswap32(x)
#define be64toh(x) __builtin_bswap64(x)
@ -8,5 +18,4 @@
#define le32toh(x) (x)
#define le64toh(x) (x)
#endif // TREE_SITTER_WASM_ENDIAN_H_

View file

@ -5,11 +5,17 @@
#define NULL ((void*)0)
void* malloc(size_t);
void* calloc(size_t, size_t);
void free(void*);
void* realloc(void*, size_t);
#if defined(TREE_SITTER_WASM_STDLIB) && !defined(TS_WASM_EXPORT)
#define TS_WASM_EXPORT(name) __attribute__((visibility("default"), export_name(name)))
#elif !defined(TS_WASM_EXPORT)
#define TS_WASM_EXPORT(name)
#endif
__attribute__((noreturn)) void abort(void);
TS_WASM_EXPORT("malloc") void* malloc(size_t);
TS_WASM_EXPORT("calloc") void* calloc(size_t, size_t);
TS_WASM_EXPORT("free") void free(void*);
TS_WASM_EXPORT("realloc") void* realloc(void*, size_t);
TS_WASM_EXPORT("abort") __attribute__((noreturn)) void abort(void);
#endif // TREE_SITTER_WASM_STDLIB_H_

View file

@ -3,20 +3,47 @@
#include <stdint.h>
void *memchr(const void *src, int c, size_t n);
#ifndef NULL
#define NULL ((void *)0)
#endif
int memcmp(const void *lhs, const void *rhs, size_t count);
#ifndef weak_alias
#define weak_alias(old, new) \
extern __typeof(old) new __attribute__((__weak__, __alias__(#old)))
#endif
void *memcpy(void *restrict dst, const void *restrict src, size_t size);
#if defined(TREE_SITTER_WASM_STDLIB) && !defined(TS_WASM_EXPORT)
#define TS_WASM_EXPORT(name) __attribute__((visibility("default"), export_name(name)))
#elif !defined(TS_WASM_EXPORT)
#define TS_WASM_EXPORT(name)
#endif
void *memmove(void *dst, const void *src, size_t count);
TS_WASM_EXPORT("memchr") void *memchr(const void *src, int c, size_t n);
void *memset(void *dst, int value, size_t count);
TS_WASM_EXPORT("memcmp") int memcmp(const void *lhs, const void *rhs, size_t count);
char *strchr(const char *str, int c);
TS_WASM_EXPORT("memcpy") void *memcpy(void *restrict dst, const void *restrict src, size_t size);
size_t strlen(const char *str);
TS_WASM_EXPORT("memmove") void *memmove(void *dst, const void *src, size_t count);
int strncmp(const char *left, const char *right, size_t n);
TS_WASM_EXPORT("memset") void *memset(void *dst, int value, size_t count);
TS_WASM_EXPORT("strchr") char *strchr(const char *str, int c);
TS_WASM_EXPORT("strcmp") int strcmp(const char *left, const char *right);
TS_WASM_EXPORT("strlen") size_t strlen(const char *str);
TS_WASM_EXPORT("strncat") char *strncat(char *restrict dest, const char *restrict src, size_t count);
TS_WASM_EXPORT("strncmp") int strncmp(const char *left, const char *right, size_t n);
char *__stpncpy(char *restrict dest, const char *restrict src, size_t count);
char *__strchrnul(const char *str, int c);
char *stpncpy(char *restrict dest, const char *restrict src, size_t count);
TS_WASM_EXPORT("strncpy") char *strncpy(char *restrict dest, const char *restrict src, size_t count);
#endif // TREE_SITTER_WASM_STRING_H_

View file

@ -0,0 +1,12 @@
#ifndef TREE_SITTER_WASM_WCHAR_H_
#define TREE_SITTER_WASM_WCHAR_H_
#include <stdint.h>
typedef __WCHAR_TYPE__ wchar_t;
wchar_t *wcschr(const wchar_t *str, wchar_t c);
size_t wcslen(const wchar_t *str);
#endif // TREE_SITTER_WASM_WCHAR_H_

View file

@ -1,176 +1,40 @@
#ifndef TREE_SITTER_WASM_WCTYPE_H_
#define TREE_SITTER_WASM_WCTYPE_H_
#include <stdbool.h>
typedef unsigned int wint_t;
typedef void *locale_t;
typedef int wint_t;
#ifndef weak_alias
#define weak_alias(old, new) \
extern __typeof(old) new __attribute__((__weak__, __alias__(#old)))
#endif
int iswlower(wint_t wch);
#if defined(TREE_SITTER_WASM_STDLIB) && !defined(TS_WASM_EXPORT)
#define TS_WASM_EXPORT(name) __attribute__((visibility("default"), export_name(name)))
#elif !defined(TS_WASM_EXPORT)
#define TS_WASM_EXPORT(name)
#endif
int iswupper(wint_t wch);
TS_WASM_EXPORT("iswalnum") int iswalnum(wint_t wch);
int iswpunct(wint_t wch);
TS_WASM_EXPORT("iswalpha") int iswalpha(wint_t wch);
static inline bool iswalpha(wint_t wch) {
switch (wch) {
case L'a':
case L'b':
case L'c':
case L'd':
case L'e':
case L'f':
case L'g':
case L'h':
case L'i':
case L'j':
case L'k':
case L'l':
case L'm':
case L'n':
case L'o':
case L'p':
case L'q':
case L'r':
case L's':
case L't':
case L'u':
case L'v':
case L'w':
case L'x':
case L'y':
case L'z':
case L'A':
case L'B':
case L'C':
case L'D':
case L'E':
case L'F':
case L'G':
case L'H':
case L'I':
case L'J':
case L'K':
case L'L':
case L'M':
case L'N':
case L'O':
case L'P':
case L'Q':
case L'R':
case L'S':
case L'T':
case L'U':
case L'V':
case L'W':
case L'X':
case L'Y':
case L'Z':
return true;
default:
return false;
}
}
TS_WASM_EXPORT("iswblank") int iswblank(wint_t wch);
static inline bool iswdigit(wint_t wch) {
switch (wch) {
case L'0':
case L'1':
case L'2':
case L'3':
case L'4':
case L'5':
case L'6':
case L'7':
case L'8':
case L'9':
return true;
default:
return false;
}
}
TS_WASM_EXPORT("iswdigit") int iswdigit(wint_t wch);
static inline bool iswalnum(wint_t wch) {
switch (wch) {
case L'a':
case L'b':
case L'c':
case L'd':
case L'e':
case L'f':
case L'g':
case L'h':
case L'i':
case L'j':
case L'k':
case L'l':
case L'm':
case L'n':
case L'o':
case L'p':
case L'q':
case L'r':
case L's':
case L't':
case L'u':
case L'v':
case L'w':
case L'x':
case L'y':
case L'z':
case L'A':
case L'B':
case L'C':
case L'D':
case L'E':
case L'F':
case L'G':
case L'H':
case L'I':
case L'J':
case L'K':
case L'L':
case L'M':
case L'N':
case L'O':
case L'P':
case L'Q':
case L'R':
case L'S':
case L'T':
case L'U':
case L'V':
case L'W':
case L'X':
case L'Y':
case L'Z':
case L'0':
case L'1':
case L'2':
case L'3':
case L'4':
case L'5':
case L'6':
case L'7':
case L'8':
case L'9':
return true;
default:
return false;
}
}
TS_WASM_EXPORT("iswlower") int iswlower(wint_t wch);
static inline bool iswspace(wint_t wch) {
switch (wch) {
case L' ':
case L'\t':
case L'\n':
case L'\v':
case L'\f':
case L'\r':
return true;
default:
return false;
}
}
TS_WASM_EXPORT("iswpunct") int iswpunct(wint_t wch);
TS_WASM_EXPORT("iswspace") int iswspace(wint_t wch);
TS_WASM_EXPORT("iswupper") int iswupper(wint_t wch);
TS_WASM_EXPORT("iswxdigit") int iswxdigit(wint_t wch);
TS_WASM_EXPORT("towlower") wint_t towlower(wint_t wch);
TS_WASM_EXPORT("towupper") wint_t towupper(wint_t wch);
#endif // TREE_SITTER_WASM_WCTYPE_H_

View file

@ -1,84 +0,0 @@
#include <string.h>
// Derived from musl (MIT): https://git.musl-libc.org/cgit/musl/tree/src/string/memchr.c
void *memchr(const void *src, int c, size_t n) {
const unsigned char *s = src;
c = (unsigned char)c;
for (; n && *s != c; s++, n--);
return n ? (void *)s : 0;
}
int memcmp(const void *lhs, const void *rhs, size_t count) {
const unsigned char *l = lhs;
const unsigned char *r = rhs;
while (count--) {
if (*l != *r) {
return *l - *r;
}
l++;
r++;
}
return 0;
}
void *memcpy(void *restrict dst, const void *restrict src, size_t size) {
unsigned char *d = dst;
const unsigned char *s = src;
while (size--) {
*d++ = *s++;
}
return dst;
}
void *memmove(void *dst, const void *src, size_t count) {
unsigned char *d = dst;
const unsigned char *s = src;
if (d < s) {
while (count--) {
*d++ = *s++;
}
} else if (d > s) {
d += count;
s += count;
while (count--) {
*(--d) = *(--s);
}
}
return dst;
}
void *memset(void *dst, int value, size_t count) {
unsigned char *p = dst;
while (count--) {
*p++ = (unsigned char)value;
}
return dst;
}
char *strchr(const char *str, int c) {
while (*str != (char)c) {
if (*str == '\0') {
return 0;
}
str++;
}
return (char *)str;
}
size_t strlen(const char *str) {
const char *s = str;
while (*s) s++;
return s - str;
}
int strncmp(const char *left, const char *right, size_t n) {
while (n-- > 0) {
if (*left != *right) {
return *(unsigned char *)left - *(unsigned char *)right;
}
if (*left == '\0') break;
left++;
right++;
}
return 0;
}

View file

@ -1,16 +0,0 @@
#include <wctype.h>
int iswlower(wint_t wch) {
return (unsigned)wch - L'a' < 26;
}
int iswupper(wint_t wch) {
return (unsigned)wch - L'A' < 26;
}
int iswpunct(wint_t wch) {
return (wch >= 33 && wch <= 47) ||
(wch >= 58 && wch <= 64) ||
(wch >= 91 && wch <= 96) ||
(wch >= 123 && wch <= 126);
}

View file

@ -0,0 +1,15 @@
# Unsupported Tree-sitter 0.26 Wasm build
Published versions of the `tree-sitter` Rust crate through 0.26 compile
`stdio.c`, `stdlib.c`, and `string.c` from the directory advertised by
`tree-sitter-language`'s `wasm-src` build-script metadata when targeting
`wasm32-unknown-unknown`.
That integration used a separate, non-thread-safe allocator and an incomplete
libc implementation. Current versions of Tree-sitter instead provide their own
Wasm libc implementation and forward C allocation functions to Rust's
application-selected global allocator.
The translation units in this directory produce an actionable compilation
error for old Tree-sitter versions. They can be removed in a breaking
`tree-sitter-language` release that those older runtimes cannot select.

View file

@ -0,0 +1 @@
#error "tree-sitter 0.26 is incompatible with this version of tree-sitter-language on wasm32-unknown-unknown; upgrade tree-sitter to 0.27 or newer"

View file

@ -0,0 +1 @@
#error "tree-sitter 0.26 is incompatible with this version of tree-sitter-language on wasm32-unknown-unknown; upgrade tree-sitter to 0.27 or newer"

View file

@ -0,0 +1 @@
#error "tree-sitter 0.26 is incompatible with this version of tree-sitter-language on wasm32-unknown-unknown; upgrade tree-sitter to 0.27 or newer"

View file

@ -53,8 +53,41 @@ const EXPORTED_RUNTIME_METHODS: [&str; 20] = [
];
const WASI_SDK_VERSION: &str = include_str!("../../loader/wasi-sdk-version").trim_ascii();
const WASI_LIBC_REVISION: &str = "161b3195fc2558d2b1ba3eb9ffae3b2b47407623";
const BINARYEN_VERSION: &str = include_str!("../../loader/binaryen-version").trim_ascii();
const WASI_LIBC_FILES: &[&str] = &[
"ctype/alpha.h",
"ctype/casemap.h",
"ctype/isblank.c",
"ctype/iswalnum.c",
"ctype/iswalpha.c",
"ctype/iswblank.c",
"ctype/iswdigit.c",
"ctype/iswlower.c",
"ctype/iswpunct.c",
"ctype/iswspace.c",
"ctype/iswupper.c",
"ctype/iswxdigit.c",
"ctype/punct.h",
"ctype/towctrans.c",
"string/memchr.c",
"string/memcmp.c",
"string/memcpy.c",
"string/memmove.c",
"string/memset.c",
"string/strchr.c",
"string/strchrnul.c",
"string/strcmp.c",
"string/strlen.c",
"string/stpncpy.c",
"string/strncat.c",
"string/strncmp.c",
"string/strncpy.c",
"string/wcschr.c",
"string/wcslen.c",
];
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
const ARCH_OS: Result<&str, LoaderError> = Ok("arm64-macos");
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
@ -173,7 +206,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
let exported_functions = format!(
"{}{}",
fs::read_to_string("lib/src/wasm/stdlib-symbols.txt")?,
fs::read_to_string("lib/src/wasm-stdlib/imports.txt")?,
fs::read_to_string("lib/binding_web/lib/exports.txt")?
)
.replace('"', "")
@ -599,8 +632,80 @@ fn extract_tar_gz_with_strip(archive_path: &Path, destination: &Path) -> Result<
Ok(())
}
pub fn vendor_wasm_stdlib() -> Result<()> {
let source_dir = ensure_wasi_libc_source_exists()?;
let source_dir = source_dir.join("libc-top-half/musl");
let destination = Path::new("lib/src/wasm-stdlib/libc");
for directory in ["ctype", "string"] {
let directory = destination.join(directory);
if directory.exists() {
fs::remove_dir_all(&directory)?;
}
fs::create_dir_all(directory)?;
}
fs::copy(source_dir.join("COPYRIGHT"), destination.join("LICENSE"))?;
let source_dir = source_dir.join("src");
for relative_path in WASI_LIBC_FILES {
let relative_path = Path::new(relative_path);
fs::copy(
source_dir.join(relative_path),
destination.join(relative_path),
)?;
}
println!(
"Vendored {} wasi-libc files from {WASI_LIBC_REVISION}",
WASI_LIBC_FILES.len()
);
Ok(())
}
fn ensure_wasi_libc_source_exists() -> Result<PathBuf> {
let cache_dir = etcetera::choose_base_strategy()?
.cache_dir()
.join("tree-sitter")
.join("wasi-libc");
fs::create_dir_all(&cache_dir)?;
let source_dir = cache_dir.join(WASI_LIBC_REVISION);
if source_dir.join("libc-top-half/musl/COPYRIGHT").is_file() {
return Ok(source_dir);
}
let archive_name = format!("wasi-libc-{WASI_LIBC_REVISION}.tar.gz");
let archive_path = cache_dir.join(&archive_name);
let url =
format!("https://github.com/WebAssembly/wasi-libc/archive/{WASI_LIBC_REVISION}.tar.gz");
eprintln!("Downloading wasi-libc from {url}...");
let status = Command::new("curl")
.args(["-f", "-L", "-o"])
.arg(&archive_path)
.arg(&url)
.status()
.map_err(|error| LoaderError::Curl(url.clone(), error))?;
if !status.success() {
return Err(LoaderError::WasmToolDownload {
tool: "wasi-libc",
url,
}
.into());
}
let temporary_dir = cache_dir.join(format!(".{WASI_LIBC_REVISION}.tmp"));
if temporary_dir.exists() {
fs::remove_dir_all(&temporary_dir)?;
}
fs::create_dir_all(&temporary_dir)?;
extract_tar_gz_with_strip(&archive_path, &temporary_dir)?;
fs::rename(&temporary_dir, &source_dir)?;
fs::remove_file(archive_path)?;
Ok(source_dir)
}
pub fn run_wasm_stdlib() -> Result<()> {
let export_flags = include_str!("../../../lib/src/wasm/stdlib-symbols.txt")
let export_flags = include_str!("../../../lib/src/wasm-stdlib/imports.txt")
.lines()
.map(|line| format!("-Wl,--export={}", &line[1..line.len() - 2]))
.collect::<Vec<String>>();
@ -613,7 +718,7 @@ pub fn run_wasm_stdlib() -> Result<()> {
"stdlib.wasm",
"-Os",
"-fPIC",
"-DTREE_SITTER_FEATURE_WASM",
"-nostdlib",
"-Wl,--no-entry",
"-Wl,--stack-first",
"-Wl,-z",
@ -627,8 +732,10 @@ pub fn run_wasm_stdlib() -> Result<()> {
"-Wl,--export=reset_heap",
])
.args(&export_flags)
.arg("crates/language/wasm/src/stdlib.c")
.arg("crates/language/wasm/src/wctype.c")
.arg("-Icrates/language/wasm/include")
.arg("lib/src/wasm-stdlib/libc.c")
.arg("lib/src/wasm-stdlib/stdio.c")
.arg("lib/src/wasm-stdlib/external_scanner_allocator.c")
.output()?;
bail_on_err(
@ -656,7 +763,7 @@ pub fn run_wasm_stdlib() -> Result<()> {
"Failed to run xxd on the compiled Tree-sitter Wasm stdlib",
)?;
fs::write("lib/src/wasm/wasm-stdlib.h", xxd.stdout)?;
fs::write("lib/src/wasm-stdlib/external_scanner_stdlib.h", xxd.stdout)?;
fs::rename("stdlib.wasm", "target/stdlib.wasm")?;

View file

@ -49,8 +49,12 @@ enum Commands {
Test(Test),
/// Run the Wasm test suite
TestWasm,
/// Test the Rust binding in a WebAssembly web environment.
TestRustWasmWeb,
/// Upgrade the wasmtime dependency.
UpgradeWasmtime(UpgradeWasmtime),
/// Refresh the vendored Wasm standard-library sources.
VendorWasmStdlib,
}
#[derive(Args)]
@ -236,9 +240,11 @@ fn run() -> Result<()> {
Commands::GenerateWasmExports => generate::run_wasm_exports()?,
Commands::Test(test_options) => test::run(&test_options)?,
Commands::TestWasm => test::run_wasm()?,
Commands::TestRustWasmWeb => test::run_rust_wasm_web()?,
Commands::UpgradeWasmtime(upgrade_wasmtime_options) => {
upgrade_wasmtime::run(&upgrade_wasmtime_options)?;
}
Commands::VendorWasmStdlib => build_wasm::vendor_wasm_stdlib()?,
}
Ok(())

View file

@ -7,7 +7,7 @@ use std::{
use anyhow::{Result, anyhow};
use regex::Regex;
use crate::{Test, bail_on_err};
use crate::{Test, bail_on_err, build_wasm::ensure_wasi_sdk_exists};
pub fn run(args: &Test) -> Result<()> {
let test_flags = if args.address_sanitizer {
@ -157,3 +157,106 @@ pub fn run_wasm() -> Result<()> {
Ok(())
}
pub fn run_rust_wasm_web() -> Result<()> {
let clang = ensure_wasi_sdk_exists()?;
let manifest_path = Path::new("test/fixtures/rust_wasm_web/Cargo.toml");
let target_dir = Path::new("target/rust-wasm-web-test");
let target = "wasm32-unknown-unknown";
std::fs::create_dir_all(target_dir)?;
let mut language_paths = Vec::new();
for language_name in ["python", "ruby"] {
let language_dir = Path::new("test/fixtures/grammars").join(language_name);
for source_path in ["src/parser.c", "src/scanner.c"] {
if !language_dir.join(source_path).is_file() {
return Err(anyhow!(
"Missing generated {language_name} source `{source_path}`; run `cargo xtask generate-fixtures --wasm` first"
));
}
}
let language_path =
env::current_dir()?.join(target_dir.join(format!("tree-sitter-{language_name}.wasm")));
let mut compile_language = Command::new(&clang);
compile_language.current_dir(&language_dir).args([
"--target=wasm32-wasip1",
"-matomics",
"-mbulk-memory",
"-o",
language_path.to_str().unwrap(),
"-fPIC",
"-shared",
"--no-wasm-opt",
"-Os",
&format!("-Wl,--export=tree_sitter_{language_name}"),
"-Wl,--allow-undefined",
"-Wl,--no-entry",
"-Wl,--shared-memory",
"-Wl,--max-memory=268435456",
"-nostdlib",
"-fno-exceptions",
"-fvisibility=hidden",
"-I",
"src",
"-I",
env::current_dir()?
.join("crates/language/wasm/include")
.to_str()
.unwrap(),
"src/parser.c",
"src/scanner.c",
]);
bail_on_err(
&compile_language.output()?,
&format!("Failed to compile the {language_name} Rust Wasm web test language"),
)?;
language_paths.push(language_path);
}
let mut cargo = Command::new("cargo");
let nightly_toolchain =
env::var("TREE_SITTER_NIGHTLY_TOOLCHAIN").unwrap_or_else(|_| "nightly".to_string());
cargo
.args([
&format!("+{nightly_toolchain}"),
"build",
"-Z",
"build-std=std,panic_abort",
"--locked",
"--manifest-path",
manifest_path.to_str().unwrap(),
"--target",
target,
"--target-dir",
target_dir.to_str().unwrap(),
])
.env("CC_wasm32_unknown_unknown", clang)
.env(
"CFLAGS_wasm32_unknown_unknown",
format!(
"-matomics -mbulk-memory -I{}",
env::current_dir()?.join("crates/language/wasm/include").display()
),
)
.env(
"RUSTFLAGS",
"-D warnings -A unstable-features -C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--import-memory -C link-arg=--shared-memory -C link-arg=--max-memory=268435456 -C link-arg=--export-table -C link-arg=--growable-table -C link-arg=--export=__stack_pointer",
);
bail_on_err(&cargo.output()?, "Failed to compile the Rust Wasm web test")?;
let runtime_path = target_dir
.join(target)
.join("debug")
.join("rust_wasm_web_test.wasm");
let node = env::var_os("EMSDK_NODE").unwrap_or_else(|| "node".into());
let mut command = Command::new(node);
command
.arg("test/fixtures/rust_wasm_web/run.mjs")
.arg(runtime_path)
.args(language_paths);
if !command.status()?.success() {
return Err(anyhow!("Failed to run the Rust Wasm web test"));
}
println!("Rust Wasm web test passed");
Ok(())
}

View file

@ -140,7 +140,15 @@ cargo xtask test-wasm
#### Wasm Stdlib
The tree-sitter Wasm stdlib can be built via xtask:
The libc sources shared by Wasm external scanners and
`wasm32-unknown-unknown` Rust applications can be refreshed from the
`wasi-libc` revision pinned by the current WASI SDK:
```sh
cargo xtask vendor-wasm-stdlib
```
The external scanner Wasm stdlib can then be built via xtask:
```sh
cargo xtask build-wasm-stdlib
@ -150,7 +158,9 @@ This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WA
environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases]
page. Similarly, this command also looks for [ the `wasm-opt` tool from binaryen][binaryen] indicated by the `TREE_SITTER_BINARYEN_PATH`
environment variable. `wasm-opt` and the rest of the binaryen tool suite can be downloaded from the project's [releases][binaryen-releases]
page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via `cargo xtask build-wasm-stdlib`.
page. Note that changes under `lib/src/wasm-stdlib/` require rebuilding
`lib/src/wasm-stdlib/external_scanner_stdlib.h` via
`cargo xtask build-wasm-stdlib`.
### Debugging

View file

@ -23,7 +23,7 @@ include = [
"/src/*.c",
"/src/portable/*",
"/src/unicode/*",
"/src/wasm/*",
"/src/wasm-stdlib/**/*",
"/include/tree_sitter/api.h",
"/LICENSE",
]

View file

@ -203,7 +203,7 @@ unsafe extern "C" {
pub fn ts_parser_language(self_: *const TSParser) -> *const TSLanguage;
}
unsafe extern "C" {
#[doc = " Set the language that the parser should use for parsing.\n\n Returns a boolean indicating whether or not the language was successfully\n assigned. True means assignment succeeded. False means there was a version\n mismatch: the language was generated with an incompatible version of the\n Tree-sitter CLI. Check the language's ABI version using [`ts_language_abi_version`]\n and compare it to this library's [`TREE_SITTER_LANGUAGE_VERSION`] and\n [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants."]
#[doc = " Set the language that the parser should use for parsing.\n\n Returns a boolean indicating whether or not the language was successfully\n assigned. True means assignment succeeded. False means the language cannot\n be used for parsing, or it was generated with an incompatible version of the\n Tree-sitter CLI. Check whether the language can be used for parsing with\n [`ts_language_is_parseable`]. Check the language's ABI version using\n [`ts_language_abi_version`] and compare it to this library's\n [`TREE_SITTER_LANGUAGE_VERSION`] and\n [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants."]
pub fn ts_parser_set_language(self_: *mut TSParser, language: *const TSLanguage) -> bool;
}
unsafe extern "C" {
@ -291,7 +291,7 @@ unsafe extern "C" {
) -> TSNode;
}
unsafe extern "C" {
#[doc = " Get the language that was used to parse the syntax tree."]
#[doc = " Get the language that was used to parse the syntax tree.\n\n When Tree-sitter is compiled to WebAssembly, this returns the original\n language if the tree is being accessed from the same WebAssembly instance\n that created it. Otherwise, this returns a copy of the language that can be\n used to inspect the tree but cannot be assigned to a parser."]
pub fn ts_tree_language(self_: *const TSTree) -> *const TSLanguage;
}
unsafe extern "C" {
@ -323,7 +323,7 @@ unsafe extern "C" {
pub fn ts_node_symbol(self_: TSNode) -> TSSymbol;
}
unsafe extern "C" {
#[doc = " Get the node's language."]
#[doc = " Get the node's language.\n\n When Tree-sitter is compiled to WebAssembly, this returns the original\n language if the node is being accessed from the same WebAssembly instance\n that created its tree. Otherwise, this returns a copy of the language that\n can be used to inspect the tree but cannot be assigned to a parser."]
pub fn ts_node_language(self_: TSNode) -> *const TSLanguage;
}
unsafe extern "C" {
@ -761,6 +761,10 @@ unsafe extern "C" {
#[doc = " Free any dynamically-allocated resources for this language, if\n this is the last reference."]
pub fn ts_language_delete(self_: *const TSLanguage);
}
unsafe extern "C" {
#[doc = " Check whether this language can be assigned to a parser.\n\n Languages obtained from a syntax tree may be used to inspect that tree, but\n are not necessarily usable for parsing. When Tree-sitter is compiled to\n WebAssembly, a language obtained from a tree can be used for parsing only\n within the same WebAssembly instance that created the tree, because lexer\n function pointers are local to a WebAssembly instance."]
pub fn ts_language_is_parseable(self_: *const TSLanguage) -> bool;
}
unsafe extern "C" {
#[doc = " Get the number of distinct node types in the language."]
pub fn ts_language_symbol_count(self_: *const TSLanguage) -> u32;

View file

@ -8,7 +8,7 @@ fn main() {
generate_bindings(&out_dir);
fs::copy(
"src/wasm/stdlib-symbols.txt",
"src/wasm-stdlib/imports.txt",
out_dir.join("stdlib-symbols.txt"),
)
.unwrap();
@ -64,19 +64,9 @@ fn configure_wasm_build(config: &mut cc::Build) {
"Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"
);
};
let Ok(wasm_src) = env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(PathBuf::from) else {
panic!(
"Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"
);
};
config.include(&wasm_headers);
config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
wasm_src.join("wctype.c"),
]);
config
.define("TREE_SITTER_WASM_STDLIB", "")
.include(&wasm_headers);
}
#[cfg(feature = "bindgen")]

View file

@ -5,7 +5,10 @@
pub mod ffi;
mod util;
#[cfg(not(feature = "std"))]
#[cfg(any(
not(feature = "std"),
all(target_arch = "wasm32", target_os = "unknown")
))]
extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, format, string::String, string::ToString, vec::Vec};
@ -36,6 +39,9 @@ mod wasm_language;
#[cfg_attr(docsrs, doc(cfg(feature = "wasm")))]
pub use wasm_language::*;
#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
mod wasm_allocator;
/// The latest ABI version that is supported by the current version of the
/// library.
///
@ -444,12 +450,13 @@ pub struct QueryCapture<'tree> {
pub index: u32,
}
/// An error that occurred when trying to assign an incompatible [`Language`] to
/// a [`Parser`]. If the `wasm` feature is enabled, this can also indicate a failure
/// to load the Wasm store.
/// An error that occurred when trying to assign a [`Language`] to a [`Parser`].
/// If the `wasm` feature is enabled, this can also indicate a failure to load
/// the Wasm store.
#[derive(Debug, PartialEq, Eq)]
pub enum LanguageError {
Version(usize),
NotParseable,
#[cfg(feature = "wasm")]
Wasm,
}
@ -505,6 +512,18 @@ impl Language {
Self(unsafe { builder.into_raw()().cast() })
}
/// Check whether this language can be assigned to a parser.
///
/// When Tree-sitter is compiled to WebAssembly, languages obtained from a
/// syntax tree can be used for parsing only within the same WebAssembly
/// instance that created the tree. In other instances, such languages can
/// still be used to inspect syntax trees.
#[doc(alias = "ts_language_is_parseable")]
#[must_use]
pub fn is_parseable(&self) -> bool {
unsafe { ffi::ts_language_is_parseable(self.0) }
}
/// Get the name of this language. This returns `None` in older parsers.
#[doc(alias = "ts_language_name")]
#[must_use]
@ -738,15 +757,17 @@ impl Parser {
/// Set the language that the parser should use for parsing.
///
/// Returns a Result indicating whether or not the language was successfully
/// assigned. True means assignment succeeded. False means there was a
/// version mismatch: the language was generated with an incompatible
/// version of the Tree-sitter CLI. Check the language's version using
/// [`Language::version`] and compare it to this library's
/// [`LANGUAGE_VERSION`] and [`MIN_COMPATIBLE_LANGUAGE_VERSION`] constants.
/// assigned. Assignment fails if the language cannot be used for parsing,
/// or if it was generated with an incompatible version of the Tree-sitter
/// CLI. Check this using [`Language::is_parseable`] and
/// [`Language::abi_version`].
#[doc(alias = "ts_parser_set_language")]
pub fn set_language(&mut self, language: &Language) -> Result<(), LanguageError> {
let version = language.abi_version();
if (MIN_COMPATIBLE_LANGUAGE_VERSION..=LANGUAGE_VERSION).contains(&version) {
if !language.is_parseable() {
return Err(LanguageError::NotParseable);
}
#[cfg_attr(
not(feature = "wasm"),
expect(unused_variables, reason = "only used when wasm feature is enabled")
@ -1480,6 +1501,11 @@ impl Tree {
}
/// Get the language that was used to parse the syntax tree.
///
/// When Tree-sitter is compiled to WebAssembly, this returns the original
/// language if the tree is being accessed from the same WebAssembly
/// instance that created it. Otherwise, this returns a copy of the language
/// that can be used to inspect the tree but cannot be assigned to a parser.
#[doc(alias = "ts_tree_language")]
#[must_use]
pub fn language(&self) -> LanguageRef {
@ -1644,6 +1670,12 @@ impl<'tree> Node<'tree> {
}
/// Get the [`Language`] that was used to parse this node's syntax tree.
///
/// When Tree-sitter is compiled to WebAssembly, this returns the original
/// language if the node is being accessed from the same WebAssembly
/// instance that created its tree. Otherwise, this returns a copy of the
/// language that can be used to inspect the tree but cannot be assigned to
/// a parser.
#[doc(alias = "ts_node_language")]
#[must_use]
pub fn language(&self) -> LanguageRef<'tree> {
@ -3858,6 +3890,9 @@ impl fmt::Display for LanguageError {
"Incompatible language version {version}. Expected minimum {MIN_COMPATIBLE_LANGUAGE_VERSION}, maximum {LANGUAGE_VERSION}",
)
}
Self::NotParseable => {
write!(f, "Language cannot be used for parsing.")
}
#[cfg(feature = "wasm")]
Self::Wasm => {
write!(f, "Failed to load the Wasm store.")
@ -4065,19 +4100,27 @@ impl error::Error for LanguageError {}
#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
impl error::Error for QueryError {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Send for Language {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Sync for Language {}
unsafe impl Send for Node<'_> {}
unsafe impl Sync for Node<'_> {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Send for LookaheadIterator {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Sync for LookaheadIterator {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Send for LookaheadNamesIterator<'_> {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Sync for LookaheadNamesIterator<'_> {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Send for Parser {}
#[cfg(not(target_family = "wasm"))]
unsafe impl Sync for Parser {}
unsafe impl Send for Query {}

View file

@ -0,0 +1,107 @@
//! C allocation functions backed by Rust's application-selected global allocator.
//!
//! This is intentionally a Tree-sitter compatibility shim rather than a
//! general-purpose implementation of the WebAssembly C allocation ABI.
//! Tree-sitter and its supported external scanners only allocate types whose
//! alignment is at most eight bytes.
use alloc::alloc::{Layout, alloc, alloc_zeroed, dealloc, realloc as rust_realloc};
use core::{ffi::c_void, mem, ptr};
const C_ALIGNMENT: usize = 8;
#[repr(C, align(8))]
struct Header {
payload_size: usize,
}
const HEADER_SIZE: usize = mem::size_of::<Header>();
const _: () = assert!(HEADER_SIZE == C_ALIGNMENT);
fn layout(payload_size: usize) -> Option<Layout> {
let allocation_size = HEADER_SIZE.checked_add(payload_size)?;
Layout::from_size_align(allocation_size, C_ALIGNMENT).ok()
}
unsafe fn header_from_payload(payload: *mut c_void) -> *mut Header {
unsafe { payload.cast::<u8>().sub(HEADER_SIZE).cast() }
}
#[unsafe(no_mangle)]
unsafe extern "C" fn malloc(size: usize) -> *mut c_void {
let Some(layout) = layout(size).filter(|_| size != 0) else {
return ptr::null_mut();
};
let header = unsafe { alloc(layout).cast::<Header>() };
if header.is_null() {
return ptr::null_mut();
}
unsafe {
header.write(Header { payload_size: size });
header.add(1).cast()
}
}
#[unsafe(no_mangle)]
unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void {
let Some(payload_size) = count.checked_mul(size) else {
return ptr::null_mut();
};
let Some(layout) = layout(payload_size).filter(|_| payload_size != 0) else {
return ptr::null_mut();
};
let header = unsafe { alloc_zeroed(layout).cast::<Header>() };
if header.is_null() {
return ptr::null_mut();
}
unsafe {
header.write(Header { payload_size });
header.add(1).cast()
}
}
#[unsafe(no_mangle)]
unsafe extern "C" fn realloc(payload: *mut c_void, new_size: usize) -> *mut c_void {
if payload.is_null() {
return unsafe { malloc(new_size) };
}
if new_size == 0 {
unsafe { free(payload) };
return ptr::null_mut();
}
let old_header = unsafe { header_from_payload(payload) };
let old_size = unsafe { (*old_header).payload_size };
let Some(old_layout) = layout(old_size) else {
return ptr::null_mut();
};
let Some(new_layout) = layout(new_size) else {
return ptr::null_mut();
};
let new_header =
unsafe { rust_realloc(old_header.cast(), old_layout, new_layout.size()).cast::<Header>() };
if new_header.is_null() {
return ptr::null_mut();
}
unsafe {
(*new_header).payload_size = new_size;
new_header.add(1).cast()
}
}
#[unsafe(no_mangle)]
unsafe extern "C" fn free(payload: *mut c_void) {
if payload.is_null() {
return;
}
let header = unsafe { header_from_payload(payload) };
let payload_size = unsafe { (*header).payload_size };
let allocation_layout = layout(payload_size).unwrap();
unsafe { dealloc(header.cast(), allocation_layout) };
}
#[unsafe(no_mangle)]
extern "C" fn abort() -> ! {
core::arch::wasm32::unreachable()
}

View file

@ -230,10 +230,12 @@ const TSLanguage *ts_parser_language(const TSParser *self);
* Set the language that the parser should use for parsing.
*
* Returns a boolean indicating whether or not the language was successfully
* assigned. True means assignment succeeded. False means there was a version
* mismatch: the language was generated with an incompatible version of the
* Tree-sitter CLI. Check the language's ABI version using [`ts_language_abi_version`]
* and compare it to this library's [`TREE_SITTER_LANGUAGE_VERSION`] and
* assigned. True means assignment succeeded. False means the language cannot
* be used for parsing, or it was generated with an incompatible version of the
* Tree-sitter CLI. Check whether the language can be used for parsing with
* [`ts_language_is_parseable`]. Check the language's ABI version using
* [`ts_language_abi_version`] and compare it to this library's
* [`TREE_SITTER_LANGUAGE_VERSION`] and
* [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants.
*/
bool ts_parser_set_language(TSParser *self, const TSLanguage *language);
@ -434,6 +436,11 @@ TSNode ts_tree_root_node_with_offset(
/**
* Get the language that was used to parse the syntax tree.
*
* When Tree-sitter is compiled to WebAssembly, this returns the original
* language if the tree is being accessed from the same WebAssembly instance
* that created it. Otherwise, this returns a copy of the language that can be
* used to inspect the tree but cannot be assigned to a parser.
*/
const TSLanguage *ts_tree_language(const TSTree *self);
@ -504,6 +511,11 @@ TSSymbol ts_node_symbol(TSNode self);
/**
* Get the node's language.
*
* When Tree-sitter is compiled to WebAssembly, this returns the original
* language if the node is being accessed from the same WebAssembly instance
* that created its tree. Otherwise, this returns a copy of the language that
* can be used to inspect the tree but cannot be assigned to a parser.
*/
const TSLanguage *ts_node_language(TSNode self);
@ -1222,6 +1234,17 @@ const TSLanguage *ts_language_copy(const TSLanguage *self);
*/
void ts_language_delete(const TSLanguage *self);
/**
* Check whether this language can be assigned to a parser.
*
* Languages obtained from a syntax tree may be used to inspect that tree, but
* are not necessarily usable for parsing. When Tree-sitter is compiled to
* WebAssembly, a language obtained from a tree can be used for parsing only
* within the same WebAssembly instance that created the tree, because lexer
* function pointers are local to a WebAssembly instance.
*/
bool ts_language_is_parseable(const TSLanguage *self);
/**
* Get the number of distinct node types in the language.
*/

View file

@ -1,9 +1,67 @@
#include "./language.h"
#include "./atomic.h"
#include "./wasm_store.h"
#include "tree_sitter/api.h"
#include <stddef.h>
#include <string.h>
#ifdef __wasm__
typedef struct {
TSLanguage language;
volatile uint32_t ref_count;
} TSUnparseableLanguage;
// Linear memory can be shared by multiple WebAssembly instances, but a
// module-defined global belongs to one instance. Assign each instance an ID
// from a counter in shared memory so trees can identify the function table
// that owns their language callbacks.
static volatile uint32_t ts_language_next_context_id;
__asm__(
".globaltype ts_language_context_id, i32\n"
"ts_language_context_id:\n"
);
static inline TSUnparseableLanguage *ts_language__unparseable(const TSLanguage *self) {
return (TSUnparseableLanguage *)((char *)self - offsetof(TSUnparseableLanguage, language));
}
static inline bool ts_language__is_unparseable(const TSLanguage *self) {
return (
self &&
!self->lex_fn &&
self->external_scanner.states == (const bool *)self
);
}
uint32_t ts_language_current_context_id(void) {
uint32_t result;
__asm__(
"global.get ts_language_context_id\n"
"local.set %0\n"
: "=r"(result)
);
if (!result) {
result = atomic_inc(&ts_language_next_context_id);
__asm__(
"local.get %0\n"
"global.set ts_language_context_id\n"
:
: "r"(result)
);
}
return result;
}
#endif
const TSLanguage *ts_language_copy(const TSLanguage *self) {
#ifdef __wasm__
if (ts_language__is_unparseable(self)) {
atomic_inc(&ts_language__unparseable(self)->ref_count);
} else
#endif
if (self && ts_language_is_wasm(self)) {
ts_wasm_language_retain(self);
}
@ -11,11 +69,43 @@ const TSLanguage *ts_language_copy(const TSLanguage *self) {
}
void ts_language_delete(const TSLanguage *self) {
#ifdef __wasm__
if (ts_language__is_unparseable(self)) {
TSUnparseableLanguage *language = ts_language__unparseable(self);
if (atomic_dec(&language->ref_count) == 0) {
ts_free(language);
}
} else
#endif
if (self && ts_language_is_wasm(self)) {
ts_wasm_language_release(self);
}
}
bool ts_language_is_parseable(const TSLanguage *self) {
return self && self->lex_fn;
}
const TSLanguage *ts_language_copy_without_callbacks(const TSLanguage *self) {
#ifdef __wasm__
if (self && ts_language_is_parseable(self)) {
TSUnparseableLanguage *result = ts_malloc(sizeof(TSUnparseableLanguage));
result->language = *self;
result->language.lex_fn = NULL;
result->language.keyword_lex_fn = NULL;
result->language.external_scanner.states = (const bool *)&result->language;
result->language.external_scanner.create = NULL;
result->language.external_scanner.destroy = NULL;
result->language.external_scanner.scan = NULL;
result->language.external_scanner.serialize = NULL;
result->language.external_scanner.deserialize = NULL;
result->ref_count = 1;
return &result->language;
}
#endif
return ts_language_copy(self);
}
uint32_t ts_language_symbol_count(const TSLanguage *self) {
return self->symbol_count + self->alias_count;
}

View file

@ -45,6 +45,10 @@ TSLexerMode ts_language_lex_mode_for_state(const TSLanguage *self, TSStateId sta
bool ts_language_is_reserved_word(const TSLanguage *self, TSStateId state, TSSymbol symbol);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol);
TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol);
const TSLanguage *ts_language_copy_without_callbacks(const TSLanguage *self);
#ifdef __wasm__
uint32_t ts_language_current_context_id(void);
#endif
static inline const TSParseAction *ts_language_actions(
const TSLanguage *self,

View file

@ -11,3 +11,8 @@
#include "./tree_cursor.c"
#include "./tree.c"
#include "./wasm_store.c"
#ifdef TREE_SITTER_WASM_STDLIB
#include "./wasm-stdlib/libc.c"
#include "./wasm-stdlib/stdio.c"
#endif

View file

@ -467,7 +467,7 @@ const char *ts_node_type(TSNode self) {
}
const TSLanguage *ts_node_language(TSNode self) {
return self.tree->language;
return ts_tree_language(self.tree);
}
TSSymbol ts_node_grammar_symbol(TSNode self) {

View file

@ -2041,6 +2041,8 @@ bool ts_parser_set_language(TSParser *self, const TSLanguage *language) {
language->abi_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION
) return false;
if (!ts_language_is_parseable(language)) return false;
if (ts_language_is_wasm(language)) {
if (
!self->wasm_store ||

View file

@ -1,6 +1,7 @@
#include "tree_sitter/api.h"
#include "./array.h"
#include "./get_changed_ranges.h"
#include "./language.h"
#include "./length.h"
#include "./subtree.h"
#include "./tree_cursor.h"
@ -13,6 +14,10 @@ TSTree *ts_tree_new(
TSTree *result = ts_malloc(sizeof(TSTree));
result->root = root;
result->language = ts_language_copy(language);
#ifdef __wasm__
result->language_context_id = ts_language_current_context_id();
result->unparseable_language = NULL;
#endif
result->included_ranges = ts_calloc(included_range_count, sizeof(TSRange));
memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange));
result->included_range_count = included_range_count;
@ -21,7 +26,16 @@ TSTree *ts_tree_new(
TSTree *ts_tree_copy(const TSTree *self) {
ts_subtree_retain(self->root);
return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count);
TSTree *result = ts_tree_new(
self->root,
self->language,
self->included_ranges,
self->included_range_count
);
#ifdef __wasm__
result->language_context_id = self->language_context_id;
#endif
return result;
}
void ts_tree_delete(TSTree *self) {
@ -30,6 +44,9 @@ void ts_tree_delete(TSTree *self) {
SubtreePool pool = ts_subtree_pool_new(0);
ts_subtree_release(&pool, self->root);
ts_subtree_pool_delete(&pool);
#ifdef __wasm__
ts_language_delete(__atomic_load_n(&self->unparseable_language, __ATOMIC_ACQUIRE));
#endif
ts_language_delete(self->language);
ts_free(self->included_ranges);
ts_free(self);
@ -49,6 +66,31 @@ TSNode ts_tree_root_node_with_offset(
}
const TSLanguage *ts_tree_language(const TSTree *self) {
#ifdef __wasm__
if (self->language_context_id != ts_language_current_context_id()) {
TSTree *tree = (TSTree *)self;
const TSLanguage *result = __atomic_load_n(
&tree->unparseable_language,
__ATOMIC_ACQUIRE
);
if (!result) {
const TSLanguage *candidate = ts_language_copy_without_callbacks(self->language);
if (__atomic_compare_exchange_n(
&tree->unparseable_language,
&result,
candidate,
false,
__ATOMIC_RELEASE,
__ATOMIC_ACQUIRE
)) {
result = candidate;
} else {
ts_language_delete(candidate);
}
}
return result;
}
#endif
return self->language;
}

View file

@ -17,6 +17,12 @@ typedef struct {
struct TSTree {
Subtree root;
const TSLanguage *language;
#ifdef __wasm__
// The WebAssembly instance that owns the language's function pointers.
uint32_t language_context_id;
// Created lazily when another instance requests the tree's language.
const TSLanguage *volatile unparseable_language;
#endif
TSRange *included_ranges;
unsigned included_range_count;
};

View file

@ -0,0 +1,30 @@
# Tree-sitter Wasm standard library
Wasm language modules are compiled without a C standard library. Their
external scanners may import the functions listed in `imports.txt`, and the
environment loading the language must provide those functions.
This directory contains a shared implementation of that standard-library
subset:
- `libc/` contains sources vendored from the `wasi-libc` revision used by the
repository's pinned WASI SDK.
- `stdio.c` is Tree-sitter's scanner-oriented stdio implementation. It provides
in-memory formatting and intentionally implements stream operations as
no-ops.
- `external_scanner_allocator.c` is the resettable allocator used for isolated
Wasm language modules.
- `external_scanner_stdlib.h` is a generated Wasm module containing the
vendored libc subset, `stdio.c`, and the resettable allocator.
When the Tree-sitter Rust library is compiled for `wasm32-unknown-unknown`, the
same vendored libc sources and `stdio.c` are linked directly into the
application. In that environment, allocation is instead provided by Rust's
application-selected global allocator.
To refresh the vendored sources and regenerate the embedded module, run:
```sh
cargo xtask vendor-wasm-stdlib
cargo xtask build-wasm-stdlib
```

View file

@ -1,10 +1,9 @@
// This file implements a very simple allocator for external scanners running
// in Wasm. Allocation is just bumping a static pointer and growing the heap
// as needed, and freeing is just adding the freed region to a free list.
// When additional memory is allocated, the free list is searched first.
// If there is not a suitable region in the free list, the heap is
// grown as necessary, and the allocation is made at the end of the heap.
// When the heap is reset, all allocated memory is considered freed.
// This allocator is used by external scanners in separately compiled Wasm
// language modules. Allocation bumps a static pointer and grows the heap as
// needed, while freeing adds the region to a free list. When additional memory
// is allocated, the free list is searched first. If there is not a suitable
// region, the heap is grown and the allocation is made at its end. Resetting
// the heap frees every allocation at once.
#include <stdint.h>
#include <stdlib.h>
@ -15,7 +14,7 @@ extern void tree_sitter_debug_message(const char *, size_t);
#define PAGESIZE 0x10000
#define MAX_HEAP_SIZE (4 * 1024 * 1024)
typedef struct {
typedef struct Region {
size_t size;
struct Region *next;
char data[0];

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,65 @@
#ifndef BULK_MEMORY_THRESHOLD
#define BULK_MEMORY_THRESHOLD 32
#endif
#include "./libc/string/memchr.c"
#undef ALIGN
#undef HASZERO
#undef HIGHS
#undef ONES
#undef SS
#include "./libc/string/memcmp.c"
#include "./libc/string/memcpy.c"
#undef LS
#undef RS
#include "./libc/string/memmove.c"
#undef WS
#include "./libc/string/memset.c"
#include "./libc/string/strchrnul.c"
#undef ALIGN
#undef HASZERO
#undef HIGHS
#undef ONES
#include "./libc/string/strchr.c"
#include "./libc/string/strcmp.c"
#include "./libc/string/strlen.c"
#undef ALIGN
#undef HASZERO
#undef HIGHS
#undef ONES
#include "./libc/string/strncat.c"
#include "./libc/string/strncmp.c"
#include "./libc/string/stpncpy.c"
#undef ALIGN
#undef HASZERO
#undef HIGHS
#undef ONES
#include "./libc/string/strncpy.c"
#include "./libc/string/wcschr.c"
#include "./libc/string/wcslen.c"
#include "./libc/ctype/isblank.c"
#include "./libc/ctype/iswalnum.c"
#define table tree_sitter_iswalpha_table
#include "./libc/ctype/iswalpha.c"
#undef table
#include "./libc/ctype/iswblank.c"
#include "./libc/ctype/iswdigit.c"
#include "./libc/ctype/iswlower.c"
#define table tree_sitter_iswpunct_table
#include "./libc/ctype/iswpunct.c"
#undef table
#include "./libc/ctype/iswspace.c"
#include "./libc/ctype/iswupper.c"
#include "./libc/ctype/iswxdigit.c"
#include "./libc/ctype/towctrans.c"

View file

@ -0,0 +1,193 @@
musl as a whole is licensed under the following standard MIT license:
----------------------------------------------------------------------
Copyright © 2005-2020 Rich Felker, et al.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
----------------------------------------------------------------------
Authors/contributors include:
A. Wilcox
Ada Worcester
Alex Dowad
Alex Suykov
Alexander Monakov
Andre McCurdy
Andrew Kelley
Anthony G. Basile
Aric Belsito
Arvid Picciani
Bartosz Brachaczek
Benjamin Peterson
Bobby Bingham
Boris Brezillon
Brent Cook
Chris Spiegel
Clément Vasseur
Daniel Micay
Daniel Sabogal
Daurnimator
David Carlier
David Edelsohn
Denys Vlasenko
Dmitry Ivanov
Dmitry V. Levin
Drew DeVault
Emil Renner Berthing
Fangrui Song
Felix Fietkau
Felix Janda
Gianluca Anzolin
Hauke Mehrtens
He X
Hiltjo Posthuma
Isaac Dunham
Jaydeep Patil
Jens Gustedt
Jeremy Huntwork
Jo-Philipp Wich
Joakim Sindholt
John Spencer
Julien Ramseier
Justin Cormack
Kaarle Ritvanen
Khem Raj
Kylie McClain
Leah Neukirchen
Luca Barbato
Luka Perkov
M Farkas-Dyck (Strake)
Mahesh Bodapati
Markus Wichmann
Masanori Ogino
Michael Clark
Michael Forney
Mikhail Kremnyov
Natanael Copa
Nicholas J. Kain
orc
Pascal Cuoq
Patrick Oppenlander
Petr Hosek
Petr Skocik
Pierre Carrier
Reini Urban
Rich Felker
Richard Pennington
Ryan Fairfax
Samuel Holland
Segev Finer
Shiz
sin
Solar Designer
Stefan Kristiansson
Stefan O'Rear
Szabolcs Nagy
Timo Teräs
Trutz Behn
Valentin Ochs
Will Dietz
William Haddon
William Pitcock
Portions of this software are derived from third-party works licensed
under terms compatible with the above MIT license:
The TRE regular expression implementation (src/regex/reg* and
src/regex/tre*) is Copyright © 2001-2008 Ville Laurikari and licensed
under a 2-clause BSD license (license text in the source files). The
included version has been heavily modified by Rich Felker in 2012, in
the interests of size, simplicity, and namespace cleanliness.
Much of the math library code (src/math/* and src/complex/*) is
Copyright © 1993,2004 Sun Microsystems or
Copyright © 2003-2011 David Schultz or
Copyright © 2003-2009 Steven G. Kargl or
Copyright © 2003-2009 Bruce D. Evans or
Copyright © 2008 Stephen L. Moshier or
Copyright © 2017-2018 Arm Limited
and labelled as such in comments in the individual source files. All
have been licensed under extremely permissive terms.
The ARM memcpy code (src/string/arm/memcpy.S) is Copyright © 2008
The Android Open Source Project and is licensed under a two-clause BSD
license. It was taken from Bionic libc, used on Android.
The AArch64 memcpy and memset code (src/string/aarch64/*) are
Copyright © 1999-2019, Arm Limited.
The implementation of DES for crypt (src/crypt/crypt_des.c) is
Copyright © 1994 David Burren. It is licensed under a BSD license.
The implementation of blowfish crypt (src/crypt/crypt_blowfish.c) was
originally written by Solar Designer and placed into the public
domain. The code also comes with a fallback permissive license for use
in jurisdictions that may not recognize the public domain.
The smoothsort implementation (src/stdlib/qsort.c) is Copyright © 2011
Valentin Ochs and is licensed under an MIT-style license.
The x86_64 port was written by Nicholas J. Kain and is licensed under
the standard MIT terms.
The mips and microblaze ports were originally written by Richard
Pennington for use in the ellcc project. The original code was adapted
by Rich Felker for build system and code conventions during upstream
integration. It is licensed under the standard MIT terms.
The mips64 port was contributed by Imagination Technologies and is
licensed under the standard MIT terms.
The powerpc port was also originally written by Richard Pennington,
and later supplemented and integrated by John Spencer. It is licensed
under the standard MIT terms.
All other files which have no copyright comments are original works
produced specifically for use as part of this library, written either
by Rich Felker, the main author of the library, or by one or more
contibutors listed above. Details on authorship of individual files
can be found in the git version control history of the project. The
omission of copyright and license comments in each file is in the
interest of source tree size.
In addition, permission is hereby granted for all public header files
(include/* and arch/*/bits/*) and crt files intended to be linked into
applications (crt/*, ldso/dlstart.c, and arch/*/crt_arch.h) to omit
the copyright notice and permission notice otherwise required by the
license, and to use these files without any requirement of
attribution. These files include substantial contributions from:
Bobby Bingham
John Spencer
Nicholas J. Kain
Rich Felker
Richard Pennington
Stefan Kristiansson
Szabolcs Nagy
all of whom have explicitly granted such permission.
This file previously contained text expressing a belief that most of
the files covered by the above exception were sufficiently trivial not
to be subject to copyright, resulting in confusion over whether it
negated the permissions granted in the license. In the spirit of
permissive licensing, and of not having licensing issues being an
obstacle to adoption, that text has been removed.

View file

@ -0,0 +1,15 @@
# Vendored WASI libc sources
These files are copied from
[`WebAssembly/wasi-libc`](https://github.com/WebAssembly/wasi-libc) commit
`161b3195fc2558d2b1ba3eb9ffae3b2b47407623`, which is the `wasi-libc`
submodule revision pinned by WASI SDK 33.
The allowlist is maintained by `cargo xtask vendor-wasm-stdlib`. Run that
command after changing `crates/loader/wasi-sdk-version` or the pinned
`WASI_LIBC_REVISION` in `crates/xtask/src/build_wasm.rs`.
The files under `string/` and `ctype/`, as well as `LICENSE`, are copied
verbatim. `lib/src/wasm-stdlib/libc.c` combines the selected translation units
and renames colliding file-local table identifiers; it is maintained by
Tree-sitter.

View file

@ -0,0 +1,172 @@
18,17,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,17,34,35,36,17,37,38,39,40,
41,42,43,44,17,45,46,47,16,16,48,16,16,16,16,16,16,16,49,50,51,16,52,53,16,16,
17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,54,
17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
17,17,17,55,17,17,17,17,56,17,57,58,59,60,61,62,17,17,17,17,17,17,17,17,17,17,
17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
17,17,17,17,17,17,17,63,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,64,65,17,66,67,
68,69,70,71,72,73,74,17,75,76,77,78,79,80,81,16,82,83,84,85,86,87,88,89,90,91,
92,93,16,94,95,96,16,17,17,17,97,98,99,16,16,16,16,16,16,16,16,16,16,17,17,17,
17,100,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,101,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,17,17,102,103,16,16,104,105,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
17,17,17,17,17,17,17,17,17,106,17,17,107,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,
108,109,16,16,16,16,16,16,16,16,16,110,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,111,112,113,114,16,16,16,16,16,16,16,16,115,116,
117,16,16,16,16,16,118,119,16,16,16,16,120,16,16,121,16,16,16,16,16,16,16,16,
16,16,16,16,16,
16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,254,255,255,7,254,
255,255,7,0,0,0,0,0,4,32,4,255,255,127,255,255,255,127,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,195,255,3,0,31,80,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,223,188,64,215,255,255,
251,255,255,255,255,255,255,255,255,255,191,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,3,252,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,254,255,255,255,127,2,255,255,255,
255,255,1,0,0,0,0,255,191,182,0,255,255,255,135,7,0,0,0,255,7,255,255,255,255,
255,255,255,254,255,195,255,255,255,255,255,255,255,255,255,255,255,255,239,
31,254,225,255,
159,0,0,255,255,255,255,255,255,0,224,255,255,255,255,255,255,255,255,255,255,
255,255,3,0,255,255,255,255,255,7,48,4,255,255,255,252,255,31,0,0,255,255,255,
1,255,7,0,0,0,0,0,0,255,255,223,63,0,0,240,255,248,3,255,255,255,255,255,255,
255,255,255,239,255,223,225,255,207,255,254,255,239,159,249,255,255,253,197,
227,159,89,128,176,207,255,3,16,238,135,249,255,255,253,109,195,135,25,2,94,
192,255,63,0,238,191,251,255,255,253,237,227,191,27,1,0,207,255,0,30,238,159,
249,255,255,253,237,227,159,25,192,176,207,255,2,0,236,199,61,214,24,199,255,
195,199,29,129,0,192,255,0,0,239,223,253,255,255,253,255,227,223,29,96,7,207,
255,0,0,239,223,253,255,255,253,239,227,223,29,96,64,207,255,6,0,239,223,253,
255,255,255,255,231,223,93,240,128,207,255,0,252,236,255,127,252,255,255,251,
47,127,128,95,255,192,255,12,0,254,255,255,255,255,127,255,7,63,32,255,3,0,0,
0,0,214,247,255,255,175,255,255,59,95,32,255,243,0,0,0,
0,1,0,0,0,255,3,0,0,255,254,255,255,255,31,254,255,3,255,255,254,255,255,255,
31,0,0,0,0,0,0,0,0,255,255,255,255,255,255,127,249,255,3,255,255,255,255,255,
255,255,255,255,63,255,255,255,255,191,32,255,255,255,255,255,247,255,255,255,
255,255,255,255,255,255,61,127,61,255,255,255,255,255,61,255,255,255,255,61,
127,61,255,127,255,255,255,255,255,255,255,61,255,255,255,255,255,255,255,255,
7,0,0,0,0,255,255,0,0,255,255,255,255,255,255,255,255,255,255,63,63,254,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,159,255,255,254,255,255,7,255,255,255,255,255,255,255,255,
255,199,255,1,255,223,15,0,255,255,15,0,255,255,15,0,255,223,13,0,255,255,255,
255,255,255,207,255,255,1,128,16,255,3,0,0,0,0,255,3,255,255,255,255,255,255,
255,255,255,255,255,1,255,255,255,255,255,7,255,255,255,255,255,255,255,255,
63,
0,255,255,255,127,255,15,255,1,192,255,255,255,255,63,31,0,255,255,255,255,
255,15,255,255,255,3,255,3,0,0,0,0,255,255,255,15,255,255,255,255,255,255,255,
127,254,255,31,0,255,3,255,3,128,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,
255,239,255,239,15,255,3,0,0,0,0,255,255,255,255,255,243,255,255,255,255,255,
255,191,255,3,0,255,255,255,255,255,255,127,0,255,227,255,255,255,255,255,63,
255,1,255,255,255,255,255,231,0,0,0,0,0,222,111,4,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,
128,255,31,0,255,255,63,63,255,255,255,255,63,63,255,170,255,255,255,63,255,
255,255,255,255,255,223,95,220,31,207,15,255,31,220,31,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,2,128,0,0,255,31,0,0,0,0,0,0,0,0,0,0,0,0,132,252,47,62,80,189,255,243,
224,67,0,0,255,255,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,255,255,255,255,255,255,3,0,
0,255,255,255,255,255,127,255,255,255,255,255,127,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,31,120,12,0,255,255,255,255,191,32,255,
255,255,255,255,255,255,128,0,0,255,255,127,0,127,127,127,127,127,127,127,127,
255,255,255,255,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,224,0,0,0,254,3,62,31,254,255,255,255,255,255,255,255,255,255,127,224,254,
255,255,255,255,255,255,255,255,255,255,247,224,255,255,255,255,255,254,255,
255,255,255,255,255,255,255,255,255,127,0,0,255,255,255,7,0,0,0,0,0,0,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,63,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,
0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,31,0,0,
0,0,0,0,0,0,255,255,255,255,255,63,255,31,255,255,255,15,0,0,255,255,255,255,
255,127,240,143,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,
0,128,255,252,255,255,255,255,255,255,255,255,255,255,255,255,249,255,255,255,
255,255,255,124,0,0,0,0,0,128,255,191,255,255,255,255,0,0,0,255,255,255,255,
255,255,15,0,255,255,255,255,255,255,255,255,47,0,255,3,0,0,252,232,255,255,
255,255,255,7,255,255,255,255,7,0,255,255,255,31,255,255,255,255,255,255,247,
255,0,128,255,3,255,255,255,127,255,255,255,255,255,255,127,0,255,63,255,3,
255,255,127,252,255,255,255,255,255,255,255,127,5,0,0,56,255,255,60,0,126,126,
126,0,127,127,255,255,255,255,255,247,255,0,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,7,255,3,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,15,0,255,255,127,248,255,255,255,255,
255,
15,255,255,255,255,255,255,255,255,255,255,255,255,255,63,255,255,255,255,255,
255,255,255,255,255,255,255,255,3,0,0,0,0,127,0,248,224,255,253,127,95,219,
255,255,255,255,255,255,255,255,255,255,255,255,255,3,0,0,0,248,255,255,255,
255,255,255,255,255,255,255,255,255,63,0,0,255,255,255,255,255,255,255,255,
252,255,255,255,255,255,255,0,0,0,0,0,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,223,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,31,0,0,255,3,
254,255,255,7,254,255,255,7,192,255,255,255,255,255,255,255,255,255,255,127,
252,252,252,28,0,0,0,0,255,239,255,255,127,255,255,183,255,63,255,63,0,0,0,0,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,7,0,0,0,0,0,0,0,0,
255,255,255,255,255,255,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,255,255,255,31,255,255,255,255,255,255,1,0,0,0,0,
0,255,255,255,255,0,224,255,255,255,7,255,255,255,255,255,7,255,255,255,63,
255,255,255,255,15,255,62,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,63,255,3,255,255,255,255,15,255,255,255,
255,15,255,255,255,255,255,0,255,255,255,255,255,255,15,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,255,255,255,255,255,255,127,0,255,255,63,0,255,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,63,253,255,255,255,255,191,145,255,255,63,0,255,255,
127,0,255,255,255,127,0,0,0,0,0,0,0,0,255,255,55,0,255,255,63,0,255,255,255,3,
0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,192,0,0,0,0,0,0,0,0,111,240,239,
254,255,255,63,0,0,0,0,0,255,255,255,31,255,255,255,31,0,0,0,0,255,254,255,
255,31,0,0,0,255,255,255,255,255,255,63,0,255,255,63,0,255,255,7,0,255,255,3,
0,0,0,0,0,0,0,0,0,0,0,0,
0,255,255,255,255,255,255,255,255,255,1,0,0,0,0,0,0,255,255,255,255,255,255,7,
0,255,255,255,255,255,255,7,0,255,255,255,255,255,0,255,3,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,31,128,0,255,255,63,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,255,255,127,0,255,255,255,255,255,255,255,255,63,0,0,0,
192,255,0,0,252,255,255,255,255,255,255,1,0,0,255,255,255,1,255,3,255,255,255,
255,255,255,199,255,112,0,255,255,255,255,71,0,255,255,255,255,255,255,255,
255,30,0,255,23,0,0,0,0,255,255,251,255,255,255,159,64,0,0,0,0,0,0,0,0,127,
189,255,191,255,1,255,255,255,255,255,255,255,1,255,3,239,159,249,255,255,253,
237,227,159,25,129,224,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,
255,255,255,255,255,187,7,255,131,0,0,0,0,255,255,255,255,255,255,255,255,179,
0,255,3,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,63,127,0,0,0,63,0,0,
0,0,255,255,255,255,255,255,255,127,17,0,255,3,0,0,0,0,255,255,255,255,255,
255,63,1,255,3,0,0,0,0,0,0,255,255,255,231,255,7,255,3,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0,
0,255,255,255,255,255,255,255,255,255,3,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,255,252,255,255,255,255,255,252,26,0,0,0,255,255,255,255,255,255,231,
127,0,0,255,255,255,255,255,255,255,255,255,32,0,0,0,0,255,255,255,255,255,
255,255,1,255,253,255,255,255,255,127,127,1,0,255,3,0,0,252,255,255,255,252,
255,255,254,127,0,0,0,0,0,0,0,0,0,127,251,255,255,255,255,127,180,203,0,255,3,
191,253,255,255,255,127,123,1,255,3,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,127,0,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,3,0,0,
0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,127,0,
0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,
255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,
255,255,255,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,
255,255,255,255,255,255,1,255,255,255,127,255,3,0,0,0,0,0,0,0,0,0,0,0,0,255,
255,255,63,0,0,255,255,255,255,255,255,0,0,15,0,255,3,248,255,255,224,255,255,
0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,255,255,255,255,255,255,255,255,255,135,255,255,255,255,255,255,255,128,
255,255,0,0,0,0,0,0,0,0,11,0,0,0,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,7,0,255,255,255,127,0,0,0,0,0,
0,7,0,240,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,15,255,255,255,255,
255,255,255,255,255,255,255,255,255,7,255,31,255,1,255,67,0,0,0,0,0,0,0,0,0,0,
0,0,255,255,255,255,255,255,255,255,255,255,223,255,255,255,255,255,255,255,
255,223,100,222,255,235,239,255,255,255,255,255,255,
255,191,231,223,223,255,255,255,123,95,252,253,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,63,255,255,255,
253,255,255,247,255,255,255,247,255,255,223,255,255,255,223,255,255,127,255,
255,255,127,255,255,255,253,255,255,255,253,255,255,247,207,255,255,255,255,
255,255,127,255,255,249,219,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,255,255,255,255,255,31,128,63,255,67,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,
15,255,3,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,31,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,
143,8,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,239,255,255,255,150,254,247,10,132,234,150,170,150,247,247,94,255,251,255,
15,238,251,255,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,3,255,255,255,3,255,
255,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

View file

@ -0,0 +1,297 @@
static const unsigned char tab[] = {
7, 8, 9, 10, 11, 12, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
13, 6, 6, 14, 6, 6, 6, 6, 6, 6, 6, 6, 15, 16, 17, 18,
6, 19, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 20, 21, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 22, 23, 6, 6, 6, 24, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 25,
6, 6, 6, 6, 26, 6, 6, 6, 6, 6, 6, 6, 27, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 28, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 29, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 30, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36,
43, 43, 43, 43, 43, 43, 43, 43, 1, 0, 84, 86, 86, 86, 86, 86,
86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 43, 43, 43, 43, 43, 43,
43, 7, 43, 43, 91, 86, 86, 86, 86, 86, 86, 86, 74, 86, 86, 5,
49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80,
36, 80, 121, 49, 80, 49, 80, 49, 56, 80, 49, 80, 49, 80, 49, 80,
49, 80, 49, 80, 49, 80, 49, 80, 78, 49, 2, 78, 13, 13, 78, 3,
78, 0, 36, 110, 0, 78, 49, 38, 110, 81, 78, 36, 80, 78, 57, 20,
129, 27, 29, 29, 83, 49, 80, 49, 80, 13, 49, 80, 49, 80, 49, 80,
27, 83, 36, 80, 49, 2, 92, 123, 92, 123, 92, 123, 92, 123, 92, 123,
20, 121, 92, 123, 92, 123, 92, 45, 43, 73, 3, 72, 3, 120, 92, 123,
20, 0, 150, 10, 1, 43, 40, 6, 6, 0, 42, 6, 42, 42, 43, 7,
187, 181, 43, 30, 0, 43, 7, 43, 43, 43, 1, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 1, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 205, 70, 205, 43, 0, 37, 43, 7, 1, 6, 1, 85, 86, 86, 86,
86, 86, 85, 86, 86, 2, 36, 129, 129, 129, 129, 129, 21, 129, 129, 129,
0, 0, 43, 0, 178, 209, 178, 209, 178, 209, 178, 209, 0, 0, 205, 204,
1, 0, 215, 215, 215, 215, 215, 131, 129, 129, 129, 129, 129, 129, 129, 129,
129, 129, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 28, 0, 0, 0,
0, 0, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 2, 0, 0,
49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80,
49, 80, 78, 49, 80, 49, 80, 78, 49, 80, 49, 80, 49, 80, 49, 80,
49, 80, 49, 80, 49, 80, 49, 2, 135, 166, 135, 166, 135, 166, 135, 166,
135, 166, 135, 166, 135, 166, 135, 166, 42, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 0, 0, 0, 84, 86, 86, 86, 86, 86, 86, 86,
86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 84, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
12, 0, 12, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 7, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 86, 86, 108, 129, 21, 0, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 7, 108, 3, 65, 43, 43, 86, 86, 86, 86, 86, 86,
86, 86, 86, 86, 86, 86, 86, 86, 44, 86, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 12, 108, 0, 0, 0, 0, 0, 6,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37,
6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37,
6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37,
6, 37, 6, 37, 6, 37, 6, 37, 86, 122, 158, 38, 6, 37, 6, 37,
6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 37,
6, 37, 6, 37, 6, 37, 6, 37, 6, 37, 6, 1, 43, 43, 79, 86,
86, 44, 43, 127, 86, 86, 57, 43, 43, 85, 86, 86, 43, 43, 79, 86,
86, 44, 43, 127, 86, 86, 129, 55, 117, 91, 123, 92, 43, 43, 79, 86,
86, 2, 172, 4, 0, 0, 57, 43, 43, 85, 86, 86, 43, 43, 79, 86,
86, 44, 43, 43, 86, 86, 50, 19, 129, 87, 0, 111, 129, 126, 201, 215,
126, 45, 129, 129, 14, 126, 57, 127, 111, 87, 0, 129, 129, 126, 21, 0,
126, 3, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 7, 43,
36, 43, 151, 43, 43, 43, 43, 43, 43, 43, 43, 43, 42, 43, 43, 43,
43, 43, 86, 86, 86, 86, 86, 128, 129, 129, 129, 129, 57, 187, 42, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 1, 129, 129, 129, 129, 129, 129, 129, 129,
129, 129, 129, 129, 129, 129, 129, 201, 172, 172, 172, 172, 172, 172, 172, 172,
172, 172, 172, 172, 172, 172, 172, 208, 13, 0, 78, 49, 2, 180, 193, 193,
215, 215, 36, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80,
49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80, 49, 80,
49, 80, 49, 80, 215, 215, 83, 193, 71, 212, 215, 215, 215, 5, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 7, 1, 0, 1, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 49, 80, 49, 80, 49, 80,
49, 80, 49, 80, 49, 80, 49, 80, 13, 0, 0, 0, 0, 0, 36, 80,
49, 80, 49, 80, 49, 80, 49, 80, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 121, 92, 123, 92, 123, 79, 123, 92, 123, 92, 123,
92, 123, 92, 123, 92, 123, 92, 123, 92, 123, 92, 123, 92, 123, 92, 45,
43, 43, 121, 20, 92, 123, 92, 45, 121, 42, 92, 39, 92, 123, 92, 123,
92, 123, 164, 0, 10, 180, 92, 123, 92, 123, 79, 3, 42, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0,
0, 0, 0, 0, 0, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 43, 43, 43, 43, 43, 43, 43, 43, 7, 0, 72, 86, 86, 86, 86,
86, 86, 86, 86, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 85, 86, 86, 86, 86, 86, 86,
86, 86, 86, 86, 86, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 36, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 7, 0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 7, 0, 0,
0, 0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 86, 86, 86, 86, 86, 86, 86, 86,
86, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 86, 86,
86, 86, 86, 86, 86, 86, 86, 86, 14, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 85,
86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 14, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
static const int rules[] = {
0x0, 0x2001, -0x2000, 0x1dbf00, 0x2e700, 0x7900,
0x2402, 0x101, -0x100, 0x0, 0x201, -0x200,
-0xc6ff, -0xe800, -0x78ff, -0x12c00, 0xc300, 0xd201,
0xce01, 0xcd01, 0x4f01, 0xca01, 0xcb01, 0xcf01,
0x6100, 0xd301, 0xd101, 0xa300, 0xd501, 0x8200,
0xd601, 0xda01, 0xd901, 0xdb01, 0x3800, 0x3,
-0x4f00, -0x60ff, -0x37ff, 0x242802, 0x0, 0x101,
-0x100, -0xcd00, -0xda00, -0x81ff, 0x2a2b01, -0xa2ff,
0x2a2801, 0x2a3f00, -0xc2ff, 0x4501, 0x4701, 0x2a1f00,
0x2a1c00, 0x2a1e00, -0xd200, -0xce00, -0xca00, -0xcb00,
0xa54f00, 0xa54b00, -0xcf00, 0xa52800, 0xa54400, -0xd100,
-0xd300, 0x29f700, 0xa54100, 0x29fd00, -0xd500, -0xd600,
0x29e700, 0xa54300, 0xa52a00, -0x4500, -0xd900, -0x4700,
-0xdb00, 0xa51500, 0xa51200, 0x4c2402, 0x0, 0x2001,
-0x2000, 0x101, -0x100, 0x5400, 0x7401, 0x2601,
0x2501, 0x4001, 0x3f01, -0x2600, -0x2500, -0x1f00,
-0x4000, -0x3f00, 0x801, -0x3e00, -0x3900, -0x2f00,
-0x3600, -0x800, -0x5600, -0x5000, 0x700, -0x7400,
-0x3bff, -0x6000, -0x6ff, 0x701a02, 0x101, -0x100,
0x2001, -0x2000, 0x5001, 0xf01, -0xf00, 0x0,
0x3001, -0x3000, 0x101, -0x100, 0x0, 0xbc000,
0x1c6001, 0x0, 0x97d001, 0x801, -0x800, 0x8a0502,
0x0, -0xbbfff, -0x186200, 0x89c200, -0x182500, -0x186e00,
-0x186d00, -0x186400, -0x186300, -0x185c00, 0x0, 0x8a3800,
0x8a0400, 0xee600, 0x101, -0x100, 0x0, -0x3b00,
-0x1dbeff, 0x8f1d02, 0x800, -0x7ff, 0x0, 0x5600,
-0x55ff, 0x4a00, 0x6400, 0x8000, 0x7000, 0x7e00,
0x900, -0x49ff, -0x8ff, -0x1c2500, -0x63ff, -0x6fff,
-0x7fff, -0x7dff, 0xac0502, 0x0, 0x1001, -0x1000,
0x1c01, 0x101, -0x1d5cff, -0x20beff, -0x2045ff, -0x1c00,
0xb10b02, 0x101, -0x100, 0x3001, -0x3000, 0x0,
-0x29f6ff, -0xee5ff, -0x29e6ff, -0x2a2b00, -0x2a2800, -0x2a1bff,
-0x29fcff, -0x2a1eff, -0x2a1dff, -0x2a3eff, 0x0, -0x1c6000,
0x0, 0x101, -0x100, 0xbc0c02, 0x0, 0x101,
-0x100, -0xa543ff, 0x3a001, -0x8a03ff, -0xa527ff, 0x3000,
-0xa54eff, -0xa54aff, -0xa540ff, -0xa511ff, -0xa529ff, -0xa514ff,
-0x2fff, -0xa542ff, -0x8a37ff, 0x0, -0x97d000, -0x3a000,
0x0, 0x2001, -0x2000, 0x0, 0x2801, -0x2800,
0x0, 0x4001, -0x4000, 0x0, 0x2001, -0x2000,
0x0, 0x2001, -0x2000, 0x0, 0x2201, -0x2200,
};
static const unsigned char rulebases[] = {
0, 6, 39, 81, 111, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
124, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 131, 142, 146, 151,
0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 196, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 198, 201, 0, 0, 0, 219, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222,
0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
static const unsigned char exceptions[][2] = {
{ 48, 12 }, { 49, 13 }, { 120, 14 }, { 127, 15 },
{ 128, 16 }, { 129, 17 }, { 134, 18 }, { 137, 19 },
{ 138, 19 }, { 142, 20 }, { 143, 21 }, { 144, 22 },
{ 147, 19 }, { 148, 23 }, { 149, 24 }, { 150, 25 },
{ 151, 26 }, { 154, 27 }, { 156, 25 }, { 157, 28 },
{ 158, 29 }, { 159, 30 }, { 166, 31 }, { 169, 31 },
{ 174, 31 }, { 177, 32 }, { 178, 32 }, { 183, 33 },
{ 191, 34 }, { 197, 35 }, { 200, 35 }, { 203, 35 },
{ 221, 36 }, { 242, 35 }, { 246, 37 }, { 247, 38 },
{ 32, 45 }, { 58, 46 }, { 61, 47 }, { 62, 48 },
{ 63, 49 }, { 64, 49 }, { 67, 50 }, { 68, 51 },
{ 69, 52 }, { 80, 53 }, { 81, 54 }, { 82, 55 },
{ 83, 56 }, { 84, 57 }, { 89, 58 }, { 91, 59 },
{ 92, 60 }, { 97, 61 }, { 99, 62 }, { 101, 63 },
{ 102, 64 }, { 104, 65 }, { 105, 66 }, { 106, 64 },
{ 107, 67 }, { 108, 68 }, { 111, 66 }, { 113, 69 },
{ 114, 70 }, { 117, 71 }, { 125, 72 }, { 130, 73 },
{ 135, 74 }, { 137, 75 }, { 138, 76 }, { 139, 76 },
{ 140, 77 }, { 146, 78 }, { 157, 79 }, { 158, 80 },
{ 69, 87 }, { 123, 29 }, { 124, 29 }, { 125, 29 },
{ 127, 88 }, { 134, 89 }, { 136, 90 }, { 137, 90 },
{ 138, 90 }, { 140, 91 }, { 142, 92 }, { 143, 92 },
{ 172, 93 }, { 173, 94 }, { 174, 94 }, { 175, 94 },
{ 194, 95 }, { 204, 96 }, { 205, 97 }, { 206, 97 },
{ 207, 98 }, { 208, 99 }, { 209, 100 }, { 213, 101 },
{ 214, 102 }, { 215, 103 }, { 240, 104 }, { 241, 105 },
{ 242, 106 }, { 243, 107 }, { 244, 108 }, { 245, 109 },
{ 249, 110 }, { 253, 45 }, { 254, 45 }, { 255, 45 },
{ 80, 105 }, { 81, 105 }, { 82, 105 }, { 83, 105 },
{ 84, 105 }, { 85, 105 }, { 86, 105 }, { 87, 105 },
{ 88, 105 }, { 89, 105 }, { 90, 105 }, { 91, 105 },
{ 92, 105 }, { 93, 105 }, { 94, 105 }, { 95, 105 },
{ 130, 0 }, { 131, 0 }, { 132, 0 }, { 133, 0 },
{ 134, 0 }, { 135, 0 }, { 136, 0 }, { 137, 0 },
{ 192, 117 }, { 207, 118 }, { 128, 137 }, { 129, 138 },
{ 130, 139 }, { 133, 140 }, { 134, 141 }, { 112, 157 },
{ 113, 157 }, { 118, 158 }, { 119, 158 }, { 120, 159 },
{ 121, 159 }, { 122, 160 }, { 123, 160 }, { 124, 161 },
{ 125, 161 }, { 179, 162 }, { 186, 163 }, { 187, 163 },
{ 188, 164 }, { 190, 165 }, { 195, 162 }, { 204, 164 },
{ 218, 166 }, { 219, 166 }, { 229, 106 }, { 234, 167 },
{ 235, 167 }, { 236, 110 }, { 243, 162 }, { 248, 168 },
{ 249, 168 }, { 250, 169 }, { 251, 169 }, { 252, 164 },
{ 38, 176 }, { 42, 177 }, { 43, 178 }, { 78, 179 },
{ 132, 8 }, { 98, 186 }, { 99, 187 }, { 100, 188 },
{ 101, 189 }, { 102, 190 }, { 109, 191 }, { 110, 192 },
{ 111, 193 }, { 112, 194 }, { 126, 195 }, { 127, 195 },
{ 125, 207 }, { 141, 208 }, { 148, 209 }, { 171, 210 },
{ 172, 211 }, { 173, 212 }, { 176, 213 }, { 177, 214 },
{ 178, 215 }, { 196, 216 }, { 197, 217 }, { 198, 218 },
};

View file

@ -0,0 +1,13 @@
#include <ctype.h>
int isblank(int c)
{
return (c == ' ' || c == '\t');
}
int __isblank_l(int c, locale_t l)
{
return isblank(c);
}
weak_alias(__isblank_l, isblank_l);

View file

@ -0,0 +1,13 @@
#include <wctype.h>
int iswalnum(wint_t wc)
{
return iswdigit(wc) || iswalpha(wc);
}
int __iswalnum_l(wint_t c, locale_t l)
{
return iswalnum(c);
}
weak_alias(__iswalnum_l, iswalnum_l);

View file

@ -0,0 +1,21 @@
#include <wctype.h>
static const unsigned char table[] = {
#include "alpha.h"
};
int iswalpha(wint_t wc)
{
if (wc<0x20000U)
return (table[table[wc>>8]*32+((wc&255)>>3)]>>(wc&7))&1;
if (wc<0x2fffeU)
return 1;
return 0;
}
int __iswalpha_l(wint_t c, locale_t l)
{
return iswalpha(c);
}
weak_alias(__iswalpha_l, iswalpha_l);

View file

@ -0,0 +1,14 @@
#include <wctype.h>
#include <ctype.h>
int iswblank(wint_t wc)
{
return isblank(wc);
}
int __iswblank_l(wint_t c, locale_t l)
{
return iswblank(c);
}
weak_alias(__iswblank_l, iswblank_l);

View file

@ -0,0 +1,15 @@
#include <wctype.h>
#undef iswdigit
int iswdigit(wint_t wc)
{
return (unsigned)wc-'0' < 10;
}
int __iswdigit_l(wint_t c, locale_t l)
{
return iswdigit(c);
}
weak_alias(__iswdigit_l, iswdigit_l);

View file

@ -0,0 +1,13 @@
#include <wctype.h>
int iswlower(wint_t wc)
{
return towupper(wc) != wc;
}
int __iswlower_l(wint_t c, locale_t l)
{
return iswlower(c);
}
weak_alias(__iswlower_l, iswlower_l);

View file

@ -0,0 +1,19 @@
#include <wctype.h>
static const unsigned char table[] = {
#include "punct.h"
};
int iswpunct(wint_t wc)
{
if (wc<0x20000U)
return (table[table[wc>>8]*32+((wc&255)>>3)]>>(wc&7))&1;
return 0;
}
int __iswpunct_l(wint_t c, locale_t l)
{
return iswpunct(c);
}
weak_alias(__iswpunct_l, iswpunct_l);

View file

@ -0,0 +1,24 @@
#include <wchar.h>
#include <wctype.h>
/* Our definition of whitespace is the Unicode White_Space property,
* minus non-breaking spaces (U+00A0, U+2007, and U+202F) and script-
* specific characters with non-blank glyphs (U+1680 and U+180E). */
int iswspace(wint_t wc)
{
static const wchar_t spaces[] = {
' ', '\t', '\n', '\r', 11, 12, 0x0085,
0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005,
0x2006, 0x2008, 0x2009, 0x200a,
0x2028, 0x2029, 0x205f, 0x3000, 0
};
return wc && wcschr(spaces, wc);
}
int __iswspace_l(wint_t c, locale_t l)
{
return iswspace(c);
}
weak_alias(__iswspace_l, iswspace_l);

View file

@ -0,0 +1,13 @@
#include <wctype.h>
int iswupper(wint_t wc)
{
return towlower(wc) != wc;
}
int __iswupper_l(wint_t c, locale_t l)
{
return iswupper(c);
}
weak_alias(__iswupper_l, iswupper_l);

View file

@ -0,0 +1,13 @@
#include <wctype.h>
int iswxdigit(wint_t wc)
{
return (unsigned)(wc-'0') < 10 || (unsigned)((wc|32)-'a') < 6;
}
int __iswxdigit_l(wint_t c, locale_t l)
{
return iswxdigit(c);
}
weak_alias(__iswxdigit_l, iswxdigit_l);

View file

@ -0,0 +1,141 @@
18,16,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,16,16,34,35,16,36,37,38,39,
40,41,42,43,16,44,45,46,17,17,47,17,17,17,17,17,17,48,49,50,51,52,53,54,55,17,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,56,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,57,16,58,59,60,61,62,63,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,64,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,65,16,16,66,16,67,68,
69,16,70,71,72,16,73,16,16,74,75,76,77,78,16,79,80,81,82,83,84,85,86,87,88,89,
90,91,16,92,93,94,95,16,16,16,16,96,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,97,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,98,99,16,16,100,101,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,16,16,16,16,16,102,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,
16,16,16,103,104,105,106,16,16,107,108,17,17,109,16,16,16,16,16,16,110,111,16,
16,16,16,16,112,113,16,16,114,115,116,16,117,118,119,17,17,17,120,121,122,123,
124,16,16,16,16,
16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,254,255,0,252,1,0,0,248,1,
0,0,120,0,0,0,0,255,251,223,251,0,0,128,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,60,0,252,255,224,175,255,255,255,255,255,255,255,255,
255,255,223,255,255,255,255,255,32,64,176,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,252,0,0,0,0,0,230,254,255,255,255,0,64,73,0,0,0,0,0,24,0,255,255,0,216,
0,0,0,0,0,0,0,1,0,60,0,0,0,0,0,0,0,0,0,0,0,0,16,224,1,30,0,
96,255,191,0,0,0,0,0,0,255,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,248,207,
227,0,0,0,3,0,32,255,127,0,0,0,78,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,7,252,0,0,0,
0,0,0,0,0,0,16,0,32,30,0,48,0,1,0,0,0,0,0,0,0,0,16,0,32,0,0,0,0,252,111,0,0,0,
0,0,0,0,16,0,32,0,0,0,0,64,0,0,0,0,0,0,0,0,16,0,32,0,0,0,0,3,224,0,0,0,0,0,0,
0,16,0,32,0,0,0,0,253,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,255,7,16,0,0,0,0,0,0,0,0,
32,0,0,0,0,128,255,16,0,0,0,0,0,0,16,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,24,0,160,
0,127,0,0,255,3,0,0,0,0,0,0,0,0,0,4,0,0,0,0,16,0,0,0,0,0,0,128,0,128,192,223,
0,12,0,0,0,0,0,0,0,0,0,0,0,4,0,31,0,0,0,0,0,
0,254,255,255,255,0,252,255,255,0,0,0,0,0,0,0,0,252,0,0,0,0,0,0,192,255,223,
255,7,0,0,0,0,0,0,0,0,0,0,128,6,0,252,0,0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,0,
0,0,8,0,0,0,0,0,0,0,0,0,0,0,224,255,255,255,31,0,0,255,3,0,0,0,0,0,0,0,0,0,0,
0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,96,0,0,1,0,0,24,0,0,0,0,0,0,0,0,0,56,0,0,0,0,16,0,0,0,112,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,48,0,0,254,127,47,0,0,255,3,255,127,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,49,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,196,255,255,255,
255,0,0,0,192,0,0,0,0,0,0,0,0,1,0,224,159,0,0,0,0,127,63,255,127,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,16,0,16,0,0,252,255,255,255,31,0,0,0,0,0,12,0,0,0,0,0,0,64,0,
12,240,0,0,0,0,0,0,128,248,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,255,0,255,255,
255,33,144,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,
127,0,224,251,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,160,3,224,0,224,0,
224,0,96,128,248,255,255,255,252,255,255,255,255,255,127,223,255,241,127,255,
127,0,0,255,255,255,255,0,0,255,255,255,255,1,0,123,3,208,193,175,66,0,12,31,
188,255,255,0,0,0,0,0,14,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,127,0,0,0,255,7,0,0,255,255,255,255,255,255,255,255,255,
255,63,0,0,0,0,0,0,252,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,207,255,255,255,
63,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,224,135,3,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,127,255,255,255,255,0,
0,0,0,0,0,255,255,255,251,255,255,255,255,255,255,255,255,255,255,15,0,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,63,0,0,0,255,15,30,255,255,255,1,252,193,224,0,0,0,0,
0,0,0,0,0,0,0,30,1,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
255,255,0,0,0,0,255,255,255,255,15,0,0,0,255,255,255,127,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,
255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,
255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,127,0,0,0,
0,0,0,192,0,224,0,0,0,0,0,0,0,0,0,0,0,128,15,112,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
255,0,255,255,127,0,3,0,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
64,0,0,0,0,15,255,3,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,16,192,0,0,255,255,3,23,
0,0,0,0,0,248,0,0,0,0,8,128,0,0,0,0,0,0,0,0,0,0,8,0,255,63,0,192,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,240,0,0,128,3,0,0,0,0,0,0,0,128,2,0,0,192,0,0,67,0,0,0,0,0,
0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,0,
0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,2,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,252,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,255,255,255,3,255,255,255,255,255,255,247,
255,127,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,254,255,0,252,1,0,0,248,1,0,
0,248,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127,127,0,48,135,255,255,255,255,255,
143,255,0,0,0,0,0,0,224,255,255,127,255,15,1,0,0,0,0,0,255,255,255,255,255,63,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,
15,0,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
128,255,0,0,128,255,0,0,0,0,128,255,0,0,0,0,0,0,0,0,0,248,0,0,192,143,0,0,0,
128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,255,255,252,255,255,255,255,255,0,0,0,0,
0,0,0,135,255,1,255,1,0,0,0,224,0,0,0,224,0,0,0,0,0,1,0,0,96,248,127,0,0,0,0,
0,0,0,0,254,0,0,0,255,0,0,0,255,0,0,0,30,0,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,0,0,0,0,0,0,0,0,0,0,0,
0,255,255,255,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,224,127,0,0,0,192,255,255,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,192,63,252,255,63,0,0,128,3,0,0,0,0,0,0,254,3,32,0,0,0,0,0,0,0,
0,0,0,0,0,24,0,15,0,0,0,0,0,56,0,0,0,0,0,0,0,0,0,225,63,0,232,254,255,31,0,0,
0,0,0,0,0,96,63,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0,0,
24,0,32,0,0,192,31,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,
248,0,104,0,0,0,0,0,0,0,0,0,0,0,0,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,128,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,128,14,0,0,0,255,
31,0,0,0,0,0,0,0,0,192,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,8,0,252,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,252,7,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,24,128,255,0,0,0,0,0,
0,0,0,0,0,223,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,62,0,0,252,255,31,3,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52,0,0,0,0,0,0,0,0,0,128,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,128,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,
255,3,
128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,192,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,0,0,0,0,0,0,0,255,255,48,0,0,248,
3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,
255,255,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,176,15,0,0,0,0,0,0,
0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,63,
0,255,255,255,255,127,254,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,255,1,0,0,255,255,255,255,255,255,255,255,
63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,15,0,255,255,255,255,255,255,
255,255,255,255,127,0,255,255,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,8,0,0,0,8,0,0,32,0,0,0,32,0,0,128,
0,0,0,128,0,0,0,2,0,0,0,2,0,0,8,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,255,255,15,0,248,254,255,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,127,0,0,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,
128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,127,0,0,0,0,0,0,0,
0,0,0,0,0,0,112,7,0,192,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,254,255,255,255,255,255,255,255,31,0,0,0,0,0,0,0,0,0,254,255,
255,255,255,255,255,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,255,255,255,255,255,
15,255,255,255,255,255,255,255,255,255,255,255,255,15,0,255,127,254,255,254,
255,254,255,255,255,63,0,255,31,255,255,255,255,0,0,0,252,0,0,0,28,0,0,0,252,
255,255,255,31,0,0,0,0,0,0,192,255,255,255,7,0,255,255,255,255,255,15,255,1,3,
0,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,255,63,0,255,31,255,7,255,255,255,255,255,255,255,255,
255,255,255,255,255,255,15,0,255,255,255,255,255,255,255,255,255,255,255,1,
255,15,0,0,255,15,255,255,255,255,255,255,255,0,255,3,255,255,255,255,255,0,
255,255,255,63,0,0,0,0,0,0,0,0,0,0,255,239,255,255,255,255,255,255,255,255,
255,255,255,255,123,252,255,255,255,255,231,199,255,255,255,231,255,255,255,
255,255,255,255,255,255,255,255,255,255,255,255,255,15,0,255,63,15,7,7,0,63,0,
0,0,0,0,0,0,0,0,0,0,0,0,

View file

@ -0,0 +1,84 @@
#include <wctype.h>
static const unsigned char tab[];
static const unsigned char rulebases[512];
static const int rules[];
static const unsigned char exceptions[][2];
#include "casemap.h"
static int casemap(unsigned c, int dir)
{
unsigned b, x, y, v, rt, xb, xn;
int r, rd, c0 = c;
if (c >= 0x20000) return c;
b = c>>8;
c &= 255;
x = c/3;
y = c%3;
/* lookup entry in two-level base-6 table */
v = tab[tab[b]*86+x];
static const int mt[] = { 2048, 342, 57 };
v = (v*mt[y]>>11)%6;
/* use the bit vector out of the tables as an index into
* a block-specific set of rules and decode the rule into
* a type and a case-mapping delta. */
r = rules[rulebases[b]+v];
rt = r & 255;
rd = r >> 8;
/* rules 0/1 are simple lower/upper case with a delta.
* apply according to desired mapping direction. */
if (rt < 2) return c0 + (rd & -(rt^dir));
/* binary search. endpoints of the binary search for
* this block are stored in the rule delta field. */
xn = rd & 0xff;
xb = (unsigned)rd >> 8;
while (xn) {
unsigned try = exceptions[xb+xn/2][0];
if (try == c) {
r = rules[exceptions[xb+xn/2][1]];
rt = r & 255;
rd = r >> 8;
if (rt < 2) return c0 + (rd & -(rt^dir));
/* Hard-coded for the four exceptional titlecase */
return c0 + (dir ? -1 : 1);
} else if (try > c) {
xn /= 2;
} else {
xb += xn/2;
xn -= xn/2;
}
}
return c0;
}
wint_t towlower(wint_t wc)
{
return casemap(wc, 0);
}
wint_t towupper(wint_t wc)
{
return casemap(wc, 1);
}
wint_t __towupper_l(wint_t c, locale_t l)
{
return towupper(c);
}
wint_t __towlower_l(wint_t c, locale_t l)
{
return towlower(c);
}
weak_alias(__towupper_l, towupper_l);
weak_alias(__towlower_l, towlower_l);

View file

@ -0,0 +1,89 @@
#include <string.h>
#include <stdint.h>
#include <limits.h>
#ifdef __wasm_simd128__
#include <wasm_simd128.h>
#endif
#define SS (sizeof(size_t))
#define ALIGN (sizeof(size_t)-1)
#define ONES ((size_t)-1/UCHAR_MAX)
#define HIGHS (ONES * (UCHAR_MAX/2+1))
#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
void *memchr(const void *src, int c, size_t n)
{
#if defined(__wasm_simd128__) && defined(__wasilibc_simd_string)
// Skip Clang 19 and Clang 20 which have a bug (llvm/llvm-project#146574)
// which results in an ICE when inline assembly is used with a vector result.
#if __clang_major__ != 19 && __clang_major__ != 20
// When n is zero, a function that locates a character finds no occurrence.
// Otherwise, decrement n to ensure sub_overflow overflows
// when n would go equal-to-or-below zero.
if (!n--) {
return NULL;
}
// Note that reading before/after the allocation of a pointer is UB in
// C, so inline assembly is used to generate the exact machine
// instruction we want with opaque semantics to the compiler to avoid
// the UB.
uintptr_t align = (uintptr_t)src % sizeof(v128_t);
uintptr_t addr = (uintptr_t)src - align;
v128_t vc = wasm_i8x16_splat(c);
for (;;) {
v128_t v;
__asm__ (
"local.get %1\n"
"v128.load 0\n"
"local.set %0\n"
: "=r"(v)
: "r"(addr)
: "memory");
v128_t cmp = wasm_i8x16_eq(v, vc);
// Bitmask is slow on AArch64, any_true is much faster.
if (wasm_v128_any_true(cmp)) {
// Clear the bits corresponding to align (little-endian)
// so we can count trailing zeros.
int mask = wasm_i8x16_bitmask(cmp) >> align << align;
// At least one bit will be set, unless align cleared them.
// Knowing this helps the compiler if it unrolls the loop.
__builtin_assume(mask || align);
// If the mask became zero because of align,
// it's as if we didn't find anything.
if (mask) {
// Find the offset of the first one bit (little-endian).
// That's a match, unless it is beyond the end of the object.
// Recall that we decremented n, so less-than-or-equal-to is correct.
size_t ctz = __builtin_ctz(mask);
return ctz - align <= n ? (char *)src + (addr + ctz - (uintptr_t)src)
: NULL;
}
}
// Decrement n; if it overflows we're done.
if (__builtin_sub_overflow(n, sizeof(v128_t) - align, &n)) {
return NULL;
}
align = 0;
addr += sizeof(v128_t);
}
#endif
#endif
const unsigned char *s = src;
c = (unsigned char)c;
#ifdef __GNUC__
for (; ((uintptr_t)s & ALIGN) && n && *s != c; s++, n--);
if (n && *s != c) {
typedef size_t __attribute__((__may_alias__)) word;
const word *w;
size_t k = ONES * c;
for (w = (const void *)s; n>=SS && !HASZERO(*w^k); w++, n-=SS);
s = (const void *)w;
}
#endif
for (; n && *s != c; s++, n--);
return n ? (void *)s : 0;
}

View file

@ -0,0 +1,43 @@
#include <string.h>
#ifdef __wasm_simd128__
#include <wasm_simd128.h>
#endif
int memcmp(const void *vl, const void *vr, size_t n)
{
#if defined(__wasm_simd128__) && defined(__wasilibc_simd_string)
if (n >= sizeof(v128_t)) {
// memcmp is allowed to read up to n bytes from each object.
// Find the first different character in the objects.
// Unaligned loads handle the case where the objects
// have mismatching alignments.
const v128_t *v1 = (v128_t *)vl;
const v128_t *v2 = (v128_t *)vr;
while (n) {
const v128_t cmp = wasm_i8x16_eq(wasm_v128_load(v1), wasm_v128_load(v2));
// Bitmask is slow on AArch64, all_true is much faster.
if (!wasm_i8x16_all_true(cmp)) {
// Find the offset of the first zero bit (little-endian).
size_t ctz = __builtin_ctz(~wasm_i8x16_bitmask(cmp));
const unsigned char *u1 = (unsigned char *)v1 + ctz;
const unsigned char *u2 = (unsigned char *)v2 + ctz;
// This may help the compiler if the function is inlined.
__builtin_assume(*u1 - *u2 != 0);
return *u1 - *u2;
}
// This makes n a multiple of sizeof(v128_t)
// for every iteration except the first.
size_t align = (n - 1) % sizeof(v128_t) + 1;
v1 = (v128_t *)((char *)v1 + align);
v2 = (v128_t *)((char *)v2 + align);
n -= align;
}
return 0;
}
#endif
const unsigned char *l=vl, *r=vr;
for (; n && *l == *r; n--, l++, r++);
return n ? *l-*r : 0;
}

View file

@ -0,0 +1,128 @@
#include <string.h>
#include <stdint.h>
#include <endian.h>
void *memcpy(void *restrict dest, const void *restrict src, size_t n)
{
#if defined(__wasm_bulk_memory__)
if (n > BULK_MEMORY_THRESHOLD)
return __builtin_memcpy(dest, src, n);
#endif
unsigned char *d = dest;
const unsigned char *s = src;
#ifdef __GNUC__
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define LS >>
#define RS <<
#else
#define LS <<
#define RS >>
#endif
typedef uint32_t __attribute__((__may_alias__)) u32;
uint32_t w, x;
for (; (uintptr_t)s % 4 && n; n--) *d++ = *s++;
if ((uintptr_t)d % 4 == 0) {
for (; n>=16; s+=16, d+=16, n-=16) {
*(u32 *)(d+0) = *(u32 *)(s+0);
*(u32 *)(d+4) = *(u32 *)(s+4);
*(u32 *)(d+8) = *(u32 *)(s+8);
*(u32 *)(d+12) = *(u32 *)(s+12);
}
if (n&8) {
*(u32 *)(d+0) = *(u32 *)(s+0);
*(u32 *)(d+4) = *(u32 *)(s+4);
d += 8; s += 8;
}
if (n&4) {
*(u32 *)(d+0) = *(u32 *)(s+0);
d += 4; s += 4;
}
if (n&2) {
*d++ = *s++; *d++ = *s++;
}
if (n&1) {
*d = *s;
}
return dest;
}
if (n >= 32) switch ((uintptr_t)d % 4) {
case 1:
w = *(u32 *)s;
*d++ = *s++;
*d++ = *s++;
*d++ = *s++;
n -= 3;
for (; n>=17; s+=16, d+=16, n-=16) {
x = *(u32 *)(s+1);
*(u32 *)(d+0) = (w LS 24) | (x RS 8);
w = *(u32 *)(s+5);
*(u32 *)(d+4) = (x LS 24) | (w RS 8);
x = *(u32 *)(s+9);
*(u32 *)(d+8) = (w LS 24) | (x RS 8);
w = *(u32 *)(s+13);
*(u32 *)(d+12) = (x LS 24) | (w RS 8);
}
break;
case 2:
w = *(u32 *)s;
*d++ = *s++;
*d++ = *s++;
n -= 2;
for (; n>=18; s+=16, d+=16, n-=16) {
x = *(u32 *)(s+2);
*(u32 *)(d+0) = (w LS 16) | (x RS 16);
w = *(u32 *)(s+6);
*(u32 *)(d+4) = (x LS 16) | (w RS 16);
x = *(u32 *)(s+10);
*(u32 *)(d+8) = (w LS 16) | (x RS 16);
w = *(u32 *)(s+14);
*(u32 *)(d+12) = (x LS 16) | (w RS 16);
}
break;
case 3:
w = *(u32 *)s;
*d++ = *s++;
n -= 1;
for (; n>=19; s+=16, d+=16, n-=16) {
x = *(u32 *)(s+3);
*(u32 *)(d+0) = (w LS 8) | (x RS 24);
w = *(u32 *)(s+7);
*(u32 *)(d+4) = (x LS 8) | (w RS 24);
x = *(u32 *)(s+11);
*(u32 *)(d+8) = (w LS 8) | (x RS 24);
w = *(u32 *)(s+15);
*(u32 *)(d+12) = (x LS 8) | (w RS 24);
}
break;
}
if (n&16) {
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
}
if (n&8) {
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
}
if (n&4) {
*d++ = *s++; *d++ = *s++; *d++ = *s++; *d++ = *s++;
}
if (n&2) {
*d++ = *s++; *d++ = *s++;
}
if (n&1) {
*d = *s;
}
return dest;
#endif
for (; n; n--) *d++ = *s++;
return dest;
}

View file

@ -0,0 +1,46 @@
#include <string.h>
#include <stdint.h>
#ifdef __GNUC__
typedef __attribute__((__may_alias__)) size_t WT;
#define WS (sizeof(WT))
#endif
void *memmove(void *dest, const void *src, size_t n)
{
#if defined(__wasm_bulk_memory__)
if (n > BULK_MEMORY_THRESHOLD)
return __builtin_memmove(dest, src, n);
#endif
char *d = dest;
const char *s = src;
if (d==s) return d;
if ((uintptr_t)s-(uintptr_t)d-n <= -2*n) return memcpy(d, s, n);
if (d<s) {
#ifdef __GNUC__
if ((uintptr_t)s % WS == (uintptr_t)d % WS) {
while ((uintptr_t)d % WS) {
if (!n--) return dest;
*d++ = *s++;
}
for (; n>=WS; n-=WS, d+=WS, s+=WS) *(WT *)d = *(WT *)s;
}
#endif
for (; n; n--) *d++ = *s++;
} else {
#ifdef __GNUC__
if ((uintptr_t)s % WS == (uintptr_t)d % WS) {
while ((uintptr_t)(d+n) % WS) {
if (!n--) return dest;
d[n] = s[n];
}
while (n>=WS) n-=WS, *(WT *)(d+n) = *(WT *)(s+n);
}
#endif
while (n) n--, d[n] = s[n];
}
return dest;
}

View file

@ -0,0 +1,94 @@
#include <string.h>
#include <stdint.h>
void *memset(void *dest, int c, size_t n)
{
#if defined(__wasm_bulk_memory__)
if (n > BULK_MEMORY_THRESHOLD)
return __builtin_memset(dest, c, n);
#endif
unsigned char *s = dest;
size_t k;
/* Fill head and tail with minimal branching. Each
* conditional ensures that all the subsequently used
* offsets are well-defined and in the dest region. */
if (!n) return dest;
s[0] = c;
s[n-1] = c;
if (n <= 2) return dest;
s[1] = c;
s[2] = c;
s[n-2] = c;
s[n-3] = c;
if (n <= 6) return dest;
s[3] = c;
s[n-4] = c;
if (n <= 8) return dest;
/* Advance pointer to align it at a 4-byte boundary,
* and truncate n to a multiple of 4. The previous code
* already took care of any head/tail that get cut off
* by the alignment. */
k = -(uintptr_t)s & 3;
s += k;
n -= k;
n &= -4;
#ifdef __GNUC__
typedef uint32_t __attribute__((__may_alias__)) u32;
typedef uint64_t __attribute__((__may_alias__)) u64;
u32 c32 = ((u32)-1)/255 * (unsigned char)c;
/* In preparation to copy 32 bytes at a time, aligned on
* an 8-byte bounary, fill head/tail up to 28 bytes each.
* As in the initial byte-based head/tail fill, each
* conditional below ensures that the subsequent offsets
* are valid (e.g. !(n<=24) implies n>=28). */
*(u32 *)(s+0) = c32;
*(u32 *)(s+n-4) = c32;
if (n <= 8) return dest;
*(u32 *)(s+4) = c32;
*(u32 *)(s+8) = c32;
*(u32 *)(s+n-12) = c32;
*(u32 *)(s+n-8) = c32;
if (n <= 24) return dest;
*(u32 *)(s+12) = c32;
*(u32 *)(s+16) = c32;
*(u32 *)(s+20) = c32;
*(u32 *)(s+24) = c32;
*(u32 *)(s+n-28) = c32;
*(u32 *)(s+n-24) = c32;
*(u32 *)(s+n-20) = c32;
*(u32 *)(s+n-16) = c32;
/* Align to a multiple of 8 so we can fill 64 bits at a time,
* and avoid writing the same bytes twice as much as is
* practical without introducing additional branching. */
k = 24 + ((uintptr_t)s & 4);
s += k;
n -= k;
/* If this loop is reached, 28 tail bytes have already been
* filled, so any remainder when n drops below 32 can be
* safely ignored. */
u64 c64 = c32 | ((u64)c32 << 32);
for (; n >= 32; n-=32, s+=32) {
*(u64 *)(s+0) = c64;
*(u64 *)(s+8) = c64;
*(u64 *)(s+16) = c64;
*(u64 *)(s+24) = c64;
}
#else
/* Pure C fallback with no aliasing violations. */
for (; n; n--, s++) *s = c;
#endif
return dest;
}

View file

@ -0,0 +1,32 @@
#include <string.h>
#include <stdint.h>
#include <limits.h>
#define ALIGN (sizeof(size_t)-1)
#define ONES ((size_t)-1/UCHAR_MAX)
#define HIGHS (ONES * (UCHAR_MAX/2+1))
#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
char *__stpncpy(char *restrict d, const char *restrict s, size_t n)
{
#ifdef __GNUC__
typedef size_t __attribute__((__may_alias__)) word;
word *wd;
const word *ws;
if (((uintptr_t)s & ALIGN) == ((uintptr_t)d & ALIGN)) {
for (; ((uintptr_t)s & ALIGN) && n && (*d=*s); n--, s++, d++);
if (!n || !*s) goto tail;
wd=(void *)d; ws=(const void *)s;
for (; n>=sizeof(size_t) && !HASZERO(*ws);
n-=sizeof(size_t), ws++, wd++) *wd = *ws;
d=(void *)wd; s=(const void *)ws;
}
#endif
for (; n && (*d=*s); n--, s++, d++);
tail:
memset(d, 0, n);
return d;
}
weak_alias(__stpncpy, stpncpy);

View file

@ -0,0 +1,7 @@
#include <string.h>
char *strchr(const char *s, int c)
{
char *r = __strchrnul(s, c);
return *(unsigned char *)r == (unsigned char)c ? r : 0;
}

View file

@ -0,0 +1,75 @@
#include <string.h>
#include <stdint.h>
#include <limits.h>
#ifdef __wasm_simd128__
#include <wasm_simd128.h>
#endif
#define ALIGN (sizeof(size_t))
#define ONES ((size_t)-1/UCHAR_MAX)
#define HIGHS (ONES * (UCHAR_MAX/2+1))
#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
char *__strchrnul(const char *s, int c)
{
c = (unsigned char)c;
if (!c) return (char *)s + strlen(s);
#if defined(__wasm_simd128__) && defined(__wasilibc_simd_string)
// Skip Clang 19 and Clang 20 which have a bug (llvm/llvm-project#146574)
// which results in an ICE when inline assembly is used with a vector result.
#if __clang_major__ != 19 && __clang_major__ != 20
// Note that reading before/after the allocation of a pointer is UB in
// C, so inline assembly is used to generate the exact machine
// instruction we want with opaque semantics to the compiler to avoid
// the UB.
uintptr_t align = (uintptr_t)s % sizeof(v128_t);
uintptr_t addr = (uintptr_t)s - align;
v128_t vc = wasm_i8x16_splat(c);
for (;;) {
v128_t v;
__asm__ (
"local.get %1\n"
"v128.load 0\n"
"local.set %0\n"
: "=r"(v)
: "r"(addr)
: "memory");
const v128_t cmp = wasm_i8x16_eq(v, (v128_t){}) | wasm_i8x16_eq(v, vc);
// Bitmask is slow on AArch64, any_true is much faster.
if (wasm_v128_any_true(cmp)) {
// Clear the bits corresponding to align (little-endian)
// so we can count trailing zeros.
int mask = wasm_i8x16_bitmask(cmp) >> align << align;
// At least one bit will be set, unless align cleared them.
// Knowing this helps the compiler if it unrolls the loop.
__builtin_assume(mask || align);
// If the mask became zero because of align,
// it's as if we didn't find anything.
if (mask) {
// Find the offset of the first one bit (little-endian).
return (char *)s + (addr - (uintptr_t)s + __builtin_ctz(mask));
}
}
align = 0;
addr += sizeof(v128_t);
}
#endif
#endif
#ifdef __GNUC__
typedef size_t __attribute__((__may_alias__)) word;
const word *w;
for (; (uintptr_t)s % ALIGN; s++)
if (!*s || *(unsigned char *)s == c) return (char *)s;
size_t k = ONES * c;
for (w = (void *)s; !HASZERO(*w) && !HASZERO(*w^k); w++);
s = (void *)w;
#endif
for (; *s && *(unsigned char *)s != c; s++);
return (char *)s;
}
weak_alias(__strchrnul, strchrnul);

View file

@ -0,0 +1,7 @@
#include <string.h>
int strcmp(const char *l, const char *r)
{
for (; *l==*r && *l; l++, r++);
return *(unsigned char *)l - *(unsigned char *)r;
}

View file

@ -0,0 +1,68 @@
#include <string.h>
#include <stdint.h>
#include <limits.h>
#ifdef __wasm_simd128__
#include <wasm_simd128.h>
#endif
#define ALIGN (sizeof(size_t))
#define ONES ((size_t)-1/UCHAR_MAX)
#define HIGHS (ONES * (UCHAR_MAX/2+1))
#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
size_t strlen(const char *s)
{
#if defined(__wasm_simd128__) && defined(__wasilibc_simd_string)
// Skip Clang 19 and Clang 20 which have a bug (llvm/llvm-project#146574)
// which results in an ICE when inline assembly is used with a vector result.
#if __clang_major__ != 19 && __clang_major__ != 20
// Note that reading before/after the allocation of a pointer is UB in
// C, so inline assembly is used to generate the exact machine
// instruction we want with opaque semantics to the compiler to avoid
// the UB.
uintptr_t align = (uintptr_t)s % sizeof(v128_t);
uintptr_t addr = (uintptr_t)s - align;
for (;;) {
v128_t v;
__asm__ (
"local.get %1\n"
"v128.load 0\n"
"local.set %0\n"
: "=r"(v)
: "r"(addr)
: "memory");
// Bitmask is slow on AArch64, all_true is much faster.
if (!wasm_i8x16_all_true(v)) {
const v128_t cmp = wasm_i8x16_eq(v, (v128_t){});
// Clear the bits corresponding to align (little-endian)
// so we can count trailing zeros.
int mask = wasm_i8x16_bitmask(cmp) >> align << align;
// At least one bit will be set, unless align cleared them.
// Knowing this helps the compiler if it unrolls the loop.
__builtin_assume(mask || align);
// If the mask became zero because of align,
// it's as if we didn't find anything.
if (mask) {
// Find the offset of the first one bit (little-endian).
return addr - (uintptr_t)s + __builtin_ctz(mask);
}
}
align = 0;
addr += sizeof(v128_t);
}
#endif
#endif
const char *a = s;
#ifdef __GNUC__
typedef size_t __attribute__((__may_alias__)) word;
const word *w;
for (; (uintptr_t)s % ALIGN; s++) if (!*s) return s-a;
for (w = (const void *)s; !HASZERO(*w); w++);
s = (const void *)w;
#endif
for (; *s; s++);
return s-a;
}

View file

@ -0,0 +1,10 @@
#include <string.h>
char *strncat(char *restrict d, const char *restrict s, size_t n)
{
char *a = d;
d += strlen(d);
while (n && *s) n--, *d++ = *s++;
*d++ = 0;
return a;
}

View file

@ -0,0 +1,9 @@
#include <string.h>
int strncmp(const char *_l, const char *_r, size_t n)
{
const unsigned char *l=(void *)_l, *r=(void *)_r;
if (!n--) return 0;
for (; *l && *r && n && *l == *r ; l++, r++, n--);
return *l - *r;
}

View file

@ -0,0 +1,7 @@
#include <string.h>
char *strncpy(char *restrict d, const char *restrict s, size_t n)
{
__stpncpy(d, s, n);
return d;
}

View file

@ -0,0 +1,8 @@
#include <wchar.h>
wchar_t *wcschr(const wchar_t *s, wchar_t c)
{
if (!c) return (wchar_t *)s + wcslen(s);
for (; *s && *s != c; s++);
return *s ? (wchar_t *)s : 0;
}

View file

@ -0,0 +1,8 @@
#include <wchar.h>
size_t wcslen(const wchar_t *s)
{
const wchar_t *a;
for (a=s; *s; s++);
return s-a;
}

View file

@ -1,3 +1,6 @@
// Scanner-oriented stdio compatibility functions. Formatting writes to memory,
// while stream operations intentionally perform no I/O.
#include <stdio.h>
#include <string.h>
@ -106,14 +109,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
return 2 + len;
}
char *strncpy(char *dest, const char *src, size_t n) {
char *d = dest;
const char *s = src;
while (n-- && (*d++ = *s++));
if (n == (size_t)-1) *d = '\0';
return dest;
}
static int write_formatted_to_buffer(
char *buffer,
size_t buffer_size,

File diff suppressed because it is too large Load diff

View file

@ -1,5 +1,6 @@
#include "tree_sitter/api.h"
#include "./parser.h"
#include <stddef.h>
#include <stdint.h>
#ifdef TREE_SITTER_FEATURE_WASM
@ -9,7 +10,7 @@
#include "./atomic.h"
#include "./language.h"
#include "./lexer.h"
#include "./wasm/wasm-stdlib.h"
#include "./wasm-stdlib/external_scanner_stdlib.h"
#include "./wasm_store.h"
#include <string.h>
@ -29,7 +30,7 @@
// The following symbols from the C and C++ standard libraries are available
// for external scanners to use.
const char *STDLIB_SYMBOLS[] = {
#include "./stdlib-symbols.txt"
#include "./wasm-stdlib/imports.txt"
};
// The contents of the `dylink.0` custom section of a Wasm module,
@ -49,11 +50,11 @@ typedef struct {
volatile uint32_t is_language_deleted;
} WasmLanguageId;
// LanguageWasmModule - Additional data associated with a Wasm-backed
// `TSLanguage`. This data is read-only and does not reference a particular
// Wasm store, so it can be shared by all users of a `TSLanguage`. A pointer to
// this is stored on the language itself.
// TSWasmLanguage - A reference-counted language loaded from a Wasm module.
// This data is read-only and does not reference a particular Wasm store, so it
// can be shared by all users of the language.
typedef struct {
TSLanguage language;
volatile uint32_t ref_count;
WasmLanguageId *language_id;
wasmtime_module_t *module;
@ -61,7 +62,11 @@ typedef struct {
char *symbol_name_buffer;
char *field_name_buffer;
WasmDylinkInfo dylink_info;
} LanguageWasmModule;
} TSWasmLanguage;
static inline TSWasmLanguage *ts_language__wasm_language(const TSLanguage *self) {
return (TSWasmLanguage *)((char *)self - offsetof(TSWasmLanguage, language));
}
// LanguageWasmInstance - Additional data associated with an instantiation of
// a `TSLanguage` in a particular Wasm store. The Wasm store holds one of
@ -502,11 +507,12 @@ static void *copy_string(
}
static void delete_partially_loaded_language(
TSLanguage *language,
TSWasmLanguage *result,
StringData *symbol_name_buffer,
StringData *field_name_buffer
) {
if (language) {
if (result) {
TSLanguage *language = &result->language;
ts_free((void *)language->alias_map);
ts_free((void *)language->alias_sequences);
ts_free((void *)language->external_scanner.symbol_map);
@ -527,7 +533,7 @@ static void delete_partially_loaded_language(
ts_free((void *)language->supertype_symbols);
ts_free((void *)language->symbol_metadata);
ts_free((void *)language->symbol_names);
ts_free(language);
ts_free(result);
}
array_delete(symbol_name_buffer);
array_delete(field_name_buffer);
@ -1272,6 +1278,7 @@ const TSLanguage *ts_wasm_store_load_language(
WasmDylinkInfo dylink_info;
wasmtime_module_t *module = NULL;
wasmtime_error_t *error = NULL;
TSWasmLanguage *result = NULL;
TSLanguage *language = NULL;
StringData symbol_name_buffer = array_new();
StringData field_name_buffer = array_new();
@ -1310,15 +1317,34 @@ const TSLanguage *ts_wasm_store_load_language(
goto error;
}
// Copy all of the static data out of the language object in Wasm memory,
// constructing a native language object.
LanguageInWasmMemory wasm_language;
wasmtime_context_t *context = wasmtime_store_context(self->store);
const uint8_t *memory = wasmtime_memory_data(context, &self->memory);
WasmMemory wasm_memory = {
.data = memory,
.size = wasmtime_memory_data_size(context, &self->memory),
};
uint32_t abi_version;
if (!wasm_memory__read(&wasm_memory, language_address, &abi_version, sizeof(abi_version))) {
goto invalid_language_memory;
}
if (
abi_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION ||
abi_version > TREE_SITTER_LANGUAGE_VERSION
) {
wasm_error->kind = TSWasmErrorKindInstantiate;
format(
&wasm_error->message,
"incompatible language ABI version %u; expected between %u and %u",
abi_version,
TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION,
TREE_SITTER_LANGUAGE_VERSION
);
goto error;
}
// Copy all of the static data out of the language object in Wasm memory,
// constructing a native language object.
LanguageInWasmMemory wasm_language;
bool valid_wasm_memory = true;
if (!wasm_memory__read(&wasm_memory, language_address, &wasm_language, sizeof(LanguageInWasmMemory))) {
goto invalid_language_memory;
@ -1362,7 +1388,8 @@ const TSLanguage *ts_wasm_store_load_language(
};
uint32_t address_count = array_len(addresses);
language = ts_calloc(1, sizeof(TSLanguage));
result = ts_calloc(1, sizeof(TSWasmLanguage));
language = &result->language;
*language = (TSLanguage) {
.abi_version = wasm_language.abi_version,
.symbol_count = wasm_language.symbol_count,
@ -1596,22 +1623,18 @@ const TSLanguage *ts_wasm_store_load_language(
memcpy(name, language_name, name_len);
name[name_len] = '\0';
LanguageWasmModule *language_module = ts_malloc(sizeof(LanguageWasmModule));
*language_module = (LanguageWasmModule) {
.language_id = language_id_new(),
.module = module,
.name = name,
.symbol_name_buffer = symbol_name_buffer.contents,
.field_name_buffer = field_name_buffer.contents,
.dylink_info = dylink_info,
.ref_count = 1,
};
result->ref_count = 1;
result->language_id = language_id_new();
result->module = module;
result->name = name;
result->symbol_name_buffer = symbol_name_buffer.contents;
result->field_name_buffer = field_name_buffer.contents;
result->dylink_info = dylink_info;
// The lex functions are not used for Wasm languages. Use those two fields
// to mark this language as Wasm-based and to store the language's
// Wasm-specific data.
// The lex function is not called for Wasm languages. Use a sentinel to mark
// this language as Wasm-based.
language->lex_fn = ts_wasm_store__sentinel_lex_fn;
language->keyword_lex_fn = (bool (*)(TSLexer *, TSStateId))language_module;
language->keyword_lex_fn = NULL;
// Clear out any instances of languages that have been deleted.
for (unsigned i = 0; i < self->language_instances.size; i++) {
@ -1625,7 +1648,7 @@ const TSLanguage *ts_wasm_store_load_language(
// Store this store's instance of this language module.
array_push(&self->language_instances, ((LanguageWasmInstance) {
.language_id = language_id_clone(language_module->language_id),
.language_id = language_id_clone(result->language_id),
.instance = instance,
.external_states_address = wasm_language.external_scanner.states,
.lex_main_fn_index = wasm_language.lex_fn,
@ -1645,7 +1668,7 @@ invalid_language_memory:
goto error;
error:
delete_partially_loaded_language(language, &symbol_name_buffer, &field_name_buffer);
delete_partially_loaded_language(result, &symbol_name_buffer, &field_name_buffer);
if (module) wasmtime_module_delete(module);
return NULL;
}
@ -1656,7 +1679,7 @@ bool ts_wasm_store_add_language(
uint32_t *index
) {
wasmtime_context_t *context = wasmtime_store_context(self->store);
const LanguageWasmModule *language_module = (void *)language->keyword_lex_fn;
const TSWasmLanguage *language_data = ts_language__wasm_language(language);
// Search for this store's instance of the language module. Also clear out any
// instances of languages that have been deleted.
@ -1667,7 +1690,7 @@ bool ts_wasm_store_add_language(
language_id_delete(id);
array_erase(&self->language_instances, i);
i--;
} else if (id == language_module->language_id) {
} else if (id == language_data->language_id) {
exists = true;
*index = i;
}
@ -1682,9 +1705,9 @@ bool ts_wasm_store_add_language(
int32_t language_address;
if (!ts_wasm_store__instantiate(
self,
language_module->module,
language_module->name,
&language_module->dylink_info,
language_data->module,
language_data->name,
&language_data->dylink_info,
&instance,
&language_address,
&message
@ -1703,7 +1726,7 @@ bool ts_wasm_store_add_language(
return false;
}
array_push(&self->language_instances, ((LanguageWasmInstance) {
.language_id = language_id_clone(language_module->language_id),
.language_id = language_id_clone(language_data->language_id),
.instance = instance,
.external_states_address = wasm_language.external_scanner.states,
.lex_main_fn_index = wasm_language.lex_fn,
@ -1950,30 +1973,25 @@ bool ts_language_is_wasm(const TSLanguage *self) {
return self->lex_fn == ts_wasm_store__sentinel_lex_fn;
}
static inline LanguageWasmModule *ts_language__wasm_module(const TSLanguage *self) {
return (LanguageWasmModule *)self->keyword_lex_fn;
}
void ts_wasm_language_retain(const TSLanguage *self) {
LanguageWasmModule *module = ts_language__wasm_module(self);
ts_assert(module->ref_count > 0);
atomic_inc(&module->ref_count);
TSWasmLanguage *language = ts_language__wasm_language(self);
ts_assert(language->ref_count > 0);
atomic_inc(&language->ref_count);
}
void ts_wasm_language_release(const TSLanguage *self) {
LanguageWasmModule *module = ts_language__wasm_module(self);
ts_assert(module->ref_count > 0);
if (atomic_dec(&module->ref_count) == 0) {
TSWasmLanguage *language = ts_language__wasm_language(self);
ts_assert(language->ref_count > 0);
if (atomic_dec(&language->ref_count) == 0) {
// Update the language id to reflect that the language is deleted. This allows any Wasm stores
// that hold Wasm instances for this language to delete those instances.
atomic_inc(&module->language_id->is_language_deleted);
language_id_delete(module->language_id);
atomic_inc(&language->language_id->is_language_deleted);
language_id_delete(language->language_id);
ts_free((void *)module->field_name_buffer);
ts_free((void *)module->symbol_name_buffer);
ts_free((void *)module->name);
wasmtime_module_delete(module->module);
ts_free(module);
ts_free((void *)language->field_name_buffer);
ts_free((void *)language->symbol_name_buffer);
ts_free((void *)language->name);
wasmtime_module_delete(language->module);
ts_free((void *)self->alias_map);
ts_free((void *)self->alias_sequences);

View file

@ -0,0 +1,6 @@
/src/grammar.json
/src/node-types.json
/src/parser.c
/src/scanner.c
/src/tree_sitter/
/target/

BIN
test/fixtures/rust_wasm_web/Cargo.lock generated vendored Normal file

Binary file not shown.

19
test/fixtures/rust_wasm_web/Cargo.toml vendored Normal file
View file

@ -0,0 +1,19 @@
[package]
name = "rust-wasm-web-test"
version = "0.0.0"
edition = "2024"
rust-version = "1.90"
publish = false
[lib]
path = "src/rust_wasm_web.rs"
crate-type = ["cdylib"]
[dependencies]
tree-sitter = { path = "../../../lib" }
tree-sitter-javascript = { path = "../grammars/javascript" }
[patch.crates-io]
tree-sitter-language = { path = "../../../crates/language" }
[workspace]

3
test/fixtures/rust_wasm_web/README.md vendored Normal file
View file

@ -0,0 +1,3 @@
# Rust Wasm Web Test
This is a minimal example of using the `tree-sitter` Rust crate within a larger application that is compiled to WebAssembly and embedded within a JavaScript engine. It uses web workers to perform parsing on a background thread while still sharing memory with the main thread. One grammar crate is included in the main module as a Rust dependency, and two other grammar crates are dynamically loaded as separate modules.

102
test/fixtures/rust_wasm_web/run.mjs vendored Normal file
View file

@ -0,0 +1,102 @@
import fs from 'node:fs/promises';
import { Worker } from 'node:worker_threads';
const MEMORY_PAGES = 256;
const MAX_MEMORY_PAGES = 4096;
const control = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
const [runtimePath, pythonPath, rubyPath] = process.argv.slice(2);
if (!runtimePath || !pythonPath || !rubyPath) {
throw new Error('usage: node run.mjs <runtime.wasm> <python.wasm> <ruby.wasm>');
}
const runtimeModule = await WebAssembly.compile(await fs.readFile(runtimePath));
const languageModules = await Promise.all(
[pythonPath, rubyPath].map(async path => WebAssembly.compile(await fs.readFile(path))),
);
const memory = new WebAssembly.Memory({
initial: MEMORY_PAGES,
maximum: MAX_MEMORY_PAGES,
shared: true,
});
if (!(memory.buffer instanceof SharedArrayBuffer)) {
throw new Error('WebAssembly memory is not shared');
}
let worker;
let pendingRequest = 0;
const uiRuntime = await WebAssembly.instantiate(runtimeModule, {
env: {
memory,
request_parse(languageId, sourceAddress, sourceLength, oldTree, requestAddress) {
if (pendingRequest !== 0) {
throw new Error('Rust requested another parse before the prior request completed');
}
pendingRequest = requestAddress;
if (oldTree) {
Atomics.store(control, 0, 0);
}
worker.postMessage(
oldTree
? { languageId, sourceAddress, sourceLength, oldTree }
: { languageId, sourceAddress, sourceLength },
);
},
log(messageAddress, messageLength) {
console.log(
new TextDecoder().decode(
new Uint8Array(memory.buffer, messageAddress, messageLength),
),
);
},
notify_parsing_started() {
throw new Error('UI runtime unexpectedly paused for parsing');
},
},
});
worker = new Worker(new URL('./worker.mjs', import.meta.url), {
workerData: { control, runtimeModule, languageModules, memory },
});
try {
await new Promise((resolve, reject) => {
let finished = false;
worker.once('error', reject);
worker.once('exit', code => {
if (!finished) {
reject(new Error(`worker exited before completing the test with code ${code}`));
}
});
worker.on('message', message => {
try {
if (message?.ready === true) {
uiRuntime.exports.start();
return;
}
if (message?.parsing) {
console.log('parsing...');
Atomics.store(control, 0, 1);
Atomics.notify(control, 0);
return;
}
if (!Number.isInteger(message?.tree) || message.tree === 0) {
throw new Error('worker did not return a tree');
}
const request = pendingRequest;
if (request === 0) {
throw new Error('worker returned a tree without a pending request');
}
pendingRequest = 0;
if (uiRuntime.exports.tree_ready(request, message.tree)) {
finished = true;
resolve();
}
} catch (error) {
reject(error);
}
});
});
} finally {
await worker.terminate();
}

View file

@ -0,0 +1,311 @@
use std::{
alloc::{Layout, alloc_zeroed},
cell::UnsafeCell,
future::Future,
marker::PhantomPinned,
pin::Pin,
slice, str,
sync::{
LazyLock,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll, Waker},
};
use tree_sitter::{
InputEdit, Language, LanguageError, Parser, Point, Query, QueryCursor, StreamingIterator, Tree,
ffi::TSLanguage,
};
type Application = Pin<Box<dyn Future<Output = ()>>>;
const JAVASCRIPT: u32 = 0;
const PYTHON: u32 = 1;
const RUBY: u32 = 2;
struct ApplicationSlot(UnsafeCell<Option<Application>>);
unsafe impl Sync for ApplicationSlot {}
static APPLICATION: ApplicationSlot = ApplicationSlot(UnsafeCell::new(None));
static JAVASCRIPT_QUERY_INITIALIZATIONS: AtomicUsize = AtomicUsize::new(0);
static WORKER_QUERY_CAPTURE_COUNT: AtomicUsize = AtomicUsize::new(0);
static JAVASCRIPT_QUERY: LazyLock<Query> = LazyLock::new(|| {
JAVASCRIPT_QUERY_INITIALIZATIONS.fetch_add(1, Ordering::SeqCst);
Query::new(
&Language::from(tree_sitter_javascript::LANGUAGE),
"(number) @number",
)
.unwrap()
});
#[link(wasm_import_module = "env")]
unsafe extern "C" {
fn request_parse(
language_id: u32,
source_address: u32,
source_length: u32,
old_tree_address: u32,
request_address: u32,
);
fn log(message_address: u32, message_length: u32);
fn notify_parsing_started();
}
fn log_progress(message: &str) {
unsafe { log(message.as_ptr() as u32, message.len() as u32) }
}
fn javascript_query_capture_count(tree: &Tree, source: &str) -> usize {
let mut cursor = QueryCursor::new();
let mut captures = cursor.captures(&JAVASCRIPT_QUERY, tree.root_node(), source.as_bytes());
let mut count = 0;
while captures.next().is_some() {
count += 1;
}
count
}
fn assert_javascript_query(tree: &Tree, source: &str, expected_count: usize) {
assert_eq!(JAVASCRIPT_QUERY_INITIALIZATIONS.load(Ordering::SeqCst), 1);
assert_eq!(
WORKER_QUERY_CAPTURE_COUNT.load(Ordering::Acquire),
expected_count
);
assert_eq!(javascript_query_capture_count(tree, source), expected_count);
}
struct ParseRequest<'a> {
language_id: u32,
source: &'a str,
old_tree: Option<Tree>,
requested: bool,
result: Option<Tree>,
_pinned: PhantomPinned,
}
impl Future for ParseRequest<'_> {
type Output = Tree;
fn poll(self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll<Self::Output> {
let this = unsafe { self.get_unchecked_mut() };
if let Some(tree) = this.result.take() {
return Poll::Ready(tree);
}
if !this.requested {
let old_tree_address = this
.old_tree
.take()
.map_or(0, |tree| tree.into_raw() as u32);
this.requested = true;
let request_address = std::ptr::from_mut(this) as u32;
unsafe {
request_parse(
this.language_id,
this.source.as_ptr() as u32,
this.source.len() as u32,
old_tree_address,
request_address,
);
}
}
Poll::Pending
}
}
fn spawn_parse<'a>(language_id: u32, source: &'a str, old_tree: Option<Tree>) -> ParseRequest<'a> {
ParseRequest {
language_id,
source,
old_tree,
requested: false,
result: None,
_pinned: PhantomPinned,
}
}
async fn run() {
let mut source = String::from("const value = []\n");
let mut tree = spawn_parse(JAVASCRIPT, &source, None).await;
let language = tree.language();
assert_eq!(language.name(), Some("javascript"));
assert!(!language.is_parseable());
assert!(!tree.root_node().language().is_parseable());
assert_eq!(
Parser::new().set_language(&language),
Err(LanguageError::NotParseable)
);
assert_eq!(
tree.root_node().to_sexp(),
"(program (lexical_declaration (variable_declarator name: (identifier) value: (array))))"
);
assert_javascript_query(&tree, &source, 0);
log_progress("parsed javascript");
for integer in 0..100 {
let edit_start = source.find(']').unwrap();
let insertion = if integer == 0 {
integer.to_string()
} else {
format!(", {integer}")
};
source.insert_str(edit_start, &insertion);
let new_end = edit_start + insertion.len();
tree.edit(&InputEdit {
start_byte: edit_start,
old_end_byte: edit_start,
new_end_byte: new_end,
start_position: Point::new(0, edit_start),
old_end_position: Point::new(0, edit_start),
new_end_position: Point::new(0, new_end),
});
let prev_tree = tree.clone();
if integer == 0 {
assert_eq!(
Parser::new().set_language(&prev_tree.language()),
Err(LanguageError::NotParseable)
);
}
tree = spawn_parse(JAVASCRIPT, &source, Some(tree)).await;
let array_node = tree
.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.child_by_field_name("value")
.unwrap();
assert_eq!(array_node.named_child_count(), integer + 1);
assert!(prev_tree.changed_ranges(&tree).len() > 0);
assert_javascript_query(&tree, &source, integer + 1);
log_progress(&format!("incrementally reparsed ({} / 100)", integer + 1));
}
let python_source = String::from("def answer():\n return 42\n");
let python_tree = spawn_parse(PYTHON, &python_source, None).await;
assert_eq!(
python_tree.root_node().to_sexp(),
"(module (function_definition name: (identifier) parameters: (parameters) body: (block (return_statement (integer)))))"
);
log_progress("parsed python");
// Ruby's scanner uses wide-character classification for suffixed constants
// and unquoted heredoc delimiters.
let ruby_source = String::from("Éclair!\nvalue = <<~ÉTÉ\n héllo\nÉTÉ\n");
let ruby_tree = spawn_parse(RUBY, &ruby_source, None).await;
assert_eq!(
ruby_tree.root_node().to_sexp(),
"(program (call method: (constant)) (assignment left: (identifier) right: (heredoc_beginning)) (heredoc_body (heredoc_content) (heredoc_end)))"
);
log_progress("parsed ruby");
}
fn poll_application() -> bool {
let application = unsafe { &mut *APPLICATION.0.get() };
let future = application.as_mut().unwrap();
let mut context = Context::from_waker(Waker::noop());
if future.as_mut().poll(&mut context).is_ready() {
*application = None;
true
} else {
false
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn allocate_language_memory(size: u32, alignment: u32) -> u32 {
let layout = Layout::from_size_align(size as usize, alignment as usize).unwrap();
unsafe { alloc_zeroed(layout) as u32 }
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn run_parse(
language_address: u32,
source_address: u32,
source_length: u32,
old_tree_address: u32,
) -> u32 {
let tree = run_parse_internal(
language_address,
source_address,
source_length,
old_tree_address,
)
.ok();
tree.map_or(0, |tree| tree.into_raw() as u32)
}
fn run_parse_internal(
language_address: u32,
source_address: u32,
source_length: u32,
old_tree_address: u32,
) -> Result<Tree, u32> {
let source_bytes =
unsafe { slice::from_raw_parts(source_address as *const u8, source_length as usize) };
let Ok(source) = str::from_utf8(source_bytes) else {
return Err(1);
};
let old_tree = if old_tree_address == 0 {
None
} else {
Some(unsafe { Tree::from_raw(old_tree_address as *mut _) })
};
let mut parser = Parser::new();
if let Some(tree) = old_tree.as_ref() {
if parser.set_language(&tree.language()).is_err() {
return Err(2);
}
} else {
let language = if language_address == JAVASCRIPT {
Language::from(tree_sitter_javascript::LANGUAGE)
} else {
unsafe { Language::from_raw(language_address as *const TSLanguage) }
};
if parser.set_language(&language).is_err() {
return Err(3);
}
}
let mut paused = false;
let Some(tree) = parser.parse_with_options(
&mut |byte_offset, _| {
if old_tree.is_some() && !paused {
paused = true;
unsafe { notify_parsing_started() };
}
source.as_bytes().get(byte_offset..).unwrap_or_default()
},
old_tree.as_ref(),
None,
) else {
return Err(4);
};
if language_address == JAVASCRIPT {
WORKER_QUERY_CAPTURE_COUNT.store(
javascript_query_capture_count(&tree, source),
Ordering::Release,
);
}
Ok(tree)
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn start() {
let application = unsafe { &mut *APPLICATION.0.get() };
assert!(application.is_none());
*application = Some(Box::pin(run()));
assert!(!poll_application());
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn tree_ready(request_address: u32, tree_address: u32) -> u32 {
assert_ne!(request_address, 0);
assert_ne!(tree_address, 0);
let request = unsafe { &mut *(request_address as *mut ParseRequest<'static>) };
assert!(request.requested);
assert!(request.result.is_none());
request.result = Some(unsafe { Tree::from_raw(tree_address as *mut _) });
u32::from(poll_application())
}

99
test/fixtures/rust_wasm_web/worker.mjs vendored Normal file
View file

@ -0,0 +1,99 @@
import { parentPort, workerData } from 'node:worker_threads';
const PAGE_SIZE = 64 * 1024;
const SIDE_MODULE_DATA_PAGES = 32;
const SIDE_MODULE_STACK_PAGES = 16;
const SIDE_MODULE_TABLE_ELEMENTS = 1024;
const RUNTIME_STACK_PAGES = 16;
const { control, runtimeModule, languageModules, memory } = workerData;
const runtime = await WebAssembly.instantiate(runtimeModule, {
env: {
memory,
request_parse() {
throw new Error('parsing worker unexpectedly requested a parse');
},
log() {
throw new Error('parsing worker unexpectedly logged UI progress');
},
notify_parsing_started() {
parentPort.postMessage({ parsing: true });
Atomics.wait(control, 0, 0);
},
},
});
const {
__stack_pointer: stackPointer,
__indirect_function_table: table,
allocate_language_memory: allocateLanguageMemory,
run_parse: runParse,
} = runtime.exports;
const runtimeStackBase = allocateLanguageMemory(RUNTIME_STACK_PAGES * PAGE_SIZE, PAGE_SIZE);
if (!runtimeStackBase) {
throw new Error('Rust test module failed to allocate the worker runtime stack');
}
stackPointer.value = runtimeStackBase + RUNTIME_STACK_PAGES * PAGE_SIZE;
function loadLanguage(languageModule) {
const memoryBase = allocateLanguageMemory(
(SIDE_MODULE_DATA_PAGES + SIDE_MODULE_STACK_PAGES) * PAGE_SIZE,
PAGE_SIZE,
);
const tableBase = table.length;
table.grow(SIDE_MODULE_TABLE_ELEMENTS);
const env = {
memory,
__indirect_function_table: table,
__memory_base: new WebAssembly.Global({ value: 'i32', mutable: false }, memoryBase),
__table_base: new WebAssembly.Global({ value: 'i32', mutable: false }, tableBase),
__stack_pointer: new WebAssembly.Global(
{ value: 'i32', mutable: true },
memoryBase + (SIDE_MODULE_DATA_PAGES + SIDE_MODULE_STACK_PAGES) * PAGE_SIZE,
),
};
for (const { module, name, kind } of WebAssembly.Module.imports(languageModule)) {
if (module === 'env' && kind === 'function') {
const implementation = runtime.exports[name];
if (typeof implementation !== 'function') {
throw new Error(`Rust test module does not export ${name}`);
}
env[name] = implementation;
}
}
return WebAssembly.instantiate(languageModule, { env }).then(language => {
if (typeof language.exports.__wasm_apply_data_relocs === 'function') {
language.exports.__wasm_apply_data_relocs();
}
const languageFunction = Object.entries(language.exports).find(
([name, value]) => /^tree_sitter_\w+$/.test(name) && typeof value === 'function',
);
if (!languageFunction) {
throw new Error('language module did not export a tree_sitter_* function');
}
return languageFunction[1]();
});
}
const languageAddresses = await Promise.all(languageModules.map(loadLanguage));
parentPort.postMessage({ ready: true });
parentPort.on('message', message => {
const languageAddress =
message.languageId === 0 ? 0 : languageAddresses[message.languageId - 1];
const tree = runParse(
languageAddress,
message.sourceAddress,
message.sourceLength,
message.oldTree ?? 0,
);
if (tree === 0) {
throw new Error(`parsing worker failed to parse language ${message.languageId}`);
}
parentPort.postMessage({ tree });
});

View file

@ -1,7 +1,7 @@
#include "tree_sitter/parser.h"
// Constant copied from `crates/language/wasm/src/stdlib.c`,
// Must be kept in sync for a reliable repro.
// Constant copied from `lib/src/wasm-stdlib/external_scanner_allocator.c`.
// Must be kept in sync for a reliable repro.
#define MAX_HEAP_SIZE (4 * 1024 * 1024)
enum TokenType {