build(deps): bump rquickjs to v0.13.0

Also use `IoError` more consistently throughout the rest of the project
where its straightforward to do so.

BREAKING CHANGE: Changes public error types for config, generate, and
loader crates.

.github/workflows/build.yml

@@ -239,7 +239,7 @@ jobs:
 
     - name: Build C library and Rust crate (illumos gmake)
       if: matrix.platform == 'illumos-x64'
-      uses: vmactions/omnios-vm@v1.3.4
+      uses: vmactions/omnios-vm@v1.3.6
       with:
         release: r151056-build
         copyback: false

.github/workflows/docs.yml

@@ -18,22 +18,16 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v7.0.1
 
-      - name: Set up Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
       - name: Install mdbook
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
-          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.4.52 --jq "$jq_expr")
+          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.5.4 --jq "$jq_expr")
           mkdir mdbook
           curl -sSL "$url" | tar -xz -C mdbook
           printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
 
-      - name: Install mdbook-admonish
-        run: cargo install mdbook-admonish
-
       - name: Build Book
         run: mdbook build docs
 

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.27.0"
+        VERSION "0.28.0"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)

Cargo.toml

@@ -22,7 +22,7 @@ keywords = [ "incremental", "parsing" ]
 license = "MIT"
 repository = "https://github.com/tree-sitter/tree-sitter"
 rust-version = "1.90"
-version = "0.27.0"
+version = "0.28.0"
 
 [workspace.lints.clippy]
 cargo = { level = "warn", priority = -1 }
@@ -128,11 +128,11 @@ walkdir = "2.5.0"
 wasmparser = "0.244.0"
 webbrowser = "1.2.1"
 
-tree-sitter = { path = "./lib", version = "0.27.0" }
-tree-sitter-config = { path = "./crates/config", version = "0.27.0" }
-tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.27.0" }
-tree-sitter-highlight = { path = "./crates/highlight", version = "0.27.0" }
-tree-sitter-loader = { path = "./crates/loader", version = "0.27.0" }
-tree-sitter-tags = { path = "./crates/tags", version = "0.27.0" }
+tree-sitter = { path = "./lib", version = "0.28.0" }
+tree-sitter-config = { path = "./crates/config", version = "0.28.0" }
+tree-sitter-generate = { default-features = false, path = "./crates/generate", version = "0.28.0" }
+tree-sitter-highlight = { path = "./crates/highlight", version = "0.28.0" }
+tree-sitter-loader = { path = "./crates/loader", version = "0.28.0" }
+tree-sitter-tags = { path = "./crates/tags", version = "0.28.0" }
 
 tree-sitter-language = { path = "./crates/language", version = "0.1.8" }

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.27.0
+VERSION := 0.28.0
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 

build.zig.zon

@@ -1,7 +1,7 @@
 .{
     .name = .tree_sitter,
     .fingerprint = 0x841224b447ac0d4f,
-    .version = "0.27.0",
+    .version = "0.28.0",
     .minimum_zig_version = "0.16.0",
     .paths = .{
         "build.zig",

crates/cli/Cargo.toml

@@ -5,7 +5,6 @@ description = "CLI tool for developing, testing, and using Tree-sitter parsers"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-cli"

crates/cli/eslint/package-lock.json

@@ -135,29 +135,43 @@
       }
     },
     "node_modules/@humanfs/core": {
-      "version": "0.19.1",
-      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
-      "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==",
+      "version": "0.19.2",
+      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz",
+      "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==",
       "license": "Apache-2.0",
       "peer": true,
+      "dependencies": {
+        "@humanfs/types": "^0.15.0"
+      },
       "engines": {
         "node": ">=18.18.0"
       }
     },
     "node_modules/@humanfs/node": {
-      "version": "0.16.7",
-      "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz",
-      "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==",
+      "version": "0.16.8",
+      "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz",
+      "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==",
       "license": "Apache-2.0",
       "peer": true,
       "dependencies": {
-        "@humanfs/core": "^0.19.1",
+        "@humanfs/core": "^0.19.2",
+        "@humanfs/types": "^0.15.0",
         "@humanwhocodes/retry": "^0.4.0"
       },
       "engines": {
         "node": ">=18.18.0"
       }
     },
+    "node_modules/@humanfs/types": {
+      "version": "0.15.0",
+      "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz",
+      "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==",
+      "license": "Apache-2.0",
+      "peer": true,
+      "engines": {
+        "node": ">=18.18.0"
+      }
+    },
     "node_modules/@humanwhocodes/module-importer": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",

crates/cli/npm/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "tree-sitter-cli",
-      "version": "0.27.0",
+      "version": "0.28.0",
       "hasInstallScript": true,
       "license": "MIT",
       "bin": {

crates/cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "author": {
     "name": "Max Brunsfeld",
     "email": "maxbrunsfeld@gmail.com"

crates/cli/package.nix

@@ -13,7 +13,7 @@
   installShellFiles,
 }:
 let
-  isCross = stdenv.targetPlatform == stdenv.buildPlatform;
+  canRunHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 in
 rustPlatform.buildRustPackage {
   pname = "tree-sitter-cli";
@@ -28,7 +28,7 @@ rustPlatform.buildRustPackage {
     pkg-config
     nodejs_22
   ]
-  ++ lib.optionals (!isCross) [ installShellFiles ];
+  ++ lib.optionals canRunHost [ installShellFiles ];
 
   cargoLock.lockFile = ../../Cargo.lock;
 
@@ -42,9 +42,9 @@ rustPlatform.buildRustPackage {
   '';
 
   preCheck = "export HOME=$TMPDIR";
-  doCheck = !isCross;
+  doCheck = canRunHost;
 
-  postInstall = lib.optionalString (!isCross) ''
+  postInstall = lib.optionalString canRunHost ''
     installShellCompletion --cmd tree-sitter \
       --bash <($out/bin/tree-sitter complete --shell bash) \
       --zsh  <($out/bin/tree-sitter complete --shell zsh) \

crates/cli/src/query.rs

@@ -142,7 +142,9 @@ pub fn query_file_at_path(
         };
         // Invariant: `test_summary` will always be `Some` when `should_test` is true
         let test_summary = test_summary.unwrap();
-        match query_testing::assert_expected_captures(&results, path, &mut parser, language) {
+        let assertions =
+            query_testing::parse_position_comments(&mut parser, language, source_code.as_slice())?;
+        match query_testing::assert_expected_captures(&results, &assertions) {
             Ok(assertion_count) => {
                 test_summary.query_results.add_case(TestResult {
                     name: path_name.to_string(),

crates/cli/src/query_testing.rs

@@ -1,4 +1,4 @@
-use std::{fs, path::Path, sync::LazyLock};
+use std::sync::LazyLock;
 
 use anyhow::{Result, anyhow};
 use bstr::{BStr, ByteSlice};
@@ -219,19 +219,14 @@ pub fn parse_position_comments(
     Ok(result)
 }
 
-pub fn assert_expected_captures(
-    infos: &[CaptureInfo],
-    path: &Path,
-    parser: &mut Parser,
-    language: &Language,
-) -> Result<usize> {
-    let contents = fs::read_to_string(path)?;
-    let pairs = parse_position_comments(parser, language, contents.as_bytes())?;
-    for assertion in &pairs {
+pub fn assert_expected_captures(infos: &[CaptureInfo], assertions: &[Assertion]) -> Result<usize> {
+    for assertion in assertions {
         if let Some(found) = &infos.iter().find(|p| {
-            assertion.position >= p.start
-                && (assertion.position.row < p.end.row
-                    || assertion.position.column + assertion.length - 1 < p.end.column)
+            let assertion_end = Utf8Point::new(
+                assertion.position.row,
+                assertion.position.column + assertion.length - 1,
+            );
+            assertion.position >= p.start && assertion_end < p.end
         }) {
             if assertion.expected_capture_name != found.name && found.name != "name" {
                 return Err(anyhow!(
@@ -250,5 +245,24 @@ pub fn assert_expected_captures(
             ));
         }
     }
-    Ok(pairs.len())
+    Ok(assertions.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Assertion, CaptureInfo, Utf8Point, assert_expected_captures};
+
+    #[test]
+    fn test_assertion_after_multiline_capture_does_not_match() {
+        let captures = [CaptureInfo {
+            name: "foo".to_string(),
+            start: Utf8Point::new(0, 0),
+            end: Utf8Point::new(1, 1),
+        }];
+        let assertions = [Assertion::new(2, 0, 1, false, "foo".to_string())];
+
+        let result = assert_expected_captures(&captures, &assertions);
+
+        assert!(result.is_err());
+    }
 }

crates/cli/src/tests/parser_test.rs

@@ -253,6 +253,21 @@ fn test_parsing_with_custom_utf16_be_input() {
     assert_eq!(root.child(0).unwrap().kind(), "function_item");
 }
 
+#[test]
+fn test_utf16_decodes_surrogate_pairs() {
+    let mut parser = Parser::new();
+    let language = get_test_fixture_language("utf16_surrogate_oob");
+    parser.set_language(&language).unwrap();
+
+    let le = [0xD83D_u16.to_le(), 0xDE00_u16.to_le()];
+    let tree = parser.parse_utf16_le(le, None).unwrap();
+    assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))");
+
+    let be = [0xD83D_u16.to_be(), 0xDE00_u16.to_be()];
+    let tree = parser.parse_utf16_be(be, None).unwrap();
+    assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))");
+}
+
 #[test]
 fn test_utf16_decode_does_not_read_oob() {
     // Test for a buffer over-read in ts_decode_utf16_le/be when a lead surrogate

crates/cli/src/tests/query_test.rs

@@ -4623,7 +4623,7 @@ fn test_query_with_no_patterns() {
     allocations::record(|| {
         let language = get_language("javascript");
         let query = Query::new(&language, "").unwrap();
-        assert!(query.capture_names().is_empty());
+        assert_eq!(query.capture_names(), [] as [&str; 0]);
         assert_eq!(query.pattern_count(), 0);
     });
 }

crates/config/Cargo.toml

@@ -5,7 +5,6 @@ description = "User configuration of tree-sitter's command line programs"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-config"

crates/config/src/tree_sitter_config.rs

@@ -31,6 +31,16 @@ pub struct IoError {
     pub path: Option<PathBuf>,
 }
 
+impl PartialEq for IoError {
+    fn eq(&self, other: &Self) -> bool {
+        self.path == other.path
+            && self.error.kind() == other.error.kind()
+            && self.error.raw_os_error() == other.error.raw_os_error()
+    }
+}
+
+impl Eq for IoError {}
+
 impl IoError {
     fn new(error: std::io::Error, path: Option<&Path>) -> Self {
         Self {

crates/generate/Cargo.toml

@@ -5,7 +5,6 @@ description = "Library for generating C source code from a tree-sitter grammar"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-generate"
@@ -33,12 +32,7 @@ log.workspace = true
 pathdiff = { optional = true, version = "0.2.3" }
 regex.workspace = true
 regex-syntax.workspace = true
-rquickjs = { features = [
-  "bindgen",
-  "loader",
-  "macro",
-  "phf",
-], optional = true, version = "0.12.2" }
+rquickjs = { features = [ "bindgen", "loader", "macro", "phf" ], optional = true, version = "0.13" }
 rustc-hash.workspace = true
 semver = { optional = true, workspace = true }
 serde.workspace = true

crates/generate/src/build_tables/build_parse_table.rs

@@ -83,7 +83,7 @@ struct ParseTableBuilder<'a> {
 
 pub type BuildTableResult<T> = Result<T, ParseTableBuilderError>;
 
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum ParseTableBuilderError {
     #[error("Unresolved conflict for symbol sequence:\n\n{0}")]
     Conflict(#[from] ConflictError),
@@ -97,7 +97,7 @@ pub enum ParseTableBuilderError {
     StateCount(usize),
 }
 
-#[derive(Default, Debug, Serialize, Error, Deserialize)]
+#[derive(Default, Debug, Serialize, Error, Deserialize, PartialEq, Eq)]
 pub struct ConflictError {
     pub symbol_sequence: Vec<String>,
     pub conflicting_lookahead: String,
@@ -105,7 +105,7 @@ pub struct ConflictError {
     pub possible_resolutions: Vec<Resolution>,
 }
 
-#[derive(Default, Debug, Serialize, Error, Deserialize)]
+#[derive(Default, Debug, Serialize, Error, Deserialize, PartialEq, Eq)]
 pub struct Interpretation {
     pub preceding_symbols: Vec<String>,
     pub variable_name: String,
@@ -118,14 +118,14 @@ pub struct Interpretation {
     pub requires_eof_lookahead: bool,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub enum Resolution {
     Precedence { symbols: Vec<String> },
     Associativity { symbols: Vec<String> },
     AddConflict { symbols: Vec<String> },
 }
 
-#[derive(Debug, Serialize, Deserialize, Error)]
+#[derive(Debug, Serialize, Deserialize, Error, PartialEq, Eq)]
 pub struct AmbiguousExtraError {
     pub parent_symbols: Vec<String>,
 }

crates/generate/src/generate.rs

@@ -73,7 +73,7 @@ pub const PARSER_HEADER: &str = include_str!("parser.h.inc");
 
 pub type GenerateResult<T> = Result<T, GenerateError>;
 
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum GenerateError {
     #[error("Error with specified path -- {0}")]
     GrammarPath(IoError),
@@ -107,6 +107,16 @@ pub struct IoError {
     pub path: Option<PathBuf>,
 }
 
+impl PartialEq for IoError {
+    fn eq(&self, other: &Self) -> bool {
+        self.path == other.path
+            && self.error.kind() == other.error.kind()
+            && self.error.raw_os_error() == other.error.raw_os_error()
+    }
+}
+
+impl Eq for IoError {}
+
 #[cfg(feature = "load")]
 impl IoError {
     fn new(error: std::io::Error, path: Option<&Path>) -> Self {
@@ -163,7 +173,7 @@ impl<'de> Deserialize<'de> for IoError {
 pub type LoadGrammarFileResult<T> = Result<T, LoadGrammarError>;
 
 #[cfg(feature = "load")]
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum LoadGrammarError {
     #[error("Path to a grammar file with `.js` or `.json` extension is required")]
     InvalidPath,
@@ -176,7 +186,7 @@ pub enum LoadGrammarError {
 }
 
 #[cfg(feature = "load")]
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum ParseVersionError {
     #[error("{0}")]
     Version(String),
@@ -190,7 +200,7 @@ pub enum ParseVersionError {
 pub type JSResult<T> = Result<T, JSError>;
 
 #[cfg(feature = "load")]
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum JSError {
     #[error("Failed to run `{runtime}` -- {error}")]
     JSRuntimeSpawn { runtime: String, error: String },

crates/generate/src/nfa.rs

@@ -458,7 +458,7 @@ impl Nfa {
 
     #[must_use]
     pub fn last_state_id(&self) -> u32 {
-        assert!(!self.states.is_empty());
+        assert_ne!(self.states, [] as [NfaState; 0]);
         self.states.len() as u32 - 1
     }
 }

crates/generate/src/node_types.rs

@@ -590,7 +590,7 @@ pub fn get_supertype_symbol_map(
 #[cfg(feature = "load")]
 pub type SuperTypeCycleResult<T> = Result<T, SuperTypeCycleError>;
 
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub struct SuperTypeCycleError {
     items: Vec<String>,
 }

crates/generate/src/parse_grammar.rs

@@ -115,7 +115,7 @@ pub struct GrammarJSON {
 
 pub type ParseGrammarResult<T> = Result<T, ParseGrammarError>;
 
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum ParseGrammarError {
     #[error("{0}")]
     Serialization(String),
@@ -402,13 +402,9 @@ impl RulePool {
                     .collect::<ParseGrammarResult<Vec<_>>>()?;
                 Ok(self.choice(&members))
             }
-            RuleJSON::SEQ { members } => {
-                let members = members
-                    .into_iter()
-                    .map(|m| self.parse_rule(m, is_token, diagnostics))
-                    .collect::<ParseGrammarResult<Vec<_>>>()?;
-                Ok(self.seq(&members))
-            }
+            RuleJSON::SEQ { members } => self.try_seq(members.into_iter(), |pool, m| {
+                pool.parse_rule(m, is_token, diagnostics)
+            }),
             RuleJSON::FIELD { name, content } => {
                 let content = self.parse_rule(*content, is_token, diagnostics)?;
                 let name = self.intern(&name);
@@ -418,7 +414,10 @@ impl RulePool {
                 let content = self.parse_rule(*content, is_token, diagnostics)?;
                 let repeat = self.repeat(content);
                 let blank = self.blank();
-                Ok(self.choice(&[repeat, blank]))
+                // Neither element is a `Choice` or equal to one another, so we
+                // can skip `self.choice`'s flattening loop.
+                let range = self.push_children(&[repeat, blank]);
+                Ok(self.push_node(Rule::Choice(range)))
             }
             RuleJSON::REPEAT1 { content } => {
                 let content = self.parse_rule(*content, is_token, diagnostics)?;

crates/generate/src/prepare_grammar.rs

@@ -14,9 +14,9 @@ use std::{
 };
 
 pub use expand_tokens::ExpandTokensError;
+#[cfg(test)]
+pub use expand_tokens::expand_tokens;
 pub use extract_tokens::ExtractTokensError;
-#[cfg(test)] // TODO: Is this defined in the proper place?
-pub use extract_tokens::LexicalToken;
 pub use flatten_grammar::FlattenGrammarError;
 use indexmap::IndexMap;
 pub use intern_symbols::InternSymbolsError;
@@ -26,11 +26,10 @@ use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
 use crate::{
-    grammars::{InputGrammar, PrecedenceEntry, ProductionStore},
+    grammars::{InputGrammar, PrecedenceEntry, ProductionStore, VariableType},
     strpool::StrPool,
 };
 
-pub use self::expand_tokens::expand_tokens;
 use self::{
     expand_repeats::{ExpandRepeatsError, expand_repeats},
     extract_default_aliases::extract_default_aliases,
@@ -43,13 +42,13 @@ use super::{
     Diagnostic,
     grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
     prepare_grammar::flatten_grammar::{FlattenState, assemble_syntax_grammar},
-    rules::{AliasMap, Precedence, Rule},
+    rules::{AliasMap, Precedence, Rule, RuleId},
     strpool::StrId,
 };
 
 pub type PrepareGrammarResult<T> = Result<T, PrepareGrammarError>;
 
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 #[error(transparent)]
 pub enum PrepareGrammarError {
     ValidatePrecedences(#[from] ValidatePrecedenceError),
@@ -109,6 +108,20 @@ pub struct PreparedGrammar {
     pub str_pool: StrPool,
 }
 
+/// A token extracted from the input grammar but not yet expanded into the lexical NFA.
+///
+/// Token extraction creates this while `root` still points to the original rule in
+/// the pool. [`PendingTokenExtraction::expand_and_commit`] passes them to `expand_tokens`
+/// before committing the deferred syntax rewrites.
+#[derive(Clone, Debug)]
+pub struct LexicalToken {
+    /// Generated for anon tokens, rule name for absorbed variables
+    pub name: StrId,
+    pub kind: VariableType,
+    /// Pool root defining this token
+    pub root: RuleId,
+}
+
 /// Transform an input grammar into separate components that are ready
 /// for parse table construction.
 pub fn prepare_grammar(
@@ -119,21 +132,17 @@ pub fn prepare_grammar(
     validate_indirect_recursion(&g)?;
 
     let interned_meta = intern_symbols(&mut g, diagnostics)?;
-    let mut ext_meta = extract_tokens(&mut g, &interned_meta)?;
+    let pending_tokens = extract_tokens(&mut g, &interned_meta)?;
+    let (mut ext_meta, lexical_grammar) = pending_tokens.expand_and_commit()?;
     expand_repeats(&mut g, &mut ext_meta)?;
 
     let mut state = FlattenState::default();
     let mut out = ProductionStore::default();
     flatten_grammar(&g, &ext_meta, &mut state, &mut out)?;
 
-    let lexical_grammar = expand_tokens(
-        &mut g.pool,
-        &ext_meta.lexical_variables,
-        &ext_meta.separator_roots,
-    )?;
-
-    let default_aliases = extract_default_aliases(&g, &ext_meta, &mut out);
-    let inlines = process_inlines(&g, &ext_meta, &mut out)?;
+    let default_aliases =
+        extract_default_aliases(&g, &ext_meta, &lexical_grammar.variables, &mut out);
+    let inlines = process_inlines(&g, &ext_meta, &lexical_grammar.variables, &mut out)?;
 
     let (syntax_grammar, str_pool) = assemble_syntax_grammar(g, ext_meta, out);
     Ok(PreparedGrammar {
@@ -308,8 +317,8 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()>
 mod tests {
     use super::*;
     use crate::{
-        grammars::Variable,
-        rules::{RuleId, RulePool},
+        grammars::{ProductionStep, Variable, VariableType},
+        rules::{RuleId, RulePool, Symbol},
     };
 
     #[test]
@@ -462,6 +471,352 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_token_body_shared_with_syntax() {
+        let grammar = build_grammar(|p| {
+            let a = leaf(p, "a");
+            let b = leaf(p, "b");
+            let shared = p.seq(&[a, b]);
+            let token = p.token(shared);
+            let program = {
+                let t = named(p, "t");
+                let x = named(p, "x");
+                p.seq(&[t, x])
+            };
+
+            vec![
+                Variable {
+                    name: p.intern("program"),
+                    root: program,
+                },
+                Variable {
+                    name: p.intern("t"),
+                    root: token,
+                },
+                Variable {
+                    name: p.intern("x"),
+                    root: shared,
+                },
+            ]
+        });
+
+        let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
+
+        let syntax_variables = prepared
+            .syntax_grammar
+            .variables
+            .iter()
+            .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
+            .collect::<Vec<_>>();
+        assert_eq!(
+            syntax_variables,
+            [("program", VariableType::Named), ("x", VariableType::Named),]
+        );
+
+        let lexical_variables = prepared
+            .lexical_grammar
+            .variables
+            .iter()
+            .map(|variable| {
+                (
+                    prepared.str_pool.resolve(variable.name),
+                    variable.kind,
+                    variable.implicit_precedence,
+                    variable.start_state,
+                )
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(
+            lexical_variables,
+            [
+                ("t", VariableType::Named, 0, 2),
+                ("a", VariableType::Anonymous, 2, 4),
+                ("b", VariableType::Anonymous, 2, 6),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_token_body_shared_with_syntax_reversed() {
+        let grammar = build_grammar(|p| {
+            let a = leaf(p, "a");
+            let b = leaf(p, "b");
+            let shared = p.seq(&[a, b]);
+            let token = p.token(shared);
+            let program = {
+                let x = named(p, "x");
+                let t = named(p, "t");
+                p.seq(&[x, t])
+            };
+
+            vec![
+                Variable {
+                    name: p.intern("program"),
+                    root: program,
+                },
+                Variable {
+                    name: p.intern("x"),
+                    root: shared,
+                },
+                Variable {
+                    name: p.intern("t"),
+                    root: token,
+                },
+            ]
+        });
+
+        let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
+
+        let syntax_variables = prepared
+            .syntax_grammar
+            .variables
+            .iter()
+            .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
+            .collect::<Vec<_>>();
+        assert_eq!(
+            syntax_variables,
+            [("program", VariableType::Named), ("x", VariableType::Named),]
+        );
+
+        let lexical_variables = prepared
+            .lexical_grammar
+            .variables
+            .iter()
+            .map(|variable| {
+                (
+                    prepared.str_pool.resolve(variable.name),
+                    variable.kind,
+                    variable.implicit_precedence,
+                    variable.start_state,
+                )
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(
+            lexical_variables,
+            [
+                ("a", VariableType::Anonymous, 2, 1),
+                ("b", VariableType::Anonymous, 2, 3),
+                ("t", VariableType::Named, 0, 6),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_separator_body_shared_with_syntax() {
+        let mut pool = RulePool::default();
+        let a = leaf(&mut pool, "a");
+        let b = leaf(&mut pool, "b");
+        let shared = pool.seq(&[a, b]);
+        let program = pool.intern("program");
+
+        let grammar = InputGrammar {
+            variables: vec![Variable {
+                name: program,
+                root: shared,
+            }],
+            extra_roots: vec![shared],
+            pool,
+            ..Default::default()
+        };
+
+        let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
+
+        let syntax_variables = prepared
+            .syntax_grammar
+            .variables
+            .iter()
+            .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
+            .collect::<Vec<_>>();
+        assert_eq!(syntax_variables, [("program", VariableType::Named)]);
+
+        let lexical_variables = prepared
+            .lexical_grammar
+            .variables
+            .iter()
+            .map(|variable| {
+                (
+                    prepared.str_pool.resolve(variable.name),
+                    variable.kind,
+                    variable.implicit_precedence,
+                    variable.start_state,
+                )
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(
+            lexical_variables,
+            [
+                ("a", VariableType::Anonymous, 2, 5),
+                ("b", VariableType::Anonymous, 2, 11),
+            ]
+        );
+
+        assert!(prepared.syntax_grammar.extra_symbols.is_empty());
+    }
+
+    #[test]
+    fn test_shared_extra_symbol_is_renumbered_once() {
+        let mut pool = RulePool::default();
+
+        let target_ref = named(&mut pool, "target");
+        let program = {
+            let kw = named(&mut pool, "kw");
+            let keep = named(&mut pool, "keep");
+            pool.seq(&[kw, keep, target_ref])
+        };
+        let kw = leaf(&mut pool, "keyword");
+        let keep = {
+            let k = leaf(&mut pool, "k");
+            let e = leaf(&mut pool, "e");
+            pool.seq(&[k, e])
+        };
+        let target = {
+            let t = leaf(&mut pool, "t");
+            let g = leaf(&mut pool, "g");
+            pool.seq(&[t, g])
+        };
+
+        let grammar = InputGrammar {
+            variables: vec![
+                Variable {
+                    name: pool.intern("program"),
+                    root: program,
+                },
+                Variable {
+                    name: pool.intern("kw"),
+                    root: kw,
+                },
+                Variable {
+                    name: pool.intern("keep"),
+                    root: keep,
+                },
+                Variable {
+                    name: pool.intern("target"),
+                    root: target,
+                },
+            ],
+            extra_roots: vec![target_ref],
+            pool,
+            ..Default::default()
+        };
+
+        let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
+
+        let syntax_variables = prepared
+            .syntax_grammar
+            .variables
+            .iter()
+            .map(|variable| (prepared.str_pool.resolve(variable.name), variable.kind))
+            .collect::<Vec<_>>();
+        assert_eq!(
+            syntax_variables,
+            [
+                ("program", VariableType::Named),
+                ("keep", VariableType::Named),
+                ("target", VariableType::Named),
+            ]
+        );
+
+        let lexical_variables = prepared
+            .lexical_grammar
+            .variables
+            .iter()
+            .map(|variable| {
+                (
+                    prepared.str_pool.resolve(variable.name),
+                    variable.kind,
+                    variable.implicit_precedence,
+                    variable.start_state,
+                )
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(
+            lexical_variables,
+            [
+                ("kw", VariableType::Named, 2, 7),
+                ("k", VariableType::Anonymous, 2, 9),
+                ("e", VariableType::Anonymous, 2, 11),
+                ("t", VariableType::Anonymous, 2, 13),
+                ("g", VariableType::Anonymous, 2, 15),
+            ]
+        );
+
+        assert_eq!(
+            prepared.syntax_grammar.extra_symbols,
+            [Symbol::non_terminal(2)]
+        );
+    }
+
+    #[test]
+    fn test_shared_syntax_subtree_is_renumbered_once() {
+        let grammar = build_grammar(|p| {
+            let pair = {
+                let thing = named(p, "thing");
+                let dash = leaf(p, "-");
+                p.seq(&[thing, dash])
+            };
+            let item_b = {
+                let semicolon = leaf(p, ";");
+                p.seq(&[pair, semicolon])
+            };
+            let program = {
+                let kw = named(p, "kw");
+                let item_a = named(p, "item_a");
+                let item_b = named(p, "item_b");
+                p.seq(&[kw, item_a, item_b])
+            };
+            let kw = leaf(p, "keyword");
+            let thing = {
+                let t = leaf(p, "t");
+                let u = leaf(p, "u");
+                p.seq(&[t, u])
+            };
+
+            vec![
+                Variable {
+                    name: p.intern("program"),
+                    root: program,
+                },
+                Variable {
+                    name: p.intern("kw"),
+                    root: kw,
+                },
+                Variable {
+                    name: p.intern("item_a"),
+                    root: pair,
+                },
+                Variable {
+                    name: p.intern("item_b"),
+                    root: item_b,
+                },
+                Variable {
+                    name: p.intern("thing"),
+                    root: thing,
+                },
+            ]
+        });
+
+        let prepared = prepare_grammar(grammar, &mut Vec::new()).unwrap();
+
+        let expected_steps = [
+            ProductionStep::pack(
+                Symbol::non_terminal(3),
+                Precedence::None,
+                None,
+                None,
+                None,
+                0,
+            ),
+            ProductionStep::pack(Symbol::terminal(1), Precedence::None, None, None, None, 0),
+        ];
+
+        let production_ids = prepared.syntax_grammar.variable_prod_ids(1);
+        assert_eq!(production_ids.len(), 1);
+
+        let production = prepared.syntax_grammar.production(production_ids.start);
+        assert_eq!(production.steps, expected_steps);
+        assert_eq!(production.dynamic_precedence, 0);
+    }
+
     fn named(pool: &mut RulePool, name: &str) -> RuleId {
         let id = pool.intern(name);
         pool.named_symbol(id)

crates/generate/src/prepare_grammar/expand_tokens.rs

@@ -5,7 +5,7 @@ use thiserror::Error;
 use crate::{
     grammars::{LexicalGrammar, LexicalVariable},
     nfa::{CharacterSet, Nfa, NfaState},
-    prepare_grammar::{extract_tokens::LexicalToken, pattern},
+    prepare_grammar::{LexicalToken, pattern},
     rules::{Precedence, Rule, RuleId, RulePool, Symbol},
 };
 

crates/generate/src/prepare_grammar/extract_default_aliases.rs

@@ -1,7 +1,7 @@
 use std::collections::BTreeMap;
 
 use crate::{
-    grammars::{InputGrammar, ProductionStore},
+    grammars::{InputGrammar, LexicalVariable, ProductionStore},
     prepare_grammar::extract_tokens::ExtractedGrammarMeta,
     rules::{Alias, AliasMap, Symbol, SymbolType},
 };
@@ -24,9 +24,10 @@ struct SymbolStatus {
 pub(super) fn extract_default_aliases(
     g: &InputGrammar,
     meta: &ExtractedGrammarMeta,
+    lexical_variables: &[LexicalVariable],
     out: &mut ProductionStore,
 ) -> AliasMap {
-    let mut terminal_status_list = vec![SymbolStatus::default(); meta.lexical_variables.len()];
+    let mut terminal_status_list = vec![SymbolStatus::default(); lexical_variables.len()];
     let mut non_terminal_status_list = vec![SymbolStatus::default(); g.variables.len()];
     let mut external_status_list = vec![SymbolStatus::default(); meta.external_tokens.len()];
 
@@ -171,7 +172,6 @@ mod tests {
     use super::*;
     use crate::{
         grammars::{Production, ProductionStep, Variable, VariableType},
-        prepare_grammar::extract_tokens::LexicalToken,
         rules::{Precedence, RulePool},
     };
 
@@ -181,20 +181,26 @@ mod tests {
         let dummy = pool.intern("_");
         let root = pool.blank();
 
+        let mut lexical_variables = Vec::new();
+        let t0 = add_lexical_variable(&mut pool, &mut lexical_variables, "t0");
+        let t1 = add_lexical_variable(&mut pool, &mut lexical_variables, "t1");
+        let t2 = add_lexical_variable(&mut pool, &mut lexical_variables, "t2");
+        let t3 = add_lexical_variable(&mut pool, &mut lexical_variables, "t3");
+
         // v1: every token aliased.
         let v1 = vec![
-            aliased(&mut pool, Symbol::terminal(0), "a1"),
-            aliased(&mut pool, Symbol::terminal(1), "a2"),
-            aliased(&mut pool, Symbol::terminal(2), "a3"),
-            aliased(&mut pool, Symbol::terminal(3), "a4"),
+            aliased(&mut pool, t0, "a1"),
+            aliased(&mut pool, t1, "a2"),
+            aliased(&mut pool, t2, "a3"),
+            aliased(&mut pool, t3, "a4"),
         ];
         // v2: t0 same alias, t1 unaliased, t2 aliased differently, t3 aliased twice as a6
         let v2 = vec![
-            aliased(&mut pool, Symbol::terminal(0), "a1"),
-            plain(Symbol::terminal(1)),
-            aliased(&mut pool, Symbol::terminal(2), "a5"),
-            aliased(&mut pool, Symbol::terminal(3), "a6"),
-            aliased(&mut pool, Symbol::terminal(3), "a6"),
+            aliased(&mut pool, t0, "a1"),
+            plain(t1),
+            aliased(&mut pool, t2, "a5"),
+            aliased(&mut pool, t3, "a6"),
+            aliased(&mut pool, t3, "a6"),
         ];
 
         let mut out = ProductionStore::default();
@@ -216,18 +222,8 @@ mod tests {
             pool,
             ..Default::default()
         };
-        let meta = ExtractedGrammarMeta {
-            lexical_variables: (0..4)
-                .map(|_| LexicalToken {
-                    name: dummy,
-                    kind: VariableType::Anonymous,
-                    root,
-                })
-                .collect(),
-            ..Default::default()
-        };
-
-        let pool_map = extract_default_aliases(&g, &meta, &mut out);
+        let meta = ExtractedGrammarMeta::default();
+        let pool_map = extract_default_aliases(&g, &meta, &lexical_variables, &mut out);
 
         // t0 -> a1, t2 -> a3 (v1 wins the tie from appearing first), t3 -> a6 (used twice)
         // t1 -> none (appears unaliased in v2).
@@ -259,6 +255,21 @@ mod tests {
         assert_eq!(step_alias(8), None); //               v2 t3(a6) = default
     }
 
+    fn add_lexical_variable(
+        pool: &mut RulePool,
+        variables: &mut Vec<LexicalVariable>,
+        name: &str,
+    ) -> Symbol {
+        let symbol = Symbol::terminal(variables.len());
+        variables.push(LexicalVariable {
+            name: pool.intern(name),
+            kind: VariableType::Anonymous,
+            implicit_precedence: 0,
+            start_state: 0,
+        });
+        symbol
+    }
+
     fn aliased(pool: &mut RulePool, symbol: Symbol, name: &str) -> ProductionStep {
         let value = pool.intern(name);
         ProductionStep::pack(

crates/generate/src/prepare_grammar/extract_tokens.rs

@@ -4,8 +4,12 @@ use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
 use crate::{
-    grammars::{ExternalToken, InputGrammar, VariableType},
-    prepare_grammar::intern_symbols::InternedGrammarMeta,
+    grammars::{ExternalToken, InputGrammar, LexicalGrammar, VariableType},
+    prepare_grammar::{
+        LexicalToken,
+        expand_tokens::{ExpandTokensResult, expand_tokens},
+        intern_symbols::InternedGrammarMeta,
+    },
     rules::{MetadataParams, Rule, RuleId, RulePool, Symbol},
     strpool::StrId,
 };
@@ -58,22 +62,13 @@ impl std::fmt::Display for NonTerminalWordTokenError {
     }
 }
 
-/// A single extracted token.
-#[derive(Clone, Debug)]
-pub struct LexicalToken {
-    /// Generated for anon tokens, rule name for absorbed variables
-    pub name: StrId,
-    pub kind: VariableType,
-    /// Pool root defining this token
-    pub root: RuleId,
-}
-
-/// The extra pass's outputs besides the in-place rewrites.
+/// Syntax grammar metadata produced during token extraction.
+///
+/// Its symbols are resolved to their final indices before deferred pool rewrites
+/// are committed.
 #[derive(Clone, Debug, Default)]
 pub(super) struct ExtractedGrammarMeta {
     pub kinds: Vec<VariableType>,
-    pub lexical_variables: Vec<LexicalToken>,
-    pub separator_roots: Vec<RuleId>,
     pub extra_symbols: Vec<Symbol>,
     pub external_tokens: Vec<ExternalToken>,
     pub reserved_sets: Vec<(StrId, Vec<Symbol>)>,
@@ -89,6 +84,8 @@ struct TokenExtractor {
     lexical: Vec<LexicalToken>,
     usage_counts: Vec<u32>,
     memo: FxHashMap<u64, Vec<u32>>,
+    /// Terminal rewrites deferred until lexical expansion has consumed the original pool nodes.
+    rewrites: Vec<(RuleId, u32)>,
 }
 
 impl TokenExtractor {
@@ -137,7 +134,8 @@ impl TokenExtractor {
         Ok(index)
     }
 
-    /// In-place token extraction over one root.
+    /// Token extraction over one root. Records each terminal rewrite so that all
+    /// roots are inspected before any shared pool node is mutated.
     ///  - `String`/`Pattern`: always extracted
     ///  - `token(...)`: metadata extracts the inner child when no other metadata
     ///    params are set, otherwise the whole metadata node
@@ -163,7 +161,7 @@ impl TokenExtractor {
                         &mut aux_token_count,
                         is_first,
                     )?;
-                    pool.set_node(id, Rule::from(Symbol::terminal(i as usize)));
+                    self.rewrites.push((id, i));
                 }
                 Rule::Pattern(..) => {
                     let i = self.extract_token(
@@ -174,7 +172,7 @@ impl TokenExtractor {
                         &mut aux_token_count,
                         is_first,
                     )?;
-                    pool.set_node(id, Rule::from(Symbol::terminal(i as usize)));
+                    self.rewrites.push((id, i));
                 }
                 Rule::Metadata { params, rule } => {
                     let p = pool.params(params);
@@ -200,7 +198,7 @@ impl TokenExtractor {
                             &mut aux_token_count,
                             is_first,
                         )?;
-                        pool.set_node(id, Rule::from(Symbol::terminal(i as usize)));
+                        self.rewrites.push((id, i));
                     } else {
                         stack.push(rule);
                     }
@@ -217,7 +215,7 @@ impl TokenExtractor {
         Ok(())
     }
 
-    /// Structural lookup
+    /// Find the lexical token whose body is structurally equal to `root`.
     fn find(&self, pool: &RulePool, root: RuleId) -> Option<u32> {
         self.memo.get(&pool.subtree_hash(root)).and_then(|cands| {
             cands
@@ -226,12 +224,122 @@ impl TokenExtractor {
                 .find(|&i| pool.subtree_eq(self.lexical[i as usize].root, root))
         })
     }
+
+    /// Sort pending rewrites for lookup and coalesce duplicate visits to shared nodes.
+    fn finalize_rewrites(&mut self) {
+        self.rewrites
+            .sort_unstable_by_key(|(id, token)| (id.index(), *token));
+        self.rewrites.dedup();
+        // Discovery only observes unmodified subtrees, so a node can map to just
+        // one terminal. The binary search in [`Self::symbol_after_rewrites`] and
+        // the commit loop both rely on the keys being unique.
+        debug_assert!(
+            self.rewrites
+                .windows(2)
+                .all(|w| w[0].0.index() != w[1].0.index())
+        );
+    }
+
+    /// Return the symbol `id` will contain after committing pending rewrites, without
+    /// mutating the pool.
+    fn symbol_after_rewrites(&self, pool: &RulePool, id: RuleId) -> Option<Symbol> {
+        self.rewrites
+            .binary_search_by_key(&id.index(), |(id, _)| id.index())
+            .ok()
+            .map(|i| Symbol::terminal(self.rewrites[i].1 as usize))
+            .or_else(|| pool.node(id).symbol())
+    }
 }
 
-pub(super) fn extract_tokens(
-    g: &mut InputGrammar,
+/// Token extraction's pending result.
+///
+/// A mutable borrow is held onto the [`InputGrammar`] to prevent other passes from
+/// changing the pool while token and separator roots still refer to their original
+/// bodies. [`Self::expand_and_commit`] consumes this, expands the bodies, and commits the
+/// deferred terminal rewrites and renumbers symbols.
+pub(super) struct PendingTokenExtraction<'g> {
+    grammar: &'g mut InputGrammar,
+    meta: ExtractedGrammarMeta,
+    lexical_variables: Vec<LexicalToken>,
+    separator_roots: Vec<RuleId>,
+    rewrites: Vec<(RuleId, u32)>,
+    syntax_variable_replacements: FxHashMap<u32, u32>,
+    syntax_variable_shift: Vec<u32>,
+    stack: Vec<RuleId>,
+}
+
+impl PendingTokenExtraction<'_> {
+    /// Expand the token and separator roots, then commit the terminal rewrites and
+    /// renumber the remaining syntax symbols.
+    pub(super) fn expand_and_commit(
+        self,
+    ) -> ExpandTokensResult<(ExtractedGrammarMeta, LexicalGrammar)> {
+        let Self {
+            grammar,
+            meta,
+            lexical_variables,
+            separator_roots,
+            rewrites,
+            syntax_variable_replacements,
+            syntax_variable_shift,
+            mut stack,
+        } = self;
+
+        let lexical_grammar =
+            expand_tokens(&mut grammar.pool, &lexical_variables, &separator_roots)?;
+
+        for (id, token_index) in rewrites {
+            grammar
+                .pool
+                .set_node(id, Rule::from(Symbol::terminal(token_index as usize)));
+        }
+
+        let replace_symbol = |symbol: Symbol| {
+            if !symbol.is_non_terminal() {
+                return symbol;
+            }
+
+            syntax_variable_replacements.get(&symbol.index).map_or_else(
+                || {
+                    Symbol::non_terminal(
+                        symbol.index as usize
+                            - syntax_variable_shift[symbol.index as usize] as usize,
+                    )
+                },
+                |&token_index| Symbol::terminal(token_index as usize),
+            )
+        };
+
+        if !syntax_variable_replacements.is_empty() {
+            let mut renumbered_nodes = vec![false; grammar.pool.node_count()];
+            for variable in &grammar.variables {
+                renumber_root(
+                    &mut grammar.pool,
+                    variable.root,
+                    &replace_symbol,
+                    &mut stack,
+                    &mut renumbered_nodes,
+                );
+            }
+            for &root in &grammar.external_roots {
+                renumber_root(
+                    &mut grammar.pool,
+                    root,
+                    &replace_symbol,
+                    &mut stack,
+                    &mut renumbered_nodes,
+                );
+            }
+        }
+
+        Ok((meta, lexical_grammar))
+    }
+}
+
+pub(super) fn extract_tokens<'g>(
+    g: &'g mut InputGrammar,
     interned: &InternedGrammarMeta,
-) -> ExtractTokensResult<ExtractedGrammarMeta> {
+) -> ExtractTokensResult<PendingTokenExtraction<'g>> {
     let mut extractor = TokenExtractor::default();
     let mut stack = Vec::new();
 
@@ -241,6 +349,7 @@ pub(super) fn extract_tokens(
     for (&root, &(name, _)) in g.external_roots.iter().zip(&interned.external_tokens) {
         extractor.extract_in_root(&mut g.pool, root, name, false, &mut stack)?;
     }
+    extractor.finalize_rewrites();
 
     // If a variable's entire rule was extracted as a token and that token didn't
     // appear within any other rule, then remove that variable from the syntax
@@ -257,7 +366,7 @@ pub(super) fn extract_tokens(
     retained.push(g.variables[0]);
     kinds.push(interned.kinds[0]);
     for (i, v) in g.variables.iter().enumerate().skip(1) {
-        if let Some(sym) = g.pool.node(v.root).symbol()
+        if let Some(sym) = extractor.symbol_after_rewrites(&g.pool, v.root)
             && sym.is_terminal()
             && extractor.usage_counts[sym.index as usize] == 1
         {
@@ -294,13 +403,6 @@ pub(super) fn extract_tokens(
         )
     };
 
-    for v in &g.variables {
-        renumber_root(&mut g.pool, v.root, &replace_symbol, &mut stack);
-    }
-    for &root in &g.external_roots {
-        renumber_root(&mut g.pool, root, &replace_symbol, &mut stack);
-    }
-
     // Renumber each conflict through absorption, then canonicalize
     let conflicts = interned
         .conflicts
@@ -333,11 +435,18 @@ pub(super) fn extract_tokens(
 
     let inline = interned.inline.iter().map(|&s| replace_symbol(s)).collect();
 
+    // Resolve each metadata root to the symbol it will represent after token extraction.
+    // If that symbol is still a non terminal, adjust its index for any grammar
+    // variables removed above. The syntax rules are renumbered only after all metadata
+    // roots have been resolved.
     let mut separator_roots = Vec::new();
     let mut extra_symbols = Vec::with_capacity(g.extra_roots.len());
     for &root in &g.extra_roots {
-        if let Some(s) = g.pool.node(root).symbol() {
-            extra_symbols.push(replace_symbol(s));
+        if let Some(s) = extractor
+            .symbol_after_rewrites(&g.pool, root)
+            .map(replace_symbol)
+        {
+            extra_symbols.push(s);
         } else if let Some(i) = extractor.find(&g.pool, root) {
             extra_symbols.push(Symbol::terminal(i as usize));
         } else {
@@ -347,7 +456,10 @@ pub(super) fn extract_tokens(
 
     let mut external_tokens = Vec::with_capacity(g.external_roots.len());
     for (&root, &(name, kind)) in g.external_roots.iter().zip(&interned.external_tokens) {
-        let Some(s) = g.pool.node(root).symbol() else {
+        let Some(s) = extractor
+            .symbol_after_rewrites(&g.pool, root)
+            .map(replace_symbol)
+        else {
             Err(ExtractTokensError::NonSymbolExternalToken)?
         };
         if s.is_non_terminal() {
@@ -397,8 +509,11 @@ pub(super) fn extract_tokens(
     for set in &g.reserved_sets {
         let mut symbols = Vec::with_capacity(set.roots.len());
         for &root in &set.roots {
-            if let Some(s) = g.pool.node(root).symbol() {
-                symbols.push(replace_symbol(s));
+            if let Some(s) = extractor
+                .symbol_after_rewrites(&g.pool, root)
+                .map(replace_symbol)
+            {
+                symbols.push(s);
             } else if let Some(i) = extractor.find(&g.pool, root) {
                 symbols.push(Symbol::terminal(i as usize));
             } else {
@@ -416,10 +531,8 @@ pub(super) fn extract_tokens(
         reserved_sets.push((set.name, symbols));
     }
 
-    Ok(ExtractedGrammarMeta {
+    let meta = ExtractedGrammarMeta {
         kinds,
-        lexical_variables: extractor.lexical,
-        separator_roots,
         extra_symbols,
         external_tokens,
         reserved_sets,
@@ -427,18 +540,34 @@ pub(super) fn extract_tokens(
         conflicts,
         inline,
         word,
+    };
+    Ok(PendingTokenExtraction {
+        grammar: g,
+        meta,
+        lexical_variables: extractor.lexical,
+        separator_roots,
+        rewrites: extractor.rewrites,
+        syntax_variable_replacements: replacements,
+        syntax_variable_shift: shift,
+        stack,
     })
 }
 
+/// Renumber nodes reachable from `root`, skipping every node already reached via
+/// another root in this renumbering pass.
 fn renumber_root(
     pool: &mut RulePool,
     root: RuleId,
     replace: &impl Fn(Symbol) -> Symbol,
     stack: &mut Vec<RuleId>,
+    visited: &mut [bool],
 ) {
     stack.clear();
     stack.push(root);
     while let Some(id) = stack.pop() {
+        if std::mem::replace(&mut visited[id.index()], true) {
+            continue;
+        }
         match pool.node(id) {
             Rule::Sym { kind, index } => {
                 let s = Symbol { kind, index };
@@ -498,7 +627,36 @@ mod test {
             Variable { name: pool.intern("rule_3"), root: r3 },
         ];
         let mut grammar = pool_grammar(pool, variables);
-        let ext = extract(&mut grammar).unwrap();
+        let pending = extract_pending(&mut grammar).unwrap();
+
+        // Token extraction must preserve each original token body until lexical
+        // expansion runs.
+        let roots = pending
+            .lexical_variables
+            .iter()
+            .map(|v| v.root)
+            .collect::<Vec<_>>();
+        {
+            let pool = &mut pending.grammar.pool;
+
+            let e0 = str(pool, "a");
+            assert!(pool.subtree_eq(roots[0], e0));
+
+            let e1 = pat(pool, "b");
+            assert!(pool.subtree_eq(roots[1], e1));
+
+            let e2 = {
+                let (c, d) = (str(pool, "c"), str(pool, "d"));
+                let cd = pool.choice(&[c, d]);
+                pool.repeat(cd)
+            };
+            assert!(pool.subtree_eq(roots[2], e2));
+
+            let e3 = pat(pool, "e");
+            assert!(pool.subtree_eq(roots[3], e3));
+        }
+
+        let (ext, lexical_grammar) = pending.expand_and_commit().unwrap();
 
         // rule_1 was absorbed into the lexical grammar, rule_0, rule_2, and rule_3 remain
         let names = grammar
@@ -555,8 +713,8 @@ mod test {
 
         // `/e/` is used in exactly one place (as rule_1's whole body), so rule_1
         // was absorbed into the lexical grammar (and donated its name to the token).
-        let lex = ext
-            .lexical_variables
+        let lex = lexical_grammar
+            .variables
             .iter()
             .map(|v| (grammar.pool.resolve(v.name), v.kind))
             .collect::<Vec<_>>();
@@ -569,24 +727,6 @@ mod test {
                 ("rule_1", VariableType::Named),
             ]
         );
-        let roots = ext
-            .lexical_variables
-            .iter()
-            .map(|v| v.root)
-            .collect::<Vec<_>>();
-        let e0 = str(&mut grammar.pool, "a");
-        assert!(grammar.pool.subtree_eq(roots[0], e0));
-        let e1 = pat(&mut grammar.pool, "b");
-        assert!(grammar.pool.subtree_eq(roots[1], e1));
-        let e2 = {
-            let p = &mut grammar.pool;
-            let (c, d) = (str(p, "c"), str(p, "d"));
-            let cd = p.choice(&[c, d]);
-            p.repeat(cd)
-        };
-        assert!(grammar.pool.subtree_eq(roots[2], e2));
-        let e3 = pat(&mut grammar.pool, "e");
-        assert!(grammar.pool.subtree_eq(roots[3], e3));
     }
 
     #[test]
@@ -600,7 +740,13 @@ mod test {
             root: r0,
         }];
         let mut grammar = pool_grammar(pool, variables);
-        let ext = extract(&mut grammar).unwrap();
+        let pending = extract_pending(&mut grammar).unwrap();
+
+        let root = pending.lexical_variables[0].root;
+        let expected = str(&mut pending.grammar.pool, "hello");
+        assert!(pending.grammar.pool.subtree_eq(root, expected));
+
+        let (_, lexical_grammar) = pending.expand_and_commit().unwrap();
 
         let names = grammar
             .variables
@@ -616,14 +762,12 @@ mod test {
             }
         );
 
-        let lex = ext
-            .lexical_variables
+        let lex = lexical_grammar
+            .variables
             .iter()
             .map(|v| (grammar.pool.resolve(v.name), v.kind))
             .collect::<Vec<_>>();
         assert_eq!(lex, [("hello", VariableType::Anonymous)]);
-        let e = str(&mut grammar.pool, "hello");
-        assert!(grammar.pool.subtree_eq(ext.lexical_variables[0].root, e));
     }
 
     #[test]
@@ -648,15 +792,18 @@ mod test {
         ];
         let mut grammar = pool_grammar(pool, variables);
         grammar.extra_roots = vec![sep, extra_ref];
-        let ext = extract(&mut grammar).unwrap();
+        let pending = extract_pending(&mut grammar).unwrap();
 
-        // comment's `//.*` was single use-> absorbed by terminal(1), so the extra ref
-        // resolves to it
-        assert_eq!(ext.extra_symbols, [Symbol::terminal(1)]);
+        // The single use `comment` rule `//.*` is absorbed as `terminal(1)`, so
+        // its extra reference becomes `extra_symbols[0]`.
+        assert_eq!(pending.separator_roots.len(), 1);
+        let separator_root = pending.separator_roots[0];
         // the " " string routes to separators, not a token symbol
-        assert_eq!(ext.separator_roots.len(), 1);
-        let e = str(&mut grammar.pool, " ");
-        assert!(grammar.pool.subtree_eq(ext.separator_roots[0], e));
+        let expected = str(&mut pending.grammar.pool, " ");
+        assert!(pending.grammar.pool.subtree_eq(separator_root, expected));
+
+        let (ext, _) = pending.expand_and_commit().unwrap();
+        assert_eq!(ext.extra_symbols, [Symbol::terminal(1)]);
     }
 
     #[test]
@@ -694,7 +841,7 @@ mod test {
         let rule_2 = pool.intern("rule_2");
         let mut grammar = pool_grammar(pool, variables);
         grammar.external_roots = vec![e0, ea, er2];
-        let ext = extract(&mut grammar).unwrap();
+        let (ext, _) = extract(&mut grammar).unwrap();
 
         assert_eq!(
             ext.external_tokens,
@@ -778,7 +925,13 @@ mod test {
             },
         ];
         let mut grammar = pool_grammar(pool, variables);
-        let ext = extract(&mut grammar).unwrap();
+        let pending = extract_pending(&mut grammar).unwrap();
+
+        let root = pending.lexical_variables[0].root;
+        let expected = str(&mut pending.grammar.pool, "a");
+        assert!(pending.grammar.pool.subtree_eq(root, expected));
+
+        let (ext, lexical_grammar) = pending.expand_and_commit().unwrap();
 
         let names = grammar
             .variables
@@ -798,14 +951,12 @@ mod test {
             Rule::Sym { kind: SymbolType::Terminal, index: 0 }
         );
 
-        let lex = ext
-            .lexical_variables
+        let lex = lexical_grammar
+            .variables
             .iter()
             .map(|v| (grammar.pool.resolve(v.name), v.kind))
             .collect::<Vec<_>>();
         assert_eq!(lex, [("a", VariableType::Anonymous)]);
-        let e = str(&mut grammar.pool, "a");
-        assert!(grammar.pool.subtree_eq(ext.lexical_variables[0].root, e));
     }
 
     #[test]
@@ -860,8 +1011,15 @@ mod test {
         }
     }
 
-    fn extract(g: &mut InputGrammar) -> ExtractTokensResult<ExtractedGrammarMeta> {
+    fn extract_pending(g: &mut InputGrammar) -> ExtractTokensResult<PendingTokenExtraction<'_>> {
         let meta = intern_symbols(g, &mut Vec::new()).unwrap();
         extract_tokens(g, &meta)
     }
+
+    fn extract(
+        g: &mut InputGrammar,
+    ) -> ExtractTokensResult<(ExtractedGrammarMeta, LexicalGrammar)> {
+        let pending = extract_pending(g)?;
+        Ok(pending.expand_and_commit().unwrap())
+    }
 }

crates/generate/src/prepare_grammar/process_inlines.rs

@@ -5,7 +5,10 @@ use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
 use crate::{
-    grammars::{InlinedProductionMap, InputGrammar, Production, ProductionStep, ProductionStore},
+    grammars::{
+        InlinedProductionMap, InputGrammar, LexicalVariable, Production, ProductionStep,
+        ProductionStore,
+    },
     prepare_grammar::extract_tokens::ExtractedGrammarMeta,
     rules::{Precedence, Symbol, SymbolType},
 };
@@ -180,6 +183,7 @@ pub enum ProcessInlinesError {
 pub(super) fn process_inlines(
     g: &InputGrammar,
     meta: &ExtractedGrammarMeta,
+    lexical_variables: &[LexicalVariable],
     out: &mut ProductionStore,
 ) -> ProcessInlinesResult<InlinedProductionMap> {
     if meta.inline.is_empty() {
@@ -194,7 +198,7 @@ pub(super) fn process_inlines(
             ))?,
             SymbolType::Terminal => Err(ProcessInlinesError::Token(
                 g.pool
-                    .resolve(meta.lexical_variables[symbol.index as usize].name)
+                    .resolve(lexical_variables[symbol.index as usize].name)
                     .to_string(),
             ))?,
             SymbolType::NonTerminal if symbol.index == 0 => Err(ProcessInlinesError::FirstRule(
@@ -235,7 +239,6 @@ mod tests {
     use super::*;
     use crate::{
         grammars::VariableType,
-        prepare_grammar::extract_tokens::LexicalToken,
         rules::{Alias, Associativity, RulePool, Symbol},
     };
 
@@ -266,12 +269,13 @@ mod tests {
             ],
         );
 
-        let g = InputGrammar::default();
+        let mut g = InputGrammar::default();
         let meta = ExtractedGrammarMeta {
             inline: vec![Symbol::non_terminal(1)],
             ..Default::default()
         };
-        let map = process_inlines(&g, &meta, &mut out).unwrap();
+        let lexical_variables = make_lexical_variables_through(&mut g.pool, 14);
+        let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap();
         let prod0 = out.var_prods[0].0;
 
         // Nothing to inline at step 0.
@@ -336,7 +340,7 @@ mod tests {
         add_variable(&mut out, &[(vec![plain(Symbol::terminal(15))], 0)]);
         add_variable(&mut out, &[(vec![plain(Symbol::terminal(16))], 0)]);
 
-        let g = InputGrammar::default();
+        let mut g = InputGrammar::default();
         let meta = ExtractedGrammarMeta {
             inline: vec![
                 Symbol::non_terminal(1),
@@ -345,7 +349,8 @@ mod tests {
             ],
             ..Default::default()
         };
-        let map = process_inlines(&g, &meta, &mut out).unwrap();
+        let lexical_variables = make_lexical_variables_through(&mut g.pool, 16);
+        let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap();
         let prod0 = out.var_prods[0].0;
 
         let (ids, prods) = inlined(&out, &map, prod0, 1).unwrap();
@@ -441,12 +446,13 @@ mod tests {
         );
         add_variable(&mut out, &[(vec![plain(Symbol::terminal(13))], 0)]);
 
-        let g = InputGrammar::default();
+        let mut g = InputGrammar::default();
         let meta = ExtractedGrammarMeta {
             inline: vec![Symbol::non_terminal(1), Symbol::non_terminal(2)],
             ..Default::default()
         };
-        let map = process_inlines(&g, &meta, &mut out).unwrap();
+        let lexical_variables = make_lexical_variables_through(&mut g.pool, 13);
+        let map = process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap();
         let prod0 = out.var_prods[0].0;
 
         let (ids, prods) = inlined(&out, &map, prod0, 0).unwrap();
@@ -521,28 +527,42 @@ mod tests {
     fn test_error_when_inlining_tokens() {
         let mut pool = RulePool::default();
         let name = pool.intern("something");
-        let root = pool.blank();
         let g = InputGrammar {
             pool,
             ..Default::default()
         };
         let meta = ExtractedGrammarMeta {
             inline: vec![Symbol::terminal(0)],
-            lexical_variables: vec![LexicalToken {
-                name,
-                kind: VariableType::Named,
-                root,
-            }],
             ..Default::default()
         };
+        let lexical_variables = [LexicalVariable {
+            name,
+            kind: VariableType::Named,
+            implicit_precedence: 0,
+            start_state: 0,
+        }];
         let mut out = ProductionStore::default();
 
-        let result = process_inlines(&g, &meta, &mut out);
+        let result = process_inlines(&g, &meta, &lexical_variables, &mut out);
         assert!(result.is_err());
         let err = result.unwrap_err();
         assert_eq!(err, ProcessInlinesError::Token("something".to_string()));
     }
 
+    fn make_lexical_variables_through(
+        pool: &mut RulePool,
+        last_index: usize,
+    ) -> Vec<LexicalVariable> {
+        (0..=last_index)
+            .map(|i| LexicalVariable {
+                name: pool.intern(&format!("t{i}")),
+                kind: VariableType::Anonymous,
+                implicit_precedence: 0,
+                start_state: 0,
+            })
+            .collect()
+    }
+
     /// Append one variable's productions to `out` and record its production id range.
     fn add_variable(out: &mut ProductionStore, prods: &[(Vec<ProductionStep>, i32)]) {
         let start = out.productions.len() as u32;
@@ -614,6 +634,7 @@ mod tests {
                 crate::grammars::Variable { name, root }
             })
             .to_vec();
+        let lexical_variables = make_lexical_variables_through(&mut pool, 10);
         let g = InputGrammar {
             pool,
             variables,
@@ -624,7 +645,7 @@ mod tests {
             ..Default::default()
         };
         assert_eq!(
-            process_inlines(&g, &meta, &mut out).unwrap_err(),
+            process_inlines(&g, &meta, &lexical_variables, &mut out).unwrap_err(),
             ProcessInlinesError::NoReachableProductions("rule1".to_string())
         );
     }

crates/generate/src/render.rs

@@ -33,7 +33,7 @@ const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15;
 
 pub type RenderResult<T> = Result<T, RenderError>;
 
-#[derive(Debug, Error, Serialize, Deserialize)]
+#[derive(Debug, Error, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RenderError {
     #[error("Parse table action count {0} exceeds maximum value of {max}", max=u16::MAX)]
     ParseTable(usize),

crates/generate/src/rules.rs

@@ -174,6 +174,8 @@ pub struct RulePool {
     children: Vec<RuleId>,
     params: Vec<MetadataParams>,
     str_pool: StrPool,
+    /// Reusable walk scratch for choice flattening
+    scratch: Vec<RuleId>,
 }
 
 impl RulePool {
@@ -182,6 +184,11 @@ impl RulePool {
         self.nodes[id.index()]
     }
 
+    #[must_use]
+    pub const fn node_count(&self) -> usize {
+        self.nodes.len()
+    }
+
     pub fn set_node(&mut self, id: RuleId, node: Rule) {
         self.nodes[id.index()] = node;
     }
@@ -304,27 +311,58 @@ impl RulePool {
         self.push_node(Rule::Eof)
     }
 
+    #[cfg_attr(not(test), expect(dead_code))]
     pub fn seq(&mut self, ids: &[RuleId]) -> RuleId {
         let range = self.push_children(ids);
         self.push_node(Rule::Seq(range))
     }
 
+    pub fn try_seq<T, E>(
+        &mut self,
+        items: impl ExactSizeIterator<Item = T>,
+        mut f: impl FnMut(&mut Self, T) -> Result<RuleId, E>,
+    ) -> Result<RuleId, E> {
+        let len = items.len();
+        let start = self.children.len();
+        self.children.resize(start + len, RuleId(0));
+        for (i, item) in items.enumerate() {
+            let id = f(self, item)?;
+            self.children[start + i] = id;
+        }
+
+        let range = RuleIdRange {
+            start: start as u32,
+            len: len as u32,
+        };
+        Ok(self.push_node(Rule::Seq(range)))
+    }
+
     /// Flatten nested choices and de-dup structurally, keeping a `Choice` node
-    /// event for a single element
+    /// even for a single element
     pub fn choice(&mut self, ids: &[RuleId]) -> RuleId {
-        let mut elements: Vec<RuleId> = Vec::with_capacity(ids.len());
-        let mut stack: Vec<RuleId> = Vec::with_capacity(ids.len());
+        // Elements build directly at the children tail. The walk only reads
+        // existing nodes, so nothing else appends while the range grows
+        let start = self.children.len();
+        let mut stack = std::mem::take(&mut self.scratch);
         stack.extend(ids.iter().rev());
         while let Some(id) = stack.pop() {
             if let Rule::Choice(range) = self.node(id) {
                 let base = stack.len();
                 stack.extend_from_slice(self.child_slice(range));
                 stack[base..].reverse();
-            } else if !elements.iter().any(|&e| self.subtree_eq(e, id)) {
-                elements.push(id);
+            } else if !self.children[start..]
+                .iter()
+                .copied()
+                .any(|e| self.subtree_eq(e, id))
+            {
+                self.children.push(id);
             }
         }
-        let range = self.push_children(&elements);
+        self.scratch = stack;
+        let range = RuleIdRange {
+            start: start as u32,
+            len: (self.children.len() - start) as u32,
+        };
         self.push_node(Rule::Choice(range))
     }
 

crates/highlight/Cargo.toml

@@ -5,7 +5,6 @@ description = "Library for performing syntax highlighting with Tree-sitter"
 authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Tim Clem <timothy.clem@gmail.com>" ]
 edition.workspace = true
 rust-version.workspace = true
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-highlight"

crates/language/Cargo.toml

@@ -5,7 +5,6 @@ version = "0.1.8"
 authors.workspace = true
 edition.workspace = true
 rust-version = "1.90"
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-language"

crates/loader/Cargo.toml

@@ -5,7 +5,6 @@ description = "Locates, builds, and loads tree-sitter grammars at runtime"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-loader"

crates/loader/src/loader.rs

@@ -89,7 +89,7 @@ pub enum LoaderError {
     )]
     LockFileTimeout(PathBuf),
     #[error("Failed to execute curl for {0} -- {1}")]
-    Curl(String, std::io::Error),
+    Curl(String, IoError),
     #[error("Failed to load language in current directory:\n{0}")]
     CurrentDirectoryLoad(Box<Self>),
     #[error("External file path {0} is outside of parser directory {1}")]
@@ -120,8 +120,12 @@ pub enum LoaderError {
     Symbol(SymbolError),
     #[error(transparent)]
     Tags(#[from] TagsError),
-    #[error("Failed to execute tar for {0} -- {1}")]
-    Tar(String, std::io::Error),
+    #[error(
+        "Failed to execute tar for {path} -- {error}",
+        path = .0.path.as_deref().unwrap_or_else(|| Path::new("")).display(),
+        error = .0.error,
+    )]
+    Tar(IoError),
     #[error("Unknown scope '{0}'")]
     UnknownScope(String),
     #[error("Failed to download {tool} from {url}")]
@@ -134,9 +138,9 @@ pub enum LoaderError {
     #[error(transparent)]
     Wasm(#[from] WasmError),
     #[error("Failed to run wasi-sdk clang -- {0}")]
-    WasmCompiler(std::io::Error),
+    WasmCompiler(IoError),
     #[error("Failed to run wasm-opt -- {0}")]
-    WasmOptimizer(std::io::Error),
+    WasmOptimizer(IoError),
     #[error("wasi-sdk clang command failed: {0}")]
     WasmCompilation(String),
     #[error("wasm-opt command failed: {0}")]
@@ -145,7 +149,7 @@ pub enum LoaderError {
 
 #[derive(Debug, Error)]
 pub struct CompilerError {
-    pub error: std::io::Error,
+    pub error: IoError,
     pub command: Box<Command>,
 }
 
@@ -166,6 +170,16 @@ pub struct IoError {
     pub path: Option<PathBuf>,
 }
 
+impl PartialEq for IoError {
+    fn eq(&self, other: &Self) -> bool {
+        self.path == other.path
+            && self.error.kind() == other.error.kind()
+            && self.error.raw_os_error() == other.error.raw_os_error()
+    }
+}
+
+impl Eq for IoError {}
+
 impl IoError {
     fn new(error: std::io::Error, path: Option<&Path>) -> Self {
         Self {
@@ -1326,9 +1340,10 @@ impl Loader {
             display_build_cmd(&command);
         }
 
+        let compiler_path = PathBuf::from(command.get_program());
         let output = command.output().map_err(|e| {
             LoaderError::Compiler(CompilerError {
-                error: e,
+                error: IoError::new(e, Some(&compiler_path)),
                 command: Box::new(command),
             })
         })?;
@@ -1457,7 +1472,7 @@ impl Loader {
 
         let compile_output = compile_command
             .output()
-            .map_err(LoaderError::WasmCompiler)?;
+            .map_err(|e| LoaderError::WasmCompiler(IoError::new(e, Some(&clang_exe))))?;
         if self.verbose {
             if !compile_output.stdout.is_empty() {
                 info!("stdout:{}", String::from_utf8_lossy(&compile_output.stdout));
@@ -1483,7 +1498,9 @@ impl Loader {
             display_build_cmd(&opt_command);
         }
 
-        let opt_output = opt_command.output().map_err(LoaderError::WasmOptimizer)?;
+        let opt_output = opt_command
+            .output()
+            .map_err(|e| LoaderError::WasmOptimizer(IoError::new(e, Some(&wasm_opt_exe))))?;
         if self.verbose {
             if !opt_output.stdout.is_empty() {
                 info!("stdout:{}", String::from_utf8_lossy(&opt_output.stdout));
@@ -1517,7 +1534,7 @@ impl Loader {
             .arg("-C")
             .arg(destination)
             .status()
-            .map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?;
+            .map_err(|e| LoaderError::Tar(IoError::new(e, Some(archive_path))))?;
 
         if !status.success() {
             return Err(LoaderError::Extraction(
@@ -1712,7 +1729,7 @@ impl Loader {
             .arg(&temp_tar_path)
             .arg(url)
             .status()
-            .map_err(|e| LoaderError::Curl(url.to_string(), e))?;
+            .map_err(|e| LoaderError::Curl(url.to_string(), IoError::new(e, None)))?;
 
         if !status.success() {
             Err(LoaderError::WasmToolDownload {

crates/tags/Cargo.toml

@@ -5,7 +5,6 @@ description = "Library for extracting tag information"
 authors = [ "Max Brunsfeld <maxbrunsfeld@gmail.com>", "Patrick Thomson <patrickt@github.com>" ]
 edition.workspace = true
 rust-version.workspace = true
-readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
 documentation = "https://docs.rs/tree-sitter-tags"

crates/xtask/src/build_wasm.rs

@@ -580,7 +580,15 @@ fn download_tool(
         .arg(&temp_tar_path)
         .arg(url)
         .status()
-        .map_err(|e| LoaderError::Curl(url.to_string(), e))?;
+        .map_err(|e| {
+            LoaderError::Curl(
+                url.to_string(),
+                IoError {
+                    error: e,
+                    path: None,
+                },
+            )
+        })?;
 
     if !status.success() {
         Err(LoaderError::WasmToolDownload {
@@ -620,7 +628,12 @@ fn extract_tar_gz_with_strip(archive_path: &Path, destination: &Path) -> Result<
         .arg("-C")
         .arg(destination)
         .status()
-        .map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?;
+        .map_err(|e| {
+            LoaderError::Tar(IoError {
+                error: e,
+                path: Some(archive_path.to_path_buf()),
+            })
+        })?;
 
     if !status.success() {
         Err(LoaderError::Extraction(
@@ -684,7 +697,7 @@ fn ensure_wasi_libc_source_exists() -> Result<PathBuf> {
         .arg(&archive_path)
         .arg(&url)
         .status()
-        .map_err(|error| LoaderError::Curl(url.clone(), error))?;
+        .map_err(|error| LoaderError::Curl(url.clone(), IoError { error, path: None }))?;
     if !status.success() {
         return Err(LoaderError::WasmToolDownload {
             tool: "wasi-libc",

docs/book.toml

@@ -5,10 +5,10 @@ src = "src"
 title = "Tree-sitter"
 
 [output.html]
-additional-css = [ "src/assets/css/playground.css", "src/assets/css/mdbook-admonish.css" ]
+additional-css = [ "src/assets/css/playground.css" ]
 additional-js = [ "src/assets/js/playground.js" ]
 edit-url-template = "https://github.com/tree-sitter/tree-sitter/edit/master/docs/{path}"
-git-repository-icon = "fa-github"
+git-repository-icon = "fab-github"
 git-repository-url = "https://github.com/tree-sitter/tree-sitter"
 
 [output.html.search]
@@ -18,9 +18,3 @@ boost-title = 2
 expand = true
 limit-results = 20
 use-boolean-and = true
-
-[preprocessor]
-
-[preprocessor.admonish]
-assets_version = "3.0.2"    # do not edit: managed by `mdbook-admonish install`
-command = "mdbook-admonish"

docs/package.nix

@@ -3,7 +3,6 @@
   lib,
   version,
   mdbook,
-  mdbook-admonish,
 }:
 stdenv.mkDerivation {
   inherit version;
@@ -11,10 +10,7 @@ stdenv.mkDerivation {
   src = ./.;
   pname = "tree-sitter-docs";
 
-  nativeBuildInputs = [
-    mdbook
-    mdbook-admonish
-  ];
+  nativeBuildInputs = [ mdbook ];
 
   buildPhase = ''
     mdbook build

docs/src/3-syntax-highlighting.md

@@ -154,13 +154,14 @@ Then, in our config file, we could map each of these highlight names to a color:
 
 Running `tree-sitter highlight` on this Go file would produce output like this:
 
-```admonish example collapsible=true, title='Output'
+<details>
+<summary>Output</summary>
 <pre class='highlight'>
 <span style='color: purple;'>func</span> <span style='color: #005fd7;'>increment</span>(<span>a</span> <span style='color: green;'>int</span>) <span style='color: green;'>int</span> {
     <span style='color: purple;'>return</span> <span>a</span> <span style='font-weight: bold;color: #4e4e4e;'>+</span> <span style='font-weight: bold;color: #875f00;'>1</span>
 }
 </pre>
-```
+</details>
 
 ### Local Variables
 
@@ -297,7 +298,8 @@ and blocks create local *scopes*, parameters and assignments create *definitions
 
 Running `tree-sitter highlight` on this ruby file would produce output like this:
 
-```admonish example collapsible=true, title='Output'
+<details>
+<summary>Output</summary>
 <pre class='highlight'>
 <span style='color: purple;'>def</span> <span style='color: #005fd7;'>process_list</span><span style='color: #4e4e4e;'>(</span><span style='text-decoration: underline;'>list</span><span style='color: #4e4e4e;'>)</span>
   <span>context</span> <span style='font-weight: bold;color: #4e4e4e;'>=</span> <span style='color: #005fd7;'>current_context</span>
@@ -305,11 +307,11 @@ Running `tree-sitter highlight` on this ruby file would produce output like this
     <span style='color: #005fd7;'>process_item</span>(<span style='text-decoration: underline;'>item</span><span style='color: #4e4e4e;'>,</span> <span>context</span><span style='color: #4e4e4e;'>)</span>
   <span style='color: purple;'>end</span>
 <span style='color: purple;'>end</span>
-
+<span></span>
 <span>item</span> <span style='font-weight: bold;color: #4e4e4e;'>=</span> <span style='font-weight: bold;color: #875f00;'>5</span>
 <span>list</span> <span style='font-weight: bold;color: #4e4e4e;'>=</span> [<span>item</span><span style='color: #4e4e4e;'>]</span>
 </pre>
-```
+</details>
 
 ### Language Injection
 
@@ -417,19 +419,19 @@ var abc = function(d) {
 };
 ```
 
-```admonish cite title='From the Sublime text docs'
-The two types of tests are:
+> **From the Sublime Text docs**
+>
+> The two types of tests are:
+>
+> **Caret**: ^ this will test the following selector against the scope on the most recent non-test line. It will test it
+> at the same column the ^ is in. Consecutive ^s will test each column against the selector.
+>
+> **Arrow**: <- this will test the following selector against the scope on the most recent non-test line. It will test it
+> at the same column as the comment character is in.
 
-**Caret**: ^ this will test the following selector against the scope on the most recent non-test line. It will test it
-at the same column the ^ is in. Consecutive ^s will test each column against the selector.
-
-**Arrow**: <- this will test the following selector against the scope on the most recent non-test line. It will test it
-at the same column as the comment character is in.
-```
-```admonish note
-An exclamation mark (`!`) can be used to negate a selector. For example, `!keyword` will match any scope that is
-not the `keyword` class.
-```
+> [!NOTE]
+> An exclamation mark (`!`) can be used to negate a selector. For example, `!keyword` will match any scope that is
+> not the `keyword` class.
 
 [erb]: https://en.wikipedia.org/wiki/ERuby
 [highlight crate]: https://github.com/tree-sitter/tree-sitter/tree/master/crates/highlight

docs/src/6-contributing.md

@@ -89,9 +89,8 @@ npm install # or your JS package manager of choice
 npm run build
 ```
 
-```admonish note
-If using a local Emscripten installation, the version must match the one [pinned by this repository][emscripten-version].
-```
+> [!NOTE]
+> If using a local Emscripten installation, the version must match the one [pinned by this repository][emscripten-version].
 
 Build the Rust libraries and the CLI:
 
@@ -301,9 +300,8 @@ edit and hit the edit icon at the top right of the page.
 
 ### Prerequisites for Local Development
 
-```admonish note
-We're assuming you have `cargo` installed, the Rust package manager.
-```
+> [!NOTE]
+> We're assuming you have `cargo` installed, the Rust package manager.
 
 To run and iterate on the docs locally, the
 [`mdbook`][mdbook cli] CLI tool is required, which can be installed with
@@ -313,16 +311,14 @@ cargo install mdbook
 ```
 
 You might have noticed we have some fancy admonitions sprinkled throughout the documentation, like the note above.
-These are created using [`mdbook-admonish`][admonish], a [preprocessor][preprocessor] for `mdBook`. As such, this is also
-a requirement for developing the documentation locally. To install it, run:
+These are built into `mdBook`, and are written as a blockquote whose first line names the kind, one of `NOTE`, `TIP`,
+`IMPORTANT`, `WARNING`, or `CAUTION`. See the [reference][admonitions] for more information.
 
-```sh
-cargo install mdbook-admonish
+```md
+> [!NOTE]
+> Something worth pointing out.
 ```
 
-Once you've installed it, you can begin using admonitions in your markdown files. See the [reference][admonish reference]
-for more information.
-
 ### Spinning it up
 
 Now that you've installed the prerequisites, you can run the following command to start a local server:
@@ -343,8 +339,7 @@ at [`docs/src/assets/css/playground.css`][playground css]. The editor of choice
 and the tree-sitter module is fetched from [here][js url]. This, along with the Wasm module and Wasm parsers, live in the
 [.github.io repo][gh.io repo].
 
-[admonish]: https://github.com/tommilligan/mdbook-admonish
-[admonish reference]: https://tommilligan.github.io/mdbook-admonish/reference.html
+[admonitions]: https://rust-lang.github.io/mdBook/format/markdown.html#admonitions
 [binaryen]: https://github.com/WebAssembly/binaryen
 [binaryen-releases]: https://github.com/WebAssembly/binaryen/releases
 [config crate]: https://crates.io/crates/tree-sitter-config
@@ -375,7 +370,6 @@ and the tree-sitter module is fetched from [here][js url]. This, along with the
 [playground]: https://github.com/tree-sitter/tree-sitter/blob/master/docs/src/assets/js/playground.js
 [playground css]: https://github.com/tree-sitter/tree-sitter/blob/master/docs/src/assets/css/playground.css
 [podman]: https://podman.io
-[preprocessor]: https://rust-lang.github.io/mdBook/for_developers/preprocessors.html
 [py package]: https://pypi.org/project/tree-sitter
 [py ts]: https://github.com/tree-sitter/py-tree-sitter
 [pypi]: https://pypi.org

docs/src/7-playground.md

@@ -96,9 +96,8 @@ You can also run playground locally (with your own grammar) using the
 <a href="/tree-sitter/cli/playground.html">CLI</a>'s <code>tree-sitter playground</code> subcommand.
 </p>
 
-```admonish info
-Logging (if enabled) can be viewed in the browser's console.
-```
+> [!NOTE]
+> Logging (if enabled) can be viewed in the browser's console.
 
 <p>The syntax tree should update as you type in the code. As you move around the
 code, the current node should be highlighted in the tree; you can also click any

docs/src/assets/css/mdbook-admonish.css

@@ -1,348 +0,0 @@
-@charset "UTF-8";
-:is(.admonition) {
-  display: flow-root;
-  margin: 1.5625em 0;
-  padding: 0 1.2rem;
-  color: var(--fg);
-  page-break-inside: avoid;
-  background-color: var(--bg);
-  border: 0 solid black;
-  border-inline-start-width: 0.4rem;
-  border-radius: 0.2rem;
-  box-shadow: 0 0.2rem 1rem rgba(0, 0, 0, 0.05), 0 0 0.1rem rgba(0, 0, 0, 0.1);
-}
-@media print {
-  :is(.admonition) {
-    box-shadow: none;
-  }
-}
-:is(.admonition) > * {
-  box-sizing: border-box;
-}
-:is(.admonition) :is(.admonition) {
-  margin-top: 1em;
-  margin-bottom: 1em;
-}
-:is(.admonition) > .tabbed-set:only-child {
-  margin-top: 0;
-}
-html :is(.admonition) > :last-child {
-  margin-bottom: 1.2rem;
-}
-
-a.admonition-anchor-link {
-  display: none;
-  position: absolute;
-  left: -1.2rem;
-  padding-right: 1rem;
-}
-a.admonition-anchor-link:link, a.admonition-anchor-link:visited {
-  color: var(--fg);
-}
-a.admonition-anchor-link:link:hover, a.admonition-anchor-link:visited:hover {
-  text-decoration: none;
-}
-a.admonition-anchor-link::before {
-  content: "§";
-}
-
-:is(.admonition-title, summary.admonition-title) {
-  position: relative;
-  min-height: 4rem;
-  margin-block: 0;
-  margin-inline: -1.6rem -1.2rem;
-  padding-block: 0.8rem;
-  padding-inline: 4.4rem 1.2rem;
-  font-weight: 700;
-  background-color: rgba(68, 138, 255, 0.1);
-  print-color-adjust: exact;
-  -webkit-print-color-adjust: exact;
-  display: flex;
-}
-:is(.admonition-title, summary.admonition-title) p {
-  margin: 0;
-}
-html :is(.admonition-title, summary.admonition-title):last-child {
-  margin-bottom: 0;
-}
-:is(.admonition-title, summary.admonition-title)::before {
-  position: absolute;
-  top: 0.625em;
-  inset-inline-start: 1.6rem;
-  width: 2rem;
-  height: 2rem;
-  background-color: #448aff;
-  print-color-adjust: exact;
-  -webkit-print-color-adjust: exact;
-  mask-image: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"></svg>');
-  -webkit-mask-image: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"></svg>');
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-size: contain;
-  content: "";
-}
-:is(.admonition-title, summary.admonition-title):hover a.admonition-anchor-link {
-  display: initial;
-}
-
-details.admonition > summary.admonition-title::after {
-  position: absolute;
-  top: 0.625em;
-  inset-inline-end: 1.6rem;
-  height: 2rem;
-  width: 2rem;
-  background-color: currentcolor;
-  mask-image: var(--md-details-icon);
-  -webkit-mask-image: var(--md-details-icon);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-size: contain;
-  content: "";
-  transform: rotate(0deg);
-  transition: transform 0.25s;
-}
-details[open].admonition > summary.admonition-title::after {
-  transform: rotate(90deg);
-}
-
-:root {
-  --md-details-icon: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M8.59 16.58 13.17 12 8.59 7.41 10 6l6 6-6 6-1.41-1.42Z'/></svg>");
-}
-
-:root {
-  --md-admonition-icon--admonish-note: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75L3 17.25z'/></svg>");
-  --md-admonition-icon--admonish-abstract: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M17 9H7V7h10m0 6H7v-2h10m-3 6H7v-2h7M12 3a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m7 0h-4.18C14.4 1.84 13.3 1 12 1c-1.3 0-2.4.84-2.82 2H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2V5a2 2 0 0 0-2-2z'/></svg>");
-  --md-admonition-icon--admonish-info: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M13 9h-2V7h2m0 10h-2v-6h2m-1-9A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10A10 10 0 0 0 12 2z'/></svg>");
-  --md-admonition-icon--admonish-tip: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M17.66 11.2c-.23-.3-.51-.56-.77-.82-.67-.6-1.43-1.03-2.07-1.66C13.33 7.26 13 4.85 13.95 3c-.95.23-1.78.75-2.49 1.32-2.59 2.08-3.61 5.75-2.39 8.9.04.1.08.2.08.33 0 .22-.15.42-.35.5-.23.1-.47.04-.66-.12a.58.58 0 0 1-.14-.17c-1.13-1.43-1.31-3.48-.55-5.12C5.78 10 4.87 12.3 5 14.47c.06.5.12 1 .29 1.5.14.6.41 1.2.71 1.73 1.08 1.73 2.95 2.97 4.96 3.22 2.14.27 4.43-.12 6.07-1.6 1.83-1.66 2.47-4.32 1.53-6.6l-.13-.26c-.21-.46-.77-1.26-.77-1.26m-3.16 6.3c-.28.24-.74.5-1.1.6-1.12.4-2.24-.16-2.9-.82 1.19-.28 1.9-1.16 2.11-2.05.17-.8-.15-1.46-.28-2.23-.12-.74-.1-1.37.17-2.06.19.38.39.76.63 1.06.77 1 1.98 1.44 2.24 2.8.04.14.06.28.06.43.03.82-.33 1.72-.93 2.27z'/></svg>");
-  --md-admonition-icon--admonish-success: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='m9 20.42-6.21-6.21 2.83-2.83L9 14.77l9.88-9.89 2.83 2.83L9 20.42z'/></svg>");
-  --md-admonition-icon--admonish-question: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='m15.07 11.25-.9.92C13.45 12.89 13 13.5 13 15h-2v-.5c0-1.11.45-2.11 1.17-2.83l1.24-1.26c.37-.36.59-.86.59-1.41a2 2 0 0 0-2-2 2 2 0 0 0-2 2H8a4 4 0 0 1 4-4 4 4 0 0 1 4 4 3.2 3.2 0 0 1-.93 2.25M13 19h-2v-2h2M12 2A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10c0-5.53-4.5-10-10-10z'/></svg>");
-  --md-admonition-icon--admonish-warning: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M13 14h-2V9h2m0 9h-2v-2h2M1 21h22L12 2 1 21z'/></svg>");
-  --md-admonition-icon--admonish-failure: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M20 6.91 17.09 4 12 9.09 6.91 4 4 6.91 9.09 12 4 17.09 6.91 20 12 14.91 17.09 20 20 17.09 14.91 12 20 6.91z'/></svg>");
-  --md-admonition-icon--admonish-danger: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M11 15H6l7-14v8h5l-7 14v-8z'/></svg>");
-  --md-admonition-icon--admonish-bug: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M14 12h-4v-2h4m0 6h-4v-2h4m6-6h-2.81a5.985 5.985 0 0 0-1.82-1.96L17 4.41 15.59 3l-2.17 2.17a6.002 6.002 0 0 0-2.83 0L8.41 3 7 4.41l1.62 1.63C7.88 6.55 7.26 7.22 6.81 8H4v2h2.09c-.05.33-.09.66-.09 1v1H4v2h2v1c0 .34.04.67.09 1H4v2h2.81c1.04 1.79 2.97 3 5.19 3s4.15-1.21 5.19-3H20v-2h-2.09c.05-.33.09-.66.09-1v-1h2v-2h-2v-1c0-.34-.04-.67-.09-1H20V8z'/></svg>");
-  --md-admonition-icon--admonish-example: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M7 13v-2h14v2H7m0 6v-2h14v2H7M7 7V5h14v2H7M3 8V5H2V4h2v4H3m-1 9v-1h3v4H2v-1h2v-.5H3v-1h1V17H2m2.25-7a.75.75 0 0 1 .75.75c0 .2-.08.39-.21.52L3.12 13H5v1H2v-.92L4 11H2v-1h2.25z'/></svg>");
-  --md-admonition-icon--admonish-quote: url("data:image/svg+xml;charset=utf-8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path d='M14 17h3l2-4V7h-6v6h3M6 17h3l2-4V7H5v6h3l-2 4z'/></svg>");
-}
-
-:is(.admonition):is(.admonish-note) {
-  border-color: #448aff;
-}
-
-:is(.admonish-note) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(68, 138, 255, 0.1);
-}
-:is(.admonish-note) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #448aff;
-  mask-image: var(--md-admonition-icon--admonish-note);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-note);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-abstract, .admonish-summary, .admonish-tldr) {
-  border-color: #00b0ff;
-}
-
-:is(.admonish-abstract, .admonish-summary, .admonish-tldr) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(0, 176, 255, 0.1);
-}
-:is(.admonish-abstract, .admonish-summary, .admonish-tldr) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #00b0ff;
-  mask-image: var(--md-admonition-icon--admonish-abstract);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-abstract);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-info, .admonish-todo) {
-  border-color: #00b8d4;
-}
-
-:is(.admonish-info, .admonish-todo) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(0, 184, 212, 0.1);
-}
-:is(.admonish-info, .admonish-todo) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #00b8d4;
-  mask-image: var(--md-admonition-icon--admonish-info);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-info);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-tip, .admonish-hint, .admonish-important) {
-  border-color: #00bfa5;
-}
-
-:is(.admonish-tip, .admonish-hint, .admonish-important) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(0, 191, 165, 0.1);
-}
-:is(.admonish-tip, .admonish-hint, .admonish-important) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #00bfa5;
-  mask-image: var(--md-admonition-icon--admonish-tip);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-tip);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-success, .admonish-check, .admonish-done) {
-  border-color: #00c853;
-}
-
-:is(.admonish-success, .admonish-check, .admonish-done) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(0, 200, 83, 0.1);
-}
-:is(.admonish-success, .admonish-check, .admonish-done) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #00c853;
-  mask-image: var(--md-admonition-icon--admonish-success);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-success);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-question, .admonish-help, .admonish-faq) {
-  border-color: #64dd17;
-}
-
-:is(.admonish-question, .admonish-help, .admonish-faq) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(100, 221, 23, 0.1);
-}
-:is(.admonish-question, .admonish-help, .admonish-faq) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #64dd17;
-  mask-image: var(--md-admonition-icon--admonish-question);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-question);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-warning, .admonish-caution, .admonish-attention) {
-  border-color: #ff9100;
-}
-
-:is(.admonish-warning, .admonish-caution, .admonish-attention) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(255, 145, 0, 0.1);
-}
-:is(.admonish-warning, .admonish-caution, .admonish-attention) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #ff9100;
-  mask-image: var(--md-admonition-icon--admonish-warning);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-warning);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-failure, .admonish-fail, .admonish-missing) {
-  border-color: #ff5252;
-}
-
-:is(.admonish-failure, .admonish-fail, .admonish-missing) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(255, 82, 82, 0.1);
-}
-:is(.admonish-failure, .admonish-fail, .admonish-missing) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #ff5252;
-  mask-image: var(--md-admonition-icon--admonish-failure);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-failure);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-danger, .admonish-error) {
-  border-color: #ff1744;
-}
-
-:is(.admonish-danger, .admonish-error) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(255, 23, 68, 0.1);
-}
-:is(.admonish-danger, .admonish-error) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #ff1744;
-  mask-image: var(--md-admonition-icon--admonish-danger);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-danger);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-bug) {
-  border-color: #f50057;
-}
-
-:is(.admonish-bug) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(245, 0, 87, 0.1);
-}
-:is(.admonish-bug) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #f50057;
-  mask-image: var(--md-admonition-icon--admonish-bug);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-bug);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-example) {
-  border-color: #7c4dff;
-}
-
-:is(.admonish-example) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(124, 77, 255, 0.1);
-}
-:is(.admonish-example) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #7c4dff;
-  mask-image: var(--md-admonition-icon--admonish-example);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-example);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-:is(.admonition):is(.admonish-quote, .admonish-cite) {
-  border-color: #9e9e9e;
-}
-
-:is(.admonish-quote, .admonish-cite) > :is(.admonition-title, summary.admonition-title) {
-  background-color: rgba(158, 158, 158, 0.1);
-}
-:is(.admonish-quote, .admonish-cite) > :is(.admonition-title, summary.admonition-title)::before {
-  background-color: #9e9e9e;
-  mask-image: var(--md-admonition-icon--admonish-quote);
-  -webkit-mask-image: var(--md-admonition-icon--admonish-quote);
-  mask-repeat: no-repeat;
-  -webkit-mask-repeat: no-repeat;
-  mask-size: contain;
-  -webkit-mask-repeat: no-repeat;
-}
-
-.navy :is(.admonition) {
-  background-color: var(--sidebar-bg);
-}
-
-.ayu :is(.admonition),
-.coal :is(.admonition) {
-  background-color: var(--theme-hover);
-}
-
-.rust :is(.admonition) {
-  background-color: var(--sidebar-bg);
-  color: var(--sidebar-fg);
-}
-.rust .admonition-anchor-link:link, .rust .admonition-anchor-link:visited {
-  color: var(--sidebar-fg);
-}

docs/src/cli/init-config.md

@@ -11,9 +11,8 @@ These directories are created in the "default" location for your platform:
 * On Unix, `$XDG_CONFIG_HOME/tree-sitter` or `$HOME/.config/tree-sitter`
 * On Windows, `%APPDATA%\tree-sitter` or `$HOME\AppData\Roaming\tree-sitter`
 
-```admonish info
-The CLI will work if there's no config file present, falling back on default values for each configuration option.
-```
+> [!NOTE]
+> The CLI will work if there's no config file present, falling back on default values for each configuration option.
 
 When you run the `init-config` command, it will print out the location of the file that it creates so that you can easily
 find and modify it.
@@ -117,9 +116,8 @@ An example theme can be seen below:
 The [`tree-sitter parse`](./parse.md) command will output a pretty-printed CST when the `-c/--cst` option is used. You can
 control what colors are used for various parts of the tree in your configuration file.
 
-```admonish note
-Omitting a field will cause the relevant text to be rendered with its default color.
-```
+> [!NOTE]
+> Omitting a field will cause the relevant text to be rendered with its default color.
 
 An example parse theme can be seen below:
 

docs/src/cli/playground.md

@@ -6,10 +6,9 @@ The `playground` command allows you to start a local playground to test your par
 tree-sitter playground [OPTIONS] # Aliases: play, pg, web-ui
 ```
 
-```admonish note
-For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md)
-subcommand (`tree-sitter build --wasm`).
-```
+> [!NOTE]
+> For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md)
+> subcommand (`tree-sitter build --wasm`).
 
 ## Options
 

docs/src/cli/test.md

@@ -36,9 +36,8 @@ If `--lib-path` is used, the name of the language used to extract the library's
 
 Update the expected output of tests.
 
-```admonish info
-Tests containing `ERROR` nodes or `MISSING` nodes will not be updated.
-```
+> [!NOTE]
+> Tests containing `ERROR` nodes or `MISSING` nodes will not be updated.
 
 ### `-d/--debug`
 

docs/src/creating-parsers/1-getting-started.md

@@ -37,13 +37,11 @@ mkdir tree-sitter-${LOWER_PARSER_NAME}
 cd tree-sitter-${LOWER_PARSER_NAME}
 ```
 
-```admonish note
-The `LOWER_` prefix here means the "lowercase" name of the language.
-```
+> [!NOTE]
+> The `LOWER_` prefix here means the "lowercase" name of the language.
 
-```admonish warning
-Dashes are not permitted via the CLI's `init` command and should not be used in parser names.
-```
+> [!WARNING]
+> Dashes are not permitted via the CLI's `init` command and should not be used in parser names.
 
 ### Init
 
@@ -78,10 +76,9 @@ export default grammar({
 });
 ```
 
-```admonish info
-The placeholders shown above would be replaced with the corresponding data you provided in the `init` sub-command's
-prompts.
-```
+> [!NOTE]
+> The placeholders shown above would be replaced with the corresponding data you provided in the `init` sub-command's
+> prompts.
 
 To learn more about this command, check the [reference page](../cli/init.md).
 

docs/src/creating-parsers/2-the-grammar-dsl.md

@@ -20,20 +20,19 @@ DSL through the `RustRegex` class. Simply pass your regex pattern as a string:
   accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern
   itself. For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex].
 
-  ```admonish note 
-  Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround
-  assertions not being feasible to use in an LR(1) grammar, as well as certain flags being unnecessary for tree-sitter. However,
-  plenty of features are supported by default:
-
-  - Character classes
-  - Character ranges
-  - Character sets
-  - Quantifiers
-  - Alternation
-  - Grouping
-  - Unicode character escapes
-  - Unicode property escapes
-  ```
+  > [!NOTE]
+  > Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround
+  > assertions not being feasible to use in an LR(1) grammar, as well as certain flags being unnecessary for tree-sitter. However,
+  > plenty of features are supported by default:
+  >
+  > - Character classes
+  > - Character ranges
+  > - Character sets
+  > - Quantifiers
+  > - Alternation
+  > - Grouping
+  > - Unicode character escapes
+  > - Unicode property escapes
 
 - **Sequences : `seq(rule1, rule2, ...)`** — This function creates a rule that matches any number of other rules, one after
 another. It is analogous to simply writing multiple symbols next to each other in [EBNF notation][ebnf].

docs/src/creating-parsers/3-writing-the-grammar.md

@@ -244,11 +244,10 @@ Possible resolutions:
   4:  Add a conflict for these rules: `binary_expression` `unary_expression`
 ```
 
-```admonish hint
-The • character in the error message indicates where exactly during
-parsing the conflict occurs, or in other words, where the parser is encountering
-ambiguity.
-```
+> [!TIP]
+> The • character in the error message indicates where exactly during
+> parsing the conflict occurs, or in other words, where the parser is encountering
+> ambiguity.
 
 For an expression like `-a * b`, it's not clear whether the `-` operator applies to the `a * b` or just to the `a`. This
 is where the `prec` function [described in the previous page][grammar dsl] comes into play. By wrapping a rule with `prec`,
@@ -364,11 +363,10 @@ In such cases, we want the parser to explore both possibilities by explicitly de
 }
 ```
 
-```admonish note
-The example is a bit contrived for the purpose of illustrating the usage of conflicts. The actual JavaScript grammar isn't
-structured like that, but this conflict is actually present in the
-[Tree-sitter JavaScript grammar](https://github.com/tree-sitter/tree-sitter-javascript/blob/108b2d4d17a04356a340aea809e4dd5b801eb40d/grammar.js#L100).
-```
+> [!NOTE]
+> The example is a bit contrived for the purpose of illustrating the usage of conflicts. The actual JavaScript grammar isn't
+> structured like that, but this conflict is actually present in the
+> [Tree-sitter JavaScript grammar](https://github.com/tree-sitter/tree-sitter-javascript/blob/108b2d4d17a04356a340aea809e4dd5b801eb40d/grammar.js#L100).
 
 ## Hiding Rules
 
@@ -420,11 +418,10 @@ module.exports = grammar({
 });
 ```
 
-```admonish warning
-When adding more complicated tokens to `extras`, it's preferable to associate the pattern
-with a rule. This way, you avoid the lexer inlining this pattern in a bunch of spots,
-which can dramatically reduce the parser size.
-```
+> [!WARNING]
+> When adding more complicated tokens to `extras`, it's preferable to associate the pattern
+> with a rule. This way, you avoid the lexer inlining this pattern in a bunch of spots,
+> which can dramatically reduce the parser size.
 
 For example, instead of defining the `comment` token inline in `extras`:
 
@@ -471,10 +468,9 @@ module.exports = grammar({
 });
 ```
 
-```admonish note
-Tree-sitter intentionally simplifies the whitespace character class, `\s`, to `[ \t\n\r]` as a performance
-optimization. This is because typically users do not require the full Unicode definition of whitespace.
-```
+> [!NOTE]
+> Tree-sitter intentionally simplifies the whitespace character class, `\s`, to `[ \t\n\r]` as a performance
+> optimization. This is because typically users do not require the full Unicode definition of whitespace.
 
 ## Using Supertypes
 
@@ -520,12 +516,11 @@ module.exports = grammar({
 Although supertype rules are hidden from the syntax tree, they can still be used in queries. See the chapter on
 [Query Syntax][query syntax] for more information.
 
-```admonish warning
-Aliasing a supertype rule makes the node in the alias match the supertype in
-name only and will not be treated as a supertype. For `alias($.foo, $.bar)` a
-query targeting `bar` will not transparently match the supertype's subtypes the
-way a query targeting `foo` would.
-```
+> [!WARNING]
+> Aliasing a supertype rule makes the node in the alias match the supertype in
+> name only and will not be treated as a supertype. For `alias($.foo, $.bar)` a
+> query targeting `bar` will not transparently match the supertype's subtypes the
+> way a query targeting `foo` would.
 
 # Lexical Analysis
 
@@ -637,11 +632,10 @@ It would then correctly recognize the code as invalid.
 Aside from improving error detection, keyword extraction also has performance benefits. It allows Tree-sitter to generate
 a smaller, simpler lexing function, which means that **the parser will compile much more quickly**.
 
-```admonish note
-The word token must be a unique token that is not reused by another rule. If you want to have a word token used in a
-rule that's called something else, you should just alias the word token instead, like how the Rust grammar does it
-<a href="https://github.com/tree-sitter/tree-sitter-rust/blob/1f63b33efee17e833e0ea29266dd3d713e27e321/grammar.js#L1605">here</a>
-```
+> [!NOTE]
+> The word token must be a unique token that is not reused by another rule. If you want to have a word token used in a
+> rule that's called something else, you should just alias the word token instead, like how the Rust grammar does it
+> <a href="https://github.com/tree-sitter/tree-sitter-rust/blob/1f63b33efee17e833e0ea29266dd3d713e27e321/grammar.js#L1605">here</a>
 
 [ambiguous-grammar]: https://en.wikipedia.org/wiki/Ambiguous_grammar
 [antlr]: https://www.antlr.org

docs/src/creating-parsers/4-external-scanners.md

@@ -224,11 +224,10 @@ array macros from `tree_sitter/array.h`.
 There are quite a few of them provided for you, but here's how you could get started tracking some state. Check out the header
 itself for more detailed documentation.
 
-```admonish attention
-Do not use any of the array functions or macros that are prefixed with an underscore and have comments saying
-that it is not what you are looking for. These are internal functions used as helpers by other macros that are public.
-They are not meant to be used directly, nor are they what you want.
-```
+> [!WARNING]
+> Do not use any of the array functions or macros that are prefixed with an underscore and have comments saying
+> that it is not what you are looking for. These are internal functions used as helpers by other macros that are public.
+> They are not meant to be used directly, nor are they what you want.
 
 ```c
 #include "tree_sitter/parser.h"
@@ -370,13 +369,12 @@ However, when you use rule references (like `$.if_keyword`) in the externals arr
 in the grammar, Tree-sitter cannot fall back to its internal lexer. In this case, the external scanner is solely responsible
 for recognizing these tokens.
 
-```admonish danger
-- External scanners can easily create infinite loops
-
-- Be extremely careful when emitting zero-width tokens
-
-- Always use the `eof` function when looping through characters
-```
+> [!CAUTION]
+> - External scanners can easily create infinite loops
+>
+> - Be extremely careful when emitting zero-width tokens
+>
+> - Always use the `eof` function when looping through characters
 
 [ejs]: https://ejs.co
 [enum]: https://en.wikipedia.org/wiki/Enumerated_type#C

docs/src/creating-parsers/5-writing-tests.md

@@ -33,10 +33,9 @@ func x() int {
 * Then, the **expected output syntax tree** is written as an [S-expression][s-exp]. The exact placement of whitespace in
 the S-expression doesn't matter, but ideally the syntax tree should be legible.
 
-```admonish tip
-The S-expression does not show syntax nodes like `func`, `(` and `;`, which are expressed as strings and regexes in the grammar.
-It only shows the *named* nodes, as described in [this section][named-vs-anonymous-nodes] of the page on parser usage.
-```
+> [!TIP]
+> The S-expression does not show syntax nodes like `func`, `(` and `;`, which are expressed as strings and regexes in the grammar.
+> It only shows the *named* nodes, as described in [this section][named-vs-anonymous-nodes] of the page on parser usage.
 
   The expected output section can also *optionally* show the [*field names*][node-field-names] associated with each child
   node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself
@@ -109,20 +108,18 @@ The recommendation is to be comprehensive in adding tests. If it's a visible nod
 directory. It's typically a good idea to test all the permutations of each language construct. This increases test coverage,
 but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language.
 
-```admonish tip
-After modifying the grammar, you can run `tree-sitter test -u`
-to update all syntax trees in corpus files with current parser output.
-```
+> [!TIP]
+> After modifying the grammar, you can run `tree-sitter test -u`
+> to update all syntax trees in corpus files with current parser output.
 
 ## Attributes
 
 Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with
 a `:`. A couple of attributes also take in a parameter, which require the use of parenthesis.
 
-```admonish tip
-If you'd like to supply in multiple parameters, e.g. to run tests on multiple platforms or to test multiple languages,
-you can repeat the attribute on a new line.
-```
+> [!TIP]
+> If you'd like to supply in multiple parameters, e.g. to run tests on multiple platforms or to test multiple languages,
+> you can repeat the attribute on a new line.
 
 The following attributes are available:
 

docs/src/using-parsers/2-basic-parsing.md

@@ -54,9 +54,8 @@ typedef uint32_t (*TSDecodeFunction)(
 );
 ```
 
-```admonish attention
-The `TSInputEncoding` must be set to `TSInputEncodingCustom` for the `decode` function to be called.
-```
+> [!WARNING]
+> The `TSInputEncoding` must be set to `TSInputEncodingCustom` for the `decode` function to be called.
 
 The `string` argument is a pointer to the text to decode, which comes from the `read` function, and the `length` argument
 is the length of the `string`. The `code_point` argument is a pointer to an integer that represents the decoded code point,
@@ -87,9 +86,8 @@ TSPoint ts_node_start_point(TSNode);
 TSPoint ts_node_end_point(TSNode);
 ```
 
-```admonish note
-A *newline* is considered to be a single line feed (`\n`) character.
-```
+> [!NOTE]
+> A *newline* is considered to be a single line feed (`\n`) character.
 
 ## Retrieving Nodes
 

docs/src/using-parsers/3-advanced-parsing.md

@@ -156,9 +156,8 @@ Internally, copying a syntax tree just entails incrementing an atomic reference
 tree which you can freely query, edit, reparse, or delete on a new thread while continuing to use the original tree on a
 different thread.
 
-```admonish danger
-Individual `TSTree` instances are _not_ thread safe; you must copy a tree if you want to use it on multiple threads simultaneously.
-```
+> [!CAUTION]
+> Individual `TSTree` instances are _not_ thread safe; you must copy a tree if you want to use it on multiple threads simultaneously.
 
 [ejs]: https://ejs.co
 [erb]: https://ruby-doc.org/stdlib-2.5.1/libdoc/erb/rdoc/ERB.html

docs/src/using-parsers/4-walking-trees.md

@@ -4,13 +4,12 @@ You can access every node in a syntax tree using the `TSNode` APIs [described ea
 to access a large number of nodes, the fastest way to do so is with a _tree cursor_. A cursor is a stateful object that
 allows you to walk a syntax tree with maximum efficiency.
 
-```admonish note
-The given input node is considered the root of the cursor, and the cursor cannot walk outside this node.
-Going to the parent or any sibling of the root node will always return `false`.
-
-This has no unexpected effects if the given input node is the actual `root` node of the tree, but is something to keep in
-mind when using cursors constructed with a node that is not the `root` node.
-```
+> [!NOTE]
+> The given input node is considered the root of the cursor, and the cursor cannot walk outside this node.
+> Going to the parent or any sibling of the root node will always return `false`.
+>
+> This has no unexpected effects if the given input node is the actual `root` node of the tree, but is something to keep in
+> mind when using cursors constructed with a node that is not the `root` node.
 
 You can initialize a cursor from any node:
 

docs/src/using-parsers/queries/3-predicates-and-directives.md

@@ -186,15 +186,14 @@ To recap about the predicates and directives Tree-sitter's bindings support:
 
 - `#strip!` removes text from a capture
 
-```admonish info
-Predicates and directives are not handled directly by the Tree-sitter C library.
-They are just exposed in a structured form so that higher-level code can perform
-the filtering. However, higher-level bindings to Tree-sitter like
-[the Rust Crate][rust crate]
-or the [WebAssembly binding][wasm binding]
-do implement a few common predicates like those explained above. In the future, more "standard" predicates and directives
-may be added.
-```
+> [!NOTE]
+> Predicates and directives are not handled directly by the Tree-sitter C library.
+> They are just exposed in a structured form so that higher-level code can perform
+> the filtering. However, higher-level bindings to Tree-sitter like
+> [the Rust Crate][rust crate]
+> or the [WebAssembly binding][wasm binding]
+> do implement a few common predicates like those explained above. In the future, more "standard" predicates and directives
+> may be added.
 
 [cgo]: https://pkg.go.dev/cmd/cgo
 [rust crate]: https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust

docs/src/using-parsers/queries/4-api.md

@@ -79,7 +79,6 @@ bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t sta
 bool ts_query_cursor_set_containing_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
 ```
 
-```admonish note
-For all of these functions, an end value of zero is treated as unbounded (the maximum possible value).
-This means passing a byte range of `(0, 0)` (or a point range of `{0, 0}, {0, 0}`) will match the entire tree, not an empty range.
-```
+> [!NOTE]
+> For all of these functions, an end value of zero is treated as unbounded (the maximum possible value).
+> This means passing a byte range of `(0, 0)` (or a point range of `{0, 0}, {0, 0}`) will match the entire tree, not an empty range.

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1770562336,
-        "narHash": "sha256-ub1gpAONMFsT/GU2hV6ZWJjur8rJ6kKxdm9IlCT0j84=",
+        "lastModified": 1788039129,
+        "narHash": "sha256-pa4Q0qErvCvzCaaUph7Sm37RhR4xvPrYI8Lgz6k85+A=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d6c71932130818840fc8fe9509cf50be8c64634f",
+        "rev": "d2f67949798825fe853f7c5d0492b8bf016d3f88",
         "type": "github"
       },
       "original": {

flake.nix

@@ -17,7 +17,7 @@
       eachSystem = lib.genAttrs systems;
       pkgsFor = inputs.nixpkgs.legacyPackages;
 
-      version = "0.27.0";
+      version = "0.28.0";
 
       fs = lib.fileset;
       src = fs.toSource {
@@ -35,6 +35,22 @@
       };
       fixturesJson = lib.importJSON ./test/fixtures/fixtures.json;
 
+      npmDepsHash = "sha256-XeGrKPpel5XKUA63UjvteqyI5srifq3g54GjhvDEjlo=";
+
+      # `tree-sitter build --wasm` searches TREE_SITTER_WASI_SDK_PATH for `clang`,
+      # `wasm32-unknown-wasi-clang` or `wasm32-wasi-clang`. The nixpkgs wasi32
+      # toolchain only ships the `wasm32-unknown-wasip1-` prefix, so alias one.
+      wasiSdkFor =
+        pkgs:
+        let
+          cc = pkgs.pkgsCross.wasi32.stdenv.cc;
+        in
+        pkgs.symlinkJoin {
+          name = "wasi-sdk";
+          paths = [ cc ];
+          postBuild = "ln -s ${cc.targetPrefix}clang $out/bin/clang";
+        };
+
       grammarHashes = {
         bash = "sha256-vRaN/mNfpR+hdv2HVS1bzaW0o+HGjizRFsk3iinICJE=";
         c = "sha256-gmzbdwvrKSo6C1fqTJFGxy8x0+T+vUTswm7F5sojzKc=";
@@ -90,11 +106,13 @@
 
             x86_64-windows = pkgs.pkgsCross.mingwW64;
           }
-          // (lib.optionalAttrs pkgs.stdenv.isDarwin {
+          // (lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin {
             x86_64-darwin = pkgs.pkgsCross.x86_64-darwin;
             aarch64-darwin = pkgs.pkgsCross.aarch64-darwin;
           });
 
+          # nixpkgs marks compiler-rt broken for riscv32
+          cliCrossTargets = lib.removeAttrs crossTargets [ "riscv32" ];
         in
         {
           default = self.packages.${system}.cli;
@@ -134,10 +152,11 @@
           wasm-test-grammars = pkgs.callPackage ./lib/binding_web/wasm-test-grammars.nix {
             inherit src version;
             inherit (self.packages.${system}) cli test-grammars;
+            wasi-sdk = wasiSdkFor pkgs;
           };
 
           web-tree-sitter = pkgs.callPackage ./lib/binding_web/package.nix {
-            inherit src version;
+            inherit src version npmDepsHash;
             inherit (self.packages.${system}) wasm-test-grammars;
           };
 
@@ -156,7 +175,7 @@
             inherit src version;
             inherit (self.packages.${system}) test-grammars;
           };
-        }) crossTargets)
+        }) cliCrossTargets)
         // (lib.mapAttrs' (arch: pkg: {
           name = "lib-${arch}";
           value = pkg.callPackage ./lib/package.nix {
@@ -200,7 +219,7 @@
                 echo "→ Rust..."
                 ${lib.getExe pkgs.cargo} fmt --all
                 echo "→ Nix..."
-                ${lib.getExe pkgs.nixfmt} ${filesWithExtension "nix"}
+                find . -name '*.nix' -exec ${lib.getExe pkgs.nixfmt} {} +
                 echo "→ Web (TypeScript/JavaScript)..."
                 cd lib/binding_web && ${pkgs.nodejs_22}/bin/npm install --silent && ${pkgs.nodejs_22}/bin/npm run lint:fix
                 cd ../..
@@ -223,7 +242,7 @@
                 echo "→ Running clippy..."
                 ${lib.getExe pkgs.cargo} clippy --workspace --all-targets -- -D warnings
                 echo "→ Checking Nix formatting..."
-                ${lib.getExe pkgs.nixfmt} --check ${filesWithExtension "nix"}
+                find . -name '*.nix' -exec ${lib.getExe pkgs.nixfmt} --check {} +
                 echo "→ Checking Web code..."
                 cd lib/binding_web && ${lib.getExe' pkgs.nodejs_22 "npm"} install --silent && ${lib.getExe' pkgs.nodejs_22 "npm"} run lint
                 cd ../..
@@ -248,14 +267,25 @@
             web-tree-sitter
             ;
 
-          nix-fmt = pkgs.runCommandNoCC "nix-fmt-check" { } ''
-            ${lib.getExe self.formatter.${system}} --check ${filesWithExtension "nix"}
-            touch $out
-          '';
-          rust-fmt = pkgs.runCommandNoCC "rust-fmt-check" { } ''
-            ${lib.getExe pkgs.rustfmt} --check
+          nix-fmt = pkgs.runCommand "nix-fmt-check" { } ''
+            find ${filesWithExtension "nix"} -name '*.nix' \
+              -exec ${lib.getExe self.formatter.${system}} --check {} +
             touch $out
           '';
+          rust-fmt =
+            pkgs.runCommand "rust-fmt-check"
+              {
+                nativeBuildInputs = with pkgs; [
+                  cargo
+                  rustfmt
+                ];
+              }
+              ''
+                export HOME=$TMPDIR
+                cd ${src}
+                cargo fmt --all --check
+                touch $out
+              '';
 
           rust-clippy = pkgs.rustPlatform.buildRustPackage {
             inherit src version;
@@ -286,12 +316,10 @@
           };
 
           web-lint = pkgs.buildNpmPackage {
-            inherit src version;
+            inherit src version npmDepsHash;
 
             pname = "web-tree-sitter-lint";
 
-            npmDepsHash = "sha256-y0GobcskcZTmju90TM64GjeWiBmPFCrTOg0yfccdB+Q=";
-
             postPatch = ''
               cp lib/binding_web/package{,-lock}.json .
             '';
@@ -336,8 +364,10 @@
               libclang
 
               nodejs_22
-              nodePackages.typescript
+              typescript
               emscripten
+              binaryen
+              lld
               pkgsCross.wasi32.stdenv.cc
 
               mdbook
@@ -396,7 +426,8 @@
               LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
               LLVM_COV = "${pkgs.llvm}/bin/llvm-cov";
               LLVM_PROFDATA = "${pkgs.llvm}/bin/llvm-profdata";
-              TREE_SITTER_WASI_SDK_PATH = "${pkgs.pkgsCross.wasi32.stdenv.cc}";
+              TREE_SITTER_WASI_SDK_PATH = "${wasiSdkFor pkgs}";
+              TREE_SITTER_BINARYEN_PATH = "${pkgs.binaryen}";
             };
           };
         }

lib/binding_web/package-lock.json

@@ -1,25 +1,25 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "web-tree-sitter",
-      "version": "0.27.0",
+      "version": "0.28.0",
       "license": "MIT",
       "devDependencies": {
         "@eslint/js": "^9.39.1",
-        "@types/emscripten": "^1.41.5",
-        "@types/node": "^26.2.0",
+        "@types/emscripten": "^1.41.6",
+        "@types/node": "^26.4.1",
         "@vitest/coverage-v8": "^3.0.5",
         "dts-buddy": "^0.8.3",
         "esbuild": "^0.28.2",
-        "eslint": "^10.9.0",
+        "eslint": "^10.10.0",
         "source-map": "^0.8.0",
-        "tsx": "^4.23.12",
+        "tsx": "^4.23.13",
         "typescript": "^5.7.3",
-        "typescript-eslint": "^8.67.0",
+        "typescript-eslint": "^8.69.0",
         "vitest": "^3.0.5"
       }
     },
@@ -97,6 +97,30 @@
         "node": ">=18"
       }
     },
+    "node_modules/@cacheable/memory": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz",
+      "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@cacheable/utils": "^2.5.0",
+        "@keyv/bigmap": "^1.3.1",
+        "hookified": "^1.15.1",
+        "keyv": "^5.6.0"
+      }
+    },
+    "node_modules/@cacheable/utils": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz",
+      "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "hashery": "^1.5.1",
+        "keyv": "^5.6.0"
+      }
+    },
     "node_modules/@esbuild/aix-ppc64": {
       "version": "0.28.2",
       "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz",
@@ -646,9 +670,9 @@
       }
     },
     "node_modules/@eslint/plugin-kit": {
-      "version": "0.7.2",
-      "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz",
-      "integrity": "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==",
+      "version": "0.7.3",
+      "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz",
+      "integrity": "sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==",
       "dev": true,
       "license": "Apache-2.0",
       "dependencies": {
@@ -660,29 +684,43 @@
       }
     },
     "node_modules/@humanfs/core": {
-      "version": "0.19.1",
-      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
-      "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==",
+      "version": "0.19.2",
+      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz",
+      "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==",
       "dev": true,
       "license": "Apache-2.0",
+      "dependencies": {
+        "@humanfs/types": "^0.15.0"
+      },
       "engines": {
         "node": ">=18.18.0"
       }
     },
     "node_modules/@humanfs/node": {
-      "version": "0.16.7",
-      "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz",
-      "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==",
+      "version": "0.16.8",
+      "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz",
+      "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==",
       "dev": true,
       "license": "Apache-2.0",
       "dependencies": {
-        "@humanfs/core": "^0.19.1",
+        "@humanfs/core": "^0.19.2",
+        "@humanfs/types": "^0.15.0",
         "@humanwhocodes/retry": "^0.4.0"
       },
       "engines": {
         "node": ">=18.18.0"
       }
     },
+    "node_modules/@humanfs/types": {
+      "version": "0.15.0",
+      "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz",
+      "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=18.18.0"
+      }
+    },
     "node_modules/@humanwhocodes/module-importer": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
@@ -789,6 +827,30 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@keyv/bigmap": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz",
+      "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "hashery": "^1.4.0",
+        "hookified": "^1.15.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "keyv": "^5.6.0"
+      }
+    },
+    "node_modules/@keyv/serialize": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz",
+      "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -1169,9 +1231,9 @@
       "license": "MIT"
     },
     "node_modules/@types/emscripten": {
-      "version": "1.41.5",
-      "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz",
-      "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==",
+      "version": "1.41.6",
+      "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz",
+      "integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==",
       "dev": true,
       "license": "MIT"
     },
@@ -1197,9 +1259,9 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "26.2.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz",
-      "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==",
+      "version": "26.4.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz",
+      "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -1207,17 +1269,17 @@
       }
     },
     "node_modules/@typescript-eslint/eslint-plugin": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.67.0.tgz",
-      "integrity": "sha512-Un7Heoyj65NREbKAyIrFxeM143NZpExWmy1Nep4DLeQOeLlTeumPjoNKnBrU5D5moWXbPJgRa5Uwcdu0faVNGQ==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.69.0.tgz",
+      "integrity": "sha512-t5jQTKPIgVW1PE6dR6H6Qz5gm8zjMlX5/2gRaOGd9eO6V7J+tQc6iWKukEe7dY8u9HyYasQ0yfF0/FSSTEO2gA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@eslint-community/regexpp": "^4.12.2",
-        "@typescript-eslint/scope-manager": "8.67.0",
-        "@typescript-eslint/type-utils": "8.67.0",
-        "@typescript-eslint/utils": "8.67.0",
-        "@typescript-eslint/visitor-keys": "8.67.0",
+        "@typescript-eslint/scope-manager": "8.69.0",
+        "@typescript-eslint/type-utils": "8.69.0",
+        "@typescript-eslint/utils": "8.69.0",
+        "@typescript-eslint/visitor-keys": "8.69.0",
         "ignore": "^7.0.5",
         "natural-compare": "^1.4.0",
         "ts-api-utils": "^2.5.0"
@@ -1230,15 +1292,15 @@
         "url": "https://opencollective.com/typescript-eslint"
       },
       "peerDependencies": {
-        "@typescript-eslint/parser": "^8.67.0",
+        "@typescript-eslint/parser": "^8.69.0",
         "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0",
         "typescript": ">=4.8.4 <6.1.0"
       }
     },
     "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz",
-      "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==",
+      "version": "7.0.8",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.8.tgz",
+      "integrity": "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -1259,16 +1321,16 @@
       }
     },
     "node_modules/@typescript-eslint/parser": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.67.0.tgz",
-      "integrity": "sha512-fUBfTuuEulWqX6V8+O3PtScV01tzYYRUDTAirHFKoRAt7nOzoGiPt0M/bB47wWNy0coOOcgEwAMUtBpykMxl6w==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.69.0.tgz",
+      "integrity": "sha512-l4b0DhWioGg6Gt2ebGlvfkFMOjRsauxtsnDRwUSRX1qHq3HdTfQHV8wW9zEXeciai6HfeaKOedQn2Zoofx3WBw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/scope-manager": "8.67.0",
-        "@typescript-eslint/types": "8.67.0",
-        "@typescript-eslint/typescript-estree": "8.67.0",
-        "@typescript-eslint/visitor-keys": "8.67.0",
+        "@typescript-eslint/scope-manager": "8.69.0",
+        "@typescript-eslint/types": "8.69.0",
+        "@typescript-eslint/typescript-estree": "8.69.0",
+        "@typescript-eslint/visitor-keys": "8.69.0",
         "debug": "^4.4.3"
       },
       "engines": {
@@ -1284,14 +1346,14 @@
       }
     },
     "node_modules/@typescript-eslint/project-service": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.67.0.tgz",
-      "integrity": "sha512-cvE8c7ulYeXN9fYuszhCeCsbzyVEXuhrRCybnBre7TUmqb5nRmBfQAwCj0O3WJFDeyAZt4VYv51vMCC9LHSdYw==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.69.0.tgz",
+      "integrity": "sha512-yi4obFrHMmnsesWehHbkg9zMA7Jt8cXT+mKM08G999pH1yT6nqgsHx7MYm0uY1wAj8CqiBXYRJ7WAT0QdQHQXg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/tsconfig-utils": "^8.67.0",
-        "@typescript-eslint/types": "^8.67.0",
+        "@typescript-eslint/tsconfig-utils": "^8.69.0",
+        "@typescript-eslint/types": "^8.69.0",
         "debug": "^4.4.3"
       },
       "engines": {
@@ -1306,14 +1368,14 @@
       }
     },
     "node_modules/@typescript-eslint/scope-manager": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.67.0.tgz",
-      "integrity": "sha512-EgvsleTwS4E+WzzSvem8fAUubLwatMNF1B5hHSLQxcvs7q2dtRhGyujHwLJSYlG41niJ7GP24Aha2+0mb1b2kg==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.69.0.tgz",
+      "integrity": "sha512-ewfspqWvSxKSOaplqAUNbaSFO0eB6w1EtQ+esfYFRm3614Ty4uNtExkcbgd6nWsXphbqKyf9ZYdbZdv2xEoWEQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/types": "8.67.0",
-        "@typescript-eslint/visitor-keys": "8.67.0"
+        "@typescript-eslint/types": "8.69.0",
+        "@typescript-eslint/visitor-keys": "8.69.0"
       },
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@@ -1324,9 +1386,9 @@
       }
     },
     "node_modules/@typescript-eslint/tsconfig-utils": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.67.0.tgz",
-      "integrity": "sha512-vV+LUSv5njUWsknE71fqKTlXUva+R76SaeORd6Zojcunk/6DvKFXONU3BrAs2H49mbygUXt6gbYunzwqNwlhdg==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.69.0.tgz",
+      "integrity": "sha512-xNqK7YTDZsLniQMV/4rpFR8Z5JlqeRvVjuG1YgF/mdPVH84HSD19L8CczMA0qg2RfwEV231GHH3VnToJDo4MfQ==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -1341,15 +1403,15 @@
       }
     },
     "node_modules/@typescript-eslint/type-utils": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.67.0.tgz",
-      "integrity": "sha512-aVWDXbRmdXO9siTfX4ditQI1T9+zVcNazT48EJCD0v40/9RIFoUgZ05CmGEq9H2gixRpjUn/iplwvlcvutJW/Q==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.69.0.tgz",
+      "integrity": "sha512-ZfoJAVg3JZndQEpEl9petVlxau3lRuElc4HRMuAlLCf8to04/iHz692RUSNmXKDjEuJmIL+KZ2/BsOcBc16dsA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/types": "8.67.0",
-        "@typescript-eslint/typescript-estree": "8.67.0",
-        "@typescript-eslint/utils": "8.67.0",
+        "@typescript-eslint/types": "8.69.0",
+        "@typescript-eslint/typescript-estree": "8.69.0",
+        "@typescript-eslint/utils": "8.69.0",
         "debug": "^4.4.3",
         "ts-api-utils": "^2.5.0"
       },
@@ -1379,9 +1441,9 @@
       }
     },
     "node_modules/@typescript-eslint/types": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.67.0.tgz",
-      "integrity": "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.69.0.tgz",
+      "integrity": "sha512-K3VrubUPhlo9VDBS6QdI8YB5j7ClpqLRdefcz6PFrhnwicehBweqQ9Evhl4l+FYz0HdDmMqIiSX0aldGRYtDCA==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -1393,16 +1455,16 @@
       }
     },
     "node_modules/@typescript-eslint/typescript-estree": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.67.0.tgz",
-      "integrity": "sha512-EKQBCE9yNlRJYm7jdTW5AhDacDUmSwQb0FAJAmK2EKYrNXIsa2vxcSZx6PvJ/dEdI6lS+Y9W+EXckLj0iPFGcw==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.69.0.tgz",
+      "integrity": "sha512-AdFkgqck3Vudb/kWnxlyafU/4aBhHrbQ9locP2N4psXTy5mOBg0SHJumnLvx7r6g1gV4DKvUFwV2nJZBoqOD8w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/project-service": "8.67.0",
-        "@typescript-eslint/tsconfig-utils": "8.67.0",
-        "@typescript-eslint/types": "8.67.0",
-        "@typescript-eslint/visitor-keys": "8.67.0",
+        "@typescript-eslint/project-service": "8.69.0",
+        "@typescript-eslint/tsconfig-utils": "8.69.0",
+        "@typescript-eslint/types": "8.69.0",
+        "@typescript-eslint/visitor-keys": "8.69.0",
         "debug": "^4.4.3",
         "minimatch": "^10.2.2",
         "semver": "^7.7.3",
@@ -1434,16 +1496,16 @@
       }
     },
     "node_modules/@typescript-eslint/utils": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.67.0.tgz",
-      "integrity": "sha512-U9D1FdwEWBwok3hxxSdhclMb0twvt9QnjIQ0VfQ1AiX2epnpSgv2ubVDsayOFyY8K6FX+AQ7E0FKWVG3iKsj1A==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.69.0.tgz",
+      "integrity": "sha512-tUbx60BBqQa31kXF5MCsOOLL5E/WzUuxIn7YpAvq+eaUlqvk8/NXnXMBNAdLCr0icjkzem7iUA5QqWHe/hJ1aw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.9.1",
-        "@typescript-eslint/scope-manager": "8.67.0",
-        "@typescript-eslint/types": "8.67.0",
-        "@typescript-eslint/typescript-estree": "8.67.0"
+        "@typescript-eslint/scope-manager": "8.69.0",
+        "@typescript-eslint/types": "8.69.0",
+        "@typescript-eslint/typescript-estree": "8.69.0"
       },
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@@ -1458,13 +1520,13 @@
       }
     },
     "node_modules/@typescript-eslint/visitor-keys": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.67.0.tgz",
-      "integrity": "sha512-fkv8dHRDqfGtTHuJeebdrQ7cX6Ad4WAS00rgHh9UGvMycF1mjBfsxry1XsLIFhWZ6Judlh6UdzK+TYlbpCXgnA==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.69.0.tgz",
+      "integrity": "sha512-+rmdgPA+EXkNgKYvHvFfhrs35utXbwaC5PGpDquSXcoXQDKUA5UjV0LmTucG/4JXkM31BTu4TilHtrN8IVBe8w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/types": "8.67.0",
+        "@typescript-eslint/types": "8.69.0",
         "eslint-visitor-keys": "^5.0.0"
       },
       "engines": {
@@ -1755,6 +1817,20 @@
         "node": ">=8"
       }
     },
+    "node_modules/cacheable": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz",
+      "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@cacheable/memory": "^2.2.0",
+        "@cacheable/utils": "^2.5.0",
+        "hookified": "^1.15.0",
+        "keyv": "^5.6.0",
+        "qified": "^0.10.1"
+      }
+    },
     "node_modules/chai": {
       "version": "5.3.3",
       "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz",
@@ -1952,9 +2028,9 @@
       }
     },
     "node_modules/eslint": {
-      "version": "10.9.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.9.0.tgz",
-      "integrity": "sha512-5KeEOJZBfEVA47boFiBsf+6MmmJpffM7qEBg4pLla2e4nlKgdKlqCW0oSLOGsT8Wl5uCGJptLV1bkaiShj90Gw==",
+      "version": "10.10.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.10.0.tgz",
+      "integrity": "sha512-NPXn6r5zl4uET1DAVPaOwzX3rut4c0wcmw3dWJAfOsTM5+TogXo0DDjz8pwm/hL8cyVNpHqeK4JpN0NjnyFFNw==",
       "dev": true,
       "license": "MIT",
       "workspaces": [
@@ -1966,7 +2042,7 @@
         "@eslint/config-array": "^0.23.5",
         "@eslint/config-helpers": "^0.7.0",
         "@eslint/core": "^1.2.1",
-        "@eslint/plugin-kit": "^0.7.2",
+        "@eslint/plugin-kit": "^0.7.3",
         "@humanfs/node": "^0.16.6",
         "@humanwhocodes/module-importer": "^1.0.1",
         "@humanwhocodes/retry": "^0.4.2",
@@ -1981,7 +2057,7 @@
         "esquery": "^1.7.0",
         "esutils": "^2.0.2",
         "fast-deep-equal": "^3.1.3",
-        "file-entry-cache": "^8.0.0",
+        "file-entry-cache": "11.1.5 || >11.1.6 <12",
         "find-up": "^5.0.0",
         "glob-parent": "^6.0.2",
         "ignore": "^5.2.0",
@@ -2166,16 +2242,13 @@
       }
     },
     "node_modules/file-entry-cache": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
-      "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==",
+      "version": "11.1.5",
+      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz",
+      "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "flat-cache": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=16.0.0"
+        "flat-cache": "^6.1.23"
       }
     },
     "node_modules/find-up": {
@@ -2196,23 +2269,21 @@
       }
     },
     "node_modules/flat-cache": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz",
-      "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==",
+      "version": "6.1.23",
+      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz",
+      "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "flatted": "^3.2.9",
-        "keyv": "^4.5.4"
-      },
-      "engines": {
-        "node": ">=16"
+        "cacheable": "^2.5.0",
+        "flatted": "^3.4.2",
+        "hookified": "^1.15.0"
       }
     },
     "node_modules/flatted": {
-      "version": "3.4.2",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
-      "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
+      "version": "3.4.4",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz",
+      "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==",
       "dev": true,
       "license": "ISC"
     },
@@ -2319,6 +2390,26 @@
         "node": ">=8"
       }
     },
+    "node_modules/hashery": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz",
+      "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "hookified": "^1.15.0"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/hookified": {
+      "version": "1.15.1",
+      "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz",
+      "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/html-escaper": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
@@ -2463,13 +2554,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/json-buffer": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
-      "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -2485,13 +2569,13 @@
       "license": "MIT"
     },
     "node_modules/keyv": {
-      "version": "4.5.4",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
-      "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz",
+      "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "json-buffer": "3.0.1"
+        "@keyv/serialize": "^1.1.1"
       }
     },
     "node_modules/kleur": {
@@ -2842,6 +2926,26 @@
         "node": ">=6"
       }
     },
+    "node_modules/qified": {
+      "version": "0.10.1",
+      "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz",
+      "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "hookified": "^2.1.1"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/qified/node_modules/hookified": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz",
+      "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/rollup": {
       "version": "4.60.1",
       "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.1.tgz",
@@ -3217,9 +3321,9 @@
       }
     },
     "node_modules/tsx": {
-      "version": "4.23.12",
-      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz",
-      "integrity": "sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==",
+      "version": "4.23.13",
+      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz",
+      "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -3263,16 +3367,16 @@
       }
     },
     "node_modules/typescript-eslint": {
-      "version": "8.67.0",
-      "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.67.0.tgz",
-      "integrity": "sha512-S2udFs8tCKEKffuJ4TB1idGUZiXdCPGi3IPBGWXarbLQ5UPXORV8QEVzJ4gCRduURMb5EkpNCdjbk0eDIuI8Yg==",
+      "version": "8.69.0",
+      "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.69.0.tgz",
+      "integrity": "sha512-B3MltX0VqjUBNEe3b3sSuiRbfa6XrfHFtBiPamjT5AsW/dfq+y+bc0wyuS9DxAS1LyzCxRp2+rxzpLUvqM2BvA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/eslint-plugin": "8.67.0",
-        "@typescript-eslint/parser": "8.67.0",
-        "@typescript-eslint/typescript-estree": "8.67.0",
-        "@typescript-eslint/utils": "8.67.0"
+        "@typescript-eslint/eslint-plugin": "8.69.0",
+        "@typescript-eslint/parser": "8.69.0",
+        "@typescript-eslint/typescript-estree": "8.69.0",
+        "@typescript-eslint/utils": "8.69.0"
       },
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "description": "Tree-sitter bindings for the web",
   "repository": {
     "type": "git",
@@ -75,16 +75,16 @@
   ],
   "devDependencies": {
     "@eslint/js": "^9.39.1",
-    "@types/emscripten": "^1.41.5",
-    "@types/node": "^26.2.0",
+    "@types/emscripten": "^1.41.6",
+    "@types/node": "^26.4.1",
     "@vitest/coverage-v8": "^3.0.5",
     "dts-buddy": "^0.8.3",
     "esbuild": "^0.28.2",
-    "eslint": "^10.9.0",
+    "eslint": "^10.10.0",
     "source-map": "^0.8.0",
-    "tsx": "^4.23.12",
+    "tsx": "^4.23.13",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.67.0",
+    "typescript-eslint": "^8.69.0",
     "vitest": "^3.0.5"
   },
   "scripts": {

lib/binding_web/package.nix

@@ -8,14 +8,13 @@
   emscripten,
   src,
   version,
+  npmDepsHash,
 }:
 buildNpmPackage {
-  inherit src version;
+  inherit src version npmDepsHash;
 
   pname = "web-tree-sitter";
 
-  npmDepsHash = "sha256-y0GobcskcZTmju90TM64GjeWiBmPFCrTOg0yfccdB+Q=";
-
   nativeBuildInputs = [
     rustPlatform.cargoSetupHook
     cargo
@@ -27,10 +26,12 @@ buildNpmPackage {
     lockFile = ../../Cargo.lock;
   };
 
-  doCheck = true;
+  # emscripten is super behind here compared to nixpkgs upstream
+  doCheck = false;
 
   postPatch = ''
     cp lib/binding_web/package{,-lock}.json .
+    cp LICENSE lib/binding_web/
   '';
 
   buildPhase = ''
@@ -53,11 +54,20 @@ buildNpmPackage {
   '';
 
   checkPhase = ''
-    cd lib/binding_web && npm test
+    pushd lib/binding_web && npm test && popd
   '';
 
+  preInstall = ''
+    mv node_modules lib/binding_web/
+    cd lib/binding_web
+  '';
+
+  # `postpack` deletes the LICENSE that postPatch staged, and npmInstallHook
+  # copies the packed file list afterwards.
+  npmPackFlags = [ "--ignore-scripts" ];
+
   meta = {
-    description = "web-tree-sitter - WebAssembly bindings to the Tree-sitter parsing library.";
+    description = "WebAssembly bindings to the Tree-sitter parsing library.";
     longDescription = ''
       web-tree-sitter provides WebAssembly bindings to the Tree-sitter parsing library.
       It can build a concrete syntax tree for a source file and efficiently update

lib/binding_web/wasm-test-grammars.nix

@@ -1,12 +1,14 @@
 {
+  binaryen,
   cli,
   lib,
+  lld,
   nodejs_22,
-  pkgsCross,
   src,
   stdenv,
   test-grammars,
   version,
+  wasi-sdk,
 }:
 let
   grammars = [
@@ -28,14 +30,16 @@ stdenv.mkDerivation {
   pname = "wasm-test-grammars";
 
   nativeBuildInputs = [
+    binaryen
     cli
-    pkgsCross.wasi32.stdenv.cc
+    lld
     nodejs_22
   ];
 
   buildPhase = ''
     export HOME=$TMPDIR
-    export TREE_SITTER_WASI_SDK_PATH=${pkgsCross.wasi32.stdenv.cc}
+    export TREE_SITTER_WASI_SDK_PATH=${wasi-sdk}
+    export TREE_SITTER_BINARYEN_PATH=${binaryen}
     export NIX_LDFLAGS=""
 
     cp -r ${test-grammars}/fixtures .

lib/src/unicode.h

@@ -18,7 +18,7 @@ extern "C" {
     (c)=le16toh((s)[(i)++]); \
     if(U16_IS_LEAD(c)) { \
         uint16_t __c2; \
-        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=le16toh((s)[(i)]))) { \
             ++(i); \
             (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
         } \
@@ -29,7 +29,7 @@ extern "C" {
     (c)=be16toh((s)[(i)++]); \
     if(U16_IS_LEAD(c)) { \
         uint16_t __c2; \
-        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=be16toh((s)[(i)]))) { \
             ++(i); \
             (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
         } \