0.24.4

(cherry picked from commit ac4595e808)

Cargo.toml

@@ -13,7 +13,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.24.1"
+version = "0.24.4"
 authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
 edition = "2021"
 rust-version = "1.74.1"
@@ -96,9 +96,9 @@ walkdir = "2.5.0"
 wasmparser = "0.217.0"
 webbrowser = "1.0.2"
 
-tree-sitter = { version = "0.24.0", path = "./lib" }
-tree-sitter-generate = { version = "0.24.0", path = "./cli/generate" }
-tree-sitter-loader = { version = "0.24.0", path = "./cli/loader" }
-tree-sitter-config = { version = "0.24.0", path = "./cli/config" }
-tree-sitter-highlight = { version = "0.24.0", path = "./highlight" }
-tree-sitter-tags = { version = "0.24.0", path = "./tags" }
+tree-sitter = { version = "0.24.4", path = "./lib" }
+tree-sitter-generate = { version = "0.24.4", path = "./cli/generate" }
+tree-sitter-loader = { version = "0.24.4", path = "./cli/loader" }
+tree-sitter-config = { version = "0.24.4", path = "./cli/config" }
+tree-sitter-highlight = { version = "0.24.4", path = "./highlight" }
+tree-sitter-tags = { version = "0.24.4", path = "./tags" }

Makefile

@@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT)
 $(error Windows is not supported)
 endif
 
-VERSION := 0.24.1
+VERSION := 0.24.4
 DESCRIPTION := An incremental parsing system for programming tools
 HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
@@ -62,8 +62,8 @@ endif
 
 tree-sitter.pc: lib/tree-sitter.pc.in
 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
-		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR)|' \
-		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR)|' \
+		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
+		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \
 		-e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \
 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@

build.zig.zon

@@ -1,6 +1,6 @@
 .{
     .name = "tree-sitter",
-    .version = "0.24.1",
+    .version = "0.24.4",
     .paths = .{
         "build.zig",
         "build.zig.zon",

cli/generate/src/build_tables/minimize_parse_table.rs

@@ -70,18 +70,17 @@ impl<'a> Minimizer<'a> {
                             production_id: 0,
                             symbol,
                             ..
-                        } => {
-                            if !self.simple_aliases.contains_key(symbol)
-                                && !self.syntax_grammar.supertype_symbols.contains(symbol)
-                                && !aliased_symbols.contains(symbol)
-                                && self.syntax_grammar.variables[symbol.index].kind
-                                    != VariableType::Named
-                                && (unit_reduction_symbol.is_none()
-                                    || unit_reduction_symbol == Some(symbol))
-                            {
-                                unit_reduction_symbol = Some(symbol);
-                                continue;
-                            }
+                        } if !self.simple_aliases.contains_key(symbol)
+                            && !self.syntax_grammar.supertype_symbols.contains(symbol)
+                            && !self.syntax_grammar.extra_symbols.contains(symbol)
+                            && !aliased_symbols.contains(symbol)
+                            && self.syntax_grammar.variables[symbol.index].kind
+                                != VariableType::Named
+                            && (unit_reduction_symbol.is_none()
+                                || unit_reduction_symbol == Some(symbol)) =>
+                        {
+                            unit_reduction_symbol = Some(symbol);
+                            continue;
                         }
                         _ => {}
                     }

cli/generate/src/prepare_grammar/intern_symbols.rs

@@ -149,7 +149,7 @@ impl<'a> Interner<'a> {
     fn check_single(&self, elements: &[Rule], name: Option<&str>) {
         if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) {
             eprintln!(
-                "Warning: rule {} is just a `seq` or `choice` rule with a single element. This is unnecessary.",
+                "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.",
                 name.unwrap_or_default()
             );
         }

cli/loader/Cargo.toml

@@ -24,6 +24,7 @@ cc.workspace = true
 dirs.workspace = true
 fs4.workspace = true
 indoc.workspace = true
+lazy_static.workspace = true
 libloading.workspace = true
 once_cell.workspace = true
 path-slash.workspace = true

cli/loader/src/lib.rs

@@ -21,6 +21,7 @@ use anyhow::Error;
 use anyhow::{anyhow, Context, Result};
 use fs4::fs_std::FileExt;
 use indoc::indoc;
+use lazy_static::lazy_static;
 use libloading::{Library, Symbol};
 use once_cell::unsync::OnceCell;
 use path_slash::PathBufExt as _;
@@ -38,6 +39,10 @@ use tree_sitter_highlight::HighlightConfiguration;
 use tree_sitter_tags::{Error as TagsError, TagsConfiguration};
 use url::Url;
 
+lazy_static! {
+    static ref GRAMMAR_NAME_REGEX: Regex = Regex::new(r#""name":\s*"(.*?)""#).unwrap();
+}
+
 pub const EMSCRIPTEN_TAG: &str = concat!("docker.io/emscripten/emsdk:", env!("EMSCRIPTEN_VERSION"));
 
 #[derive(Default, Deserialize, Serialize)]
@@ -141,12 +146,10 @@ pub struct TreeSitterJSON {
 }
 
 impl TreeSitterJSON {
-    pub fn from_file(path: &Path) -> Option<Self> {
-        if let Ok(file) = fs::File::open(path.join("tree-sitter.json")) {
-            Some(serde_json::from_reader(file).ok()?)
-        } else {
-            None
-        }
+    pub fn from_file(path: &Path) -> Result<Self> {
+        Ok(serde_json::from_str(&fs::read_to_string(
+            path.join("tree-sitter.json"),
+        )?)?)
     }
 
     pub fn has_multiple_language_configs(&self) -> bool {
@@ -161,7 +164,8 @@ pub struct Grammar {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub camelcase: Option<String>,
     pub scope: String,
-    pub path: PathBuf,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub path: Option<PathBuf>,
     #[serde(default, skip_serializing_if = "PathsJSON::is_empty")]
     pub external_files: PathsJSON,
     pub file_types: Option<Vec<String>>,
@@ -192,7 +196,6 @@ pub struct Metadata {
     pub authors: Option<Vec<Author>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub links: Option<Links>,
-    // #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(skip)]
     pub namespace: Option<String>,
 }
@@ -600,6 +603,13 @@ impl Loader {
         }
     }
 
+    pub fn language_for_configuration(
+        &self,
+        configuration: &LanguageConfiguration,
+    ) -> Result<Language> {
+        self.language_for_id(configuration.language_id)
+    }
+
     fn language_for_id(&self, id: usize) -> Result<Language> {
         let (path, language, externals) = &self.languages_by_id[id];
         language
@@ -628,27 +638,7 @@ impl Loader {
 
     pub fn load_language_at_path(&self, mut config: CompileConfig) -> Result<Language> {
         let grammar_path = config.src_path.join("grammar.json");
-
-        #[derive(Deserialize)]
-        struct GrammarJSON {
-            name: String,
-        }
-        let mut grammar_file = fs::File::open(&grammar_path).with_context(|| {
-            format!(
-                "Failed to read grammar.json file at the following path:\n{:?}",
-                &grammar_path
-            )
-        })?;
-        let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file))
-            .with_context(|| {
-                format!(
-                    "Failed to parse grammar.json file at the following path:\n{:?}",
-                    &grammar_path
-                )
-            })?;
-
-        config.name = grammar_json.name;
-
+        config.name = Self::grammar_json_name(&grammar_path)?;
         self.load_language_at_path_with_name(config)
     }
 
@@ -1125,27 +1115,16 @@ impl Loader {
         parser_path: &Path,
         set_current_path_config: bool,
     ) -> Result<&[LanguageConfiguration]> {
-        #[derive(Deserialize)]
-        struct GrammarJSON {
-            name: String,
-        }
-
         let initial_language_configuration_count = self.language_configurations.len();
 
-        if let Some(config) = TreeSitterJSON::from_file(parser_path) {
+        let ts_json = TreeSitterJSON::from_file(parser_path);
+        if let Ok(config) = ts_json {
             let language_count = self.languages_by_id.len();
             for grammar in config.grammars {
                 // Determine the path to the parser directory. This can be specified in
-                // the package.json, but defaults to the directory containing the
-                // package.json.
-                let language_path = parser_path.join(grammar.path);
-
-                let grammar_path = language_path.join("src").join("grammar.json");
-                let mut grammar_file =
-                    fs::File::open(grammar_path).with_context(|| "Failed to read grammar.json")?;
-                let grammar_json: GrammarJSON =
-                    serde_json::from_reader(BufReader::new(&mut grammar_file))
-                        .with_context(|| "Failed to parse grammar.json")?;
+                // the tree-sitter.json, but defaults to the directory containing the
+                // tree-sitter.json.
+                let language_path = parser_path.join(grammar.path.unwrap_or(PathBuf::from(".")));
 
                 // Determine if a previous language configuration in this package.json file
                 // already uses the same language.
@@ -1184,7 +1163,7 @@ impl Loader {
 
                 let configuration = LanguageConfiguration {
                     root_path: parser_path.to_path_buf(),
-                    language_name: grammar_json.name,
+                    language_name: grammar.name,
                     scope: Some(grammar.scope),
                     language_id,
                     file_types: grammar.file_types.unwrap_or_default(),
@@ -1230,20 +1209,30 @@ impl Loader {
                         Some(self.language_configurations.len() - 1);
                 }
             }
+        } else if let Err(e) = ts_json {
+            match e.downcast_ref::<std::io::Error>() {
+                // This is noisy, and not really an issue.
+                Some(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+                _ => {
+                    eprintln!(
+                        "Warning: Failed to parse {} -- {e}",
+                        parser_path.join("tree-sitter.json").display()
+                    );
+                }
+            }
         }
 
+        // If we didn't find any language configurations in the tree-sitter.json file,
+        // but there is a grammar.json file, then use the grammar file to form a simple
+        // language configuration.
         if self.language_configurations.len() == initial_language_configuration_count
             && parser_path.join("src").join("grammar.json").exists()
         {
             let grammar_path = parser_path.join("src").join("grammar.json");
-            let mut grammar_file =
-                fs::File::open(grammar_path).with_context(|| "Failed to read grammar.json")?;
-            let grammar_json: GrammarJSON =
-                serde_json::from_reader(BufReader::new(&mut grammar_file))
-                    .with_context(|| "Failed to parse grammar.json")?;
+            let language_name = Self::grammar_json_name(&grammar_path)?;
             let configuration = LanguageConfiguration {
                 root_path: parser_path.to_owned(),
-                language_name: grammar_json.name,
+                language_name,
                 language_id: self.languages_by_id.len(),
                 file_types: Vec::new(),
                 scope: None,
@@ -1279,6 +1268,36 @@ impl Loader {
         pattern.and_then(|r| RegexBuilder::new(r).multi_line(true).build().ok())
     }
 
+    fn grammar_json_name(grammar_path: &Path) -> Result<String> {
+        let file = fs::File::open(grammar_path).with_context(|| {
+            format!("Failed to open grammar.json at {}", grammar_path.display())
+        })?;
+
+        let first_three_lines = BufReader::new(file)
+            .lines()
+            .take(3)
+            .collect::<Result<Vec<_>, _>>()
+            .with_context(|| {
+                format!(
+                    "Failed to read the first three lines of grammar.json at {}",
+                    grammar_path.display()
+                )
+            })?
+            .join("\n");
+
+        let name = GRAMMAR_NAME_REGEX
+            .captures(&first_three_lines)
+            .and_then(|c| c.get(1))
+            .ok_or_else(|| {
+                anyhow!(
+                    "Failed to parse the language name from grammar.json at {}",
+                    grammar_path.display()
+                )
+            })?;
+
+        Ok(name.as_str().to_string())
+    }
+
     pub fn select_language(
         &mut self,
         path: &Path,

cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.24.1",
+  "version": "0.24.4",
   "author": "Max Brunsfeld",
   "license": "MIT",
   "repository": {

cli/src/init.rs

@@ -1,17 +1,15 @@
 use std::{
-    fs::{self, File},
-    io::BufReader,
+    fs,
     path::{Path, PathBuf},
     str::{self, FromStr},
 };
 
 use anyhow::{anyhow, Context, Result};
 use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
-use indoc::indoc;
 use regex::Regex;
 use semver::Version;
 use serde::{Deserialize, Serialize};
-use serde_json::{json, Map, Value};
+use serde_json::{Map, Value};
 use tree_sitter_generate::write_file;
 use tree_sitter_loader::{
     Author, Bindings, Grammar, Links, Metadata, PackageJSON, PackageJSONAuthor,
@@ -77,7 +75,7 @@ const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp");
 const BINDING_TEST_JS_TEMPLATE: &str = include_str!("./templates/binding_test.js");
 
 const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile");
-const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.txt");
+const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.cmake");
 const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h");
 const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in");
 
@@ -112,22 +110,6 @@ pub fn path_in_ignore(repo_path: &Path) -> bool {
     .any(|dir| repo_path.ends_with(dir))
 }
 
-fn insert_after(
-    map: Map<String, Value>,
-    after: &str,
-    key: &str,
-    value: Value,
-) -> Map<String, Value> {
-    let mut entries = map.into_iter().collect::<Vec<_>>();
-    let after_index = entries
-        .iter()
-        .position(|(k, _)| k == after)
-        .unwrap_or(entries.len() - 1)
-        + 1;
-    entries.insert(after_index, (key.to_string(), value));
-    entries.into_iter().collect()
-}
-
 #[derive(Serialize, Deserialize, Clone)]
 pub struct JsonConfigOpts {
     pub name: String,
@@ -153,9 +135,9 @@ impl JsonConfigOpts {
                 name: self.name.clone(),
                 camelcase: Some(self.camelcase),
                 scope: self.scope,
-                path: PathBuf::from("."),
+                path: None,
                 external_files: PathsJSON::Empty,
-                file_types: None,
+                file_types: Some(self.file_types),
                 highlights: PathsJSON::Empty,
                 injections: PathsJSON::Empty,
                 locals: PathsJSON::Empty,
@@ -171,7 +153,7 @@ impl JsonConfigOpts {
                 authors: Some(vec![Author {
                     name: self.author,
                     email: self.email,
-                    url: None,
+                    url: self.url.map(|url| url.to_string()),
                 }]),
                 links: Some(Links {
                     repository: self.repository.unwrap_or_else(|| {
@@ -216,6 +198,7 @@ struct GenerateOpts<'a> {
     description: Option<&'a str>,
     repository: Option<&'a str>,
     version: &'a Version,
+    camel_parser_name: &'a str,
 }
 
 // TODO: remove in 0.25
@@ -228,9 +211,9 @@ pub fn migrate_package_json(repo_path: &Path) -> Result<bool> {
         root_path.join("tree-sitter.json"),
     );
 
-    let old_config = serde_json::from_reader::<_, PackageJSON>(
-        File::open(&package_json_path)
-            .with_context(|| format!("Failed to open package.json in {}", root_path.display()))?,
+    let old_config = serde_json::from_str::<PackageJSON>(
+        &fs::read_to_string(&package_json_path)
+            .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?,
     )?;
 
     if old_config.tree_sitter.is_none() {
@@ -249,7 +232,7 @@ pub fn migrate_package_json(repo_path: &Path) -> Result<bool> {
                 name: name.clone(),
                 camelcase: Some(name.to_upper_camel_case()),
                 scope: l.scope.unwrap_or_else(|| format!("source.{name}")),
-                path: l.path,
+                path: Some(l.path),
                 external_files: l.external_files,
                 file_types: l.file_types,
                 highlights: l.highlights,
@@ -352,19 +335,19 @@ pub fn migrate_package_json(repo_path: &Path) -> Result<bool> {
 
     write_file(
         &tree_sitter_json_path,
-        serde_json::to_string_pretty(&new_config)?,
+        serde_json::to_string_pretty(&new_config)? + "\n",
     )?;
 
     // Remove the `tree-sitter` field in-place
-    let mut package_json = serde_json::from_reader::<_, Map<String, Value>>(
-        File::open(&package_json_path)
-            .with_context(|| format!("Failed to open package.json in {}", root_path.display()))?,
+    let mut package_json = serde_json::from_str::<Map<String, Value>>(
+        &fs::read_to_string(&package_json_path)
+            .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?,
     )
     .unwrap();
     package_json.remove("tree-sitter");
     write_file(
         &root_path.join("package.json"),
-        serde_json::to_string_pretty(&package_json)?,
+        serde_json::to_string_pretty(&package_json)? + "\n",
     )?;
 
     println!("Warning: your package.json's `tree-sitter` field has been automatically migrated to the new `tree-sitter.json` config file");
@@ -383,8 +366,6 @@ pub fn generate_grammar_files(
 ) -> Result<()> {
     let dashed_language_name = language_name.to_kebab_case();
 
-    // TODO: remove legacy code updates in v0.24.0
-
     let tree_sitter_config = missing_path_else(
         repo_path.join("tree-sitter.json"),
         true,
@@ -407,12 +388,16 @@ pub fn generate_grammar_files(
         },
     )?;
 
-    let tree_sitter_config = serde_json::from_reader::<_, TreeSitterJSON>(
-        File::open(tree_sitter_config.as_path())
-            .with_context(|| "Failed to open tree-sitter.json")?,
+    let tree_sitter_config = serde_json::from_str::<TreeSitterJSON>(
+        &fs::read_to_string(tree_sitter_config.as_path())
+            .with_context(|| "Failed to read tree-sitter.json")?,
     )?;
 
     let authors = tree_sitter_config.metadata.authors.as_ref();
+    let camel_name = tree_sitter_config.grammars[0]
+        .camelcase
+        .clone()
+        .unwrap_or_else(|| language_name.to_upper_camel_case());
 
     let generate_opts = GenerateOpts {
         author_name: authors
@@ -432,126 +417,18 @@ pub fn generate_grammar_files(
             .as_ref()
             .map(|l| l.repository.as_str()),
         version: &tree_sitter_config.metadata.version,
+        camel_parser_name: &camel_name,
     };
 
-    // Create or update package.json
-    missing_path_else(
-        repo_path.join("package.json"),
-        allow_update,
-        |path| {
-            generate_file(
-                path,
-                PACKAGE_JSON_TEMPLATE,
-                dashed_language_name.as_str(),
-                &generate_opts,
-            )
-        },
-        |path| {
-            let package_json_str =
-                fs::read_to_string(path).with_context(|| "Failed to read package.json")?;
-            let mut package_json = serde_json::from_str::<Map<String, Value>>(&package_json_str)
-                .with_context(|| "Failed to parse package.json")?;
-            let mut updated = false;
-
-            let dependencies = package_json
-                .entry("dependencies".to_string())
-                .or_insert_with(|| Value::Object(Map::new()))
-                .as_object_mut()
-                .unwrap();
-            if dependencies.remove("nan").is_some() {
-                eprintln!("Replacing nan dependency with node-addon-api in package.json");
-                dependencies.insert("node-addon-api".to_string(), "^8.0.0".into());
-                updated = true;
-            }
-            if !dependencies.contains_key("node-gyp-build") {
-                eprintln!("Adding node-gyp-build dependency to package.json");
-                dependencies.insert("node-gyp-build".to_string(), "^4.8.1".into());
-                updated = true;
-            }
-
-            let dev_dependencies = package_json
-                .entry("devDependencies".to_string())
-                .or_insert_with(|| Value::Object(Map::new()))
-                .as_object_mut()
-                .unwrap();
-            if !dev_dependencies.contains_key("prebuildify") {
-                eprintln!("Adding prebuildify devDependency to package.json");
-                dev_dependencies.insert("prebuildify".to_string(), "^6.0.1".into());
-                updated = true;
-            }
-
-            let node_test = "node --test bindings/node/*_test.js";
-            let scripts = package_json
-                .entry("scripts".to_string())
-                .or_insert_with(|| Value::Object(Map::new()))
-                .as_object_mut()
-                .unwrap();
-            if !scripts.get("test").is_some_and(|v| v == node_test) {
-                eprintln!("Updating package.json scripts");
-                *scripts = Map::from_iter([
-                    ("install".to_string(), "node-gyp-build".into()),
-                    ("prestart".to_string(), "tree-sitter build --wasm".into()),
-                    ("start".to_string(), "tree-sitter playground".into()),
-                    ("test".to_string(), node_test.into()),
-                ]);
-                updated = true;
-            }
-
-            // insert `peerDependencies` after `dependencies`
-            if !package_json.contains_key("peerDependencies") {
-                eprintln!("Adding peerDependencies to package.json");
-                package_json = insert_after(
-                    package_json,
-                    "dependencies",
-                    "peerDependencies",
-                    json!({"tree-sitter": "^0.21.1"}),
-                );
-
-                package_json = insert_after(
-                    package_json,
-                    "peerDependencies",
-                    "peerDependenciesMeta",
-                    json!({"tree_sitter": {"optional": true}}),
-                );
-                updated = true;
-            }
-
-            // insert `types` right after `main`
-            if !package_json.contains_key("types") {
-                eprintln!("Adding types to package.json");
-                package_json = insert_after(package_json, "main", "types", "bindings/node".into());
-                updated = true;
-            }
-
-            // insert `files` right after `keywords`
-            if !package_json.contains_key("files") {
-                eprintln!("Adding files to package.json");
-                package_json = insert_after(
-                    package_json,
-                    "keywords",
-                    "files",
-                    json!([
-                        "grammar.js",
-                        "binding.gyp",
-                        "prebuilds/**",
-                        "bindings/node/*",
-                        "queries/*",
-                        "src/**",
-                        "*.wasm"
-                    ]),
-                );
-                updated = true;
-            }
-
-            if updated {
-                let mut package_json_str = serde_json::to_string_pretty(&package_json)?;
-                package_json_str.push('\n');
-                write_file(path, package_json_str)?;
-            }
-
-            Ok(())
-        },
-    )?;
+    // Create package.json
+    missing_path(repo_path.join("package.json"), |path| {
+        generate_file(
+            path,
+            PACKAGE_JSON_TEMPLATE,
+            dashed_language_name.as_str(),
+            &generate_opts,
+        )
+    })?;
 
     // Do not create a grammar.js file in a repo with multiple language configs
     if !tree_sitter_config.has_multiple_language_configs() {
@@ -580,83 +457,22 @@ pub fn generate_grammar_files(
     // Generate Rust bindings
     if tree_sitter_config.bindings.rust {
         missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| {
-            missing_path_else(
-                path.join("lib.rs"),
-                allow_update,
-                |path| generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts),
-                |path| {
-                    let lib_rs =
-                        fs::read_to_string(path).with_context(|| "Failed to read lib.rs")?;
-                    if !lib_rs.contains("tree_sitter_language") {
-                        generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts)?;
-                        eprintln!("Updated lib.rs with `tree_sitter_language` dependency");
-                    }
-                    Ok(())
-                },
-            )?;
+            missing_path(path.join("lib.rs"), |path| {
+                generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts)
+            })?;
 
-            missing_path_else(
-                path.join("build.rs"),
-                allow_update,
-                |path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts),
-                |path| {
-                    let build_rs =
-                        fs::read_to_string(path).with_context(|| "Failed to read build.rs")?;
-                    if !build_rs.contains("-utf-8") {
-                        let index = build_rs
-                            .find("    let parser_path = src_dir.join(\"parser.c\")")
-                            .ok_or_else(|| anyhow!(indoc!{
-                                "Failed to auto-update build.rs with the `/utf-8` flag for windows.
-                                To fix this, remove `bindings/rust/build.rs` and re-run `tree-sitter generate`"}))?;
+            missing_path(path.join("build.rs"), |path| {
+                generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts)
+            })?;
 
-                        let build_rs = format!(
-                            "{}{}{}\n{}",
-                            &build_rs[..index],
-                            "    #[cfg(target_env = \"msvc\")]\n",
-                            "    c_config.flag(\"-utf-8\");\n",
-                            &build_rs[index..]
-                        );
-
-                        write_file(path, build_rs)?;
-                        eprintln!("Updated build.rs with the /utf-8 flag for Windows compilation");
-                    }
-                    Ok(())
-                },
-            )?;
-
-            missing_path_else(
-                repo_path.join("Cargo.toml"),
-                allow_update,
-                |path| generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str(), &generate_opts),
-                |path| {
-                    let cargo_toml =
-                        fs::read_to_string(path).with_context(|| "Failed to read Cargo.toml")?;
-                    if !cargo_toml.contains("tree-sitter-language") {
-                        let start_index = cargo_toml
-                            .find("tree-sitter = \"")
-                            .ok_or_else(|| anyhow!("Failed to find the `tree-sitter` dependency in Cargo.toml"))?;
-
-                        let version_start_index = start_index + "tree-sitter = \"".len();
-                        let version_end_index = cargo_toml[version_start_index..]
-                            .find('\"')
-                            .map(|i| i + version_start_index)
-                            .ok_or_else(|| anyhow!("Failed to find the end of the `tree-sitter` version in Cargo.toml"))?;
-
-                        let cargo_toml = format!(
-                            "{}{}{}\n{}\n{}",
-                            &cargo_toml[..start_index],
-                            "tree-sitter-language = \"0.1.0\"",
-                            &cargo_toml[version_end_index + 1..],
-                            "[dev-dependencies]",
-                            "tree-sitter = \"0.23\"",
-                        );
-
-                        write_file(path, cargo_toml)?;
-                        eprintln!("Updated Cargo.toml with the `tree-sitter-language` dependency");
-                    }
-                    Ok(())
-                },
-            )?;
+            missing_path(repo_path.join("Cargo.toml"), |path| {
+                generate_file(
+                    path,
+                    CARGO_TOML_TEMPLATE,
+                    dashed_language_name.as_str(),
+                    &generate_opts,
+                )
+            })?;
 
             Ok(())
         })?;
@@ -670,10 +486,8 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let index_js =
-                        fs::read_to_string(path).with_context(|| "Failed to read index.js")?;
-                    if index_js.contains("../../build/Release") {
-                        eprintln!("Replacing index.js with new binding API");
+                    let contents = fs::read_to_string(path)?;
+                    if !contents.contains("bun") {
                         generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -693,36 +507,13 @@ pub fn generate_grammar_files(
                 )
             })?;
 
-            missing_path_else(
-                path.join("binding.cc"),
-                allow_update,
-                |path| generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts),
-                |path| {
-                    let binding_cc =
-                        fs::read_to_string(path).with_context(|| "Failed to read binding.cc")?;
-                    if binding_cc.contains("NAN_METHOD(New) {}") {
-                        eprintln!("Replacing binding.cc with new binding API");
-                        generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts)?;
-                    }
-                    Ok(())
-                },
-            )?;
+            missing_path(path.join("binding.cc"), |path| {
+                generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts)
+            })?;
 
-            // Create binding.gyp, or update it with new binding API.
-            missing_path_else(
-                repo_path.join("binding.gyp"),
-                allow_update,
-                |path| generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts),
-                |path| {
-                    let binding_gyp =
-                        fs::read_to_string(path).with_context(|| "Failed to read binding.gyp")?;
-                    if binding_gyp.contains("require('nan')") {
-                        eprintln!("Replacing binding.gyp with new binding API");
-                        generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts)?;
-                    }
-                    Ok(())
-                },
-            )?;
+            missing_path(repo_path.join("binding.gyp"), |path| {
+                generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts)
+            })?;
 
             Ok(())
         })?;
@@ -752,9 +543,20 @@ pub fn generate_grammar_files(
                 generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)
             })?;
 
-            missing_path(repo_path.join("CMakeLists.txt"), |path| {
-                generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts)
-            })?;
+            missing_path_else(
+                repo_path.join("CMakeLists.txt"),
+                allow_update,
+                |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
+                |path| {
+                    let contents = fs::read_to_string(path)?;
+                    let old = "add_custom_target(test";
+                    if contents.contains(old) {
+                        write_file(path, contents.replace(old, "add_custom_target(ts-test"))
+                    } else {
+                        Ok(())
+                    }
+                },
+            )?;
 
             Ok(())
         })?;
@@ -767,39 +569,14 @@ pub fn generate_grammar_files(
                 generate_file(path, BINDING_GO_TEMPLATE, language_name, &generate_opts)
             })?;
 
-            missing_path_else(
-                path.join("binding_test.go"),
-                allow_update,
-                |path| {
-                    generate_file(
-                        path,
-                        BINDING_TEST_GO_TEMPLATE,
-                        language_name,
-                        &generate_opts,
-                    )
-                },
-                |path| {
-                    let binding_test_go = fs::read_to_string(path)
-                        .with_context(|| "Failed to read binding_test.go")?;
-                    if binding_test_go.contains("smacker") {
-                        eprintln!("Replacing binding_test.go with new binding API");
-                        generate_file(
-                            path,
-                            BINDING_TEST_GO_TEMPLATE,
-                            language_name,
-                            &generate_opts,
-                        )?;
-                    }
-                    Ok(())
-                },
-            )?;
-
-            // Delete the old go.mod file that lives inside bindings/go, it now lives in the root
-            // dir
-            let go_mod_path = path.join("go.mod");
-            if allow_update && go_mod_path.exists() {
-                fs::remove_file(go_mod_path).with_context(|| "Failed to remove old go.mod file")?;
-            }
+            missing_path(path.join("binding_test.go"), |path| {
+                generate_file(
+                    path,
+                    BINDING_TEST_GO_TEMPLATE,
+                    language_name,
+                    &generate_opts,
+                )
+            })?;
 
             missing_path(repo_path.join("go.mod"), |path| {
                 generate_file(path, GO_MOD_TEMPLATE, language_name, &generate_opts)
@@ -815,20 +592,9 @@ pub fn generate_grammar_files(
             let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
             missing_path(&lang_path, create_dir)?;
 
-            missing_path_else(
-                lang_path.join("binding.c"),
-                allow_update,
-                |path| generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts),
-                |path| {
-                    let binding_c = fs::read_to_string(path)
-                        .with_context(|| "Failed to read bindings/python/binding.c")?;
-                    if !binding_c.contains("PyCapsule_New") {
-                        eprintln!("Replacing bindings/python/binding.c with new binding API");
-                        generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts)?;
-                    }
-                    Ok(())
-                },
-            )?;
+            missing_path(lang_path.join("binding.c"), |path| {
+                generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts)
+            })?;
 
             missing_path(lang_path.join("__init__.py"), |path| {
                 generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)
@@ -874,7 +640,7 @@ pub fn generate_grammar_files(
     // Generate Swift bindings
     if tree_sitter_config.bindings.swift {
         missing_path(bindings_dir.join("swift"), create_dir)?.apply(|path| {
-            let lang_path = path.join(format!("TreeSitter{}", language_name.to_upper_camel_case()));
+            let lang_path = path.join(format!("TreeSitter{camel_name}",));
             missing_path(&lang_path, create_dir)?;
 
             missing_path(lang_path.join(format!("{language_name}.h")), |path| {
@@ -882,18 +648,12 @@ pub fn generate_grammar_files(
             })?;
 
             missing_path(
-                path.join(format!(
-                    "TreeSitter{}Tests",
-                    language_name.to_upper_camel_case()
-                )),
+                path.join(format!("TreeSitter{camel_name}Tests",)),
                 create_dir,
             )?
             .apply(|path| {
                 missing_path(
-                    path.join(format!(
-                        "TreeSitter{}Tests.swift",
-                        language_name.to_upper_camel_case()
-                    )),
+                    path.join(format!("TreeSitter{camel_name}Tests.swift")),
                     |path| generate_file(path, TESTS_SWIFT_TEMPLATE, language_name, &generate_opts),
                 )?;
 
@@ -919,15 +679,14 @@ pub fn get_root_path(path: &Path) -> Result<PathBuf> {
         let json = pathbuf
             .exists()
             .then(|| {
-                let file = File::open(pathbuf.as_path())
-                    .with_context(|| format!("Failed to open {filename}"))?;
-                let reader = BufReader::new(file);
+                let contents = fs::read_to_string(pathbuf.as_path())
+                    .with_context(|| format!("Failed to read {filename}"))?;
                 if is_package_json {
-                    serde_json::from_reader::<_, Map<String, Value>>(reader)
+                    serde_json::from_str::<Map<String, Value>>(&contents)
                         .context(format!("Failed to parse {filename}"))
                         .map(|v| v.contains_key("tree-sitter"))
                 } else {
-                    serde_json::from_reader::<_, TreeSitterJSON>(reader)
+                    serde_json::from_str::<TreeSitterJSON>(&contents)
                         .context(format!("Failed to parse {filename}"))
                         .map(|_| true)
                 }
@@ -961,7 +720,7 @@ fn generate_file(
     let mut replacement = template
         .replace(
             CAMEL_PARSER_NAME_PLACEHOLDER,
-            &language_name.to_upper_camel_case(),
+            generate_opts.camel_parser_name,
         )
         .replace(
             UPPER_PARSER_NAME_PLACEHOLDER,
@@ -1001,7 +760,12 @@ fn generate_file(
     }
 
     if let Some(email) = generate_opts.author_email {
-        replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email);
+        replacement = match filename {
+            "Cargo.toml" | "grammar.js" => {
+                replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, &format!("<{email}>"))
+            }
+            _ => replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email),
+        }
     } else {
         match filename {
             "package.json" => {
@@ -1099,7 +863,7 @@ fn generate_file(
                 PARSER_DESCRIPTION_PLACEHOLDER,
                 &format!(
                     "{} grammar for tree-sitter",
-                    language_name.to_upper_camel_case()
+                    generate_opts.camel_parser_name,
                 ),
             )
         }

cli/src/main.rs

@@ -653,11 +653,11 @@ impl Init {
 
             (opts.name.clone(), Some(opts))
         } else {
-            let json = serde_json::from_reader::<_, TreeSitterJSON>(
-                fs::File::open(current_dir.join("tree-sitter.json"))
-                    .with_context(|| "Failed to open tree-sitter.json")?,
+            let mut json = serde_json::from_str::<TreeSitterJSON>(
+                &fs::read_to_string(current_dir.join("tree-sitter.json"))
+                    .with_context(|| "Failed to read tree-sitter.json")?,
             )?;
-            (json.grammars[0].name.clone(), None)
+            (json.grammars.swap_remove(0).name, None)
         };
 
         generate_grammar_files(current_dir, &language_name, self.update, json_config_opts)?;

cli/src/templates/.editorconfig

@@ -41,3 +41,6 @@ indent_size = 8
 
 [parser.c]
 indent_size = 2
+
+[{alloc,array,parser}.h]
+indent_size = 2

cli/src/templates/PARSER_NAME.pc.in

@@ -6,6 +6,5 @@ Name: tree-sitter-PARSER_NAME
 Description: @PROJECT_DESCRIPTION@
 URL: @PROJECT_HOMEPAGE_URL@
 Version: @PROJECT_VERSION@
-Requires: @TS_REQUIRES@
 Libs: -L${libdir} -ltree-sitter-PARSER_NAME
 Cflags: -I${includedir}

cli/src/templates/_cargo.toml

@@ -12,7 +12,7 @@ edition = "2021"
 autoexamples = false
 
 build = "bindings/rust/build.rs"
-include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
+include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*", "tree-sitter.json"]
 
 [lib]
 path = "bindings/rust/lib.rs"

cli/src/templates/binding_test.js

@@ -1,9 +1,9 @@
-/// <reference types="node" />
-
 const assert = require("node:assert");
 const { test } = require("node:test");
 
+const Parser = require("tree-sitter");
+
 test("can load grammar", () => {
-  const parser = new (require("tree-sitter"))();
+  const parser = new Parser();
   assert.doesNotThrow(() => parser.setLanguage(require(".")));
 });

cli/src/templates/cmakelists.cmake

@@ -1,12 +1,13 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter-PARSER_NAME
-        VERSION "0.0.1"
-        DESCRIPTION "CAMEL_PARSER_NAME grammar for tree-sitter"
-        HOMEPAGE_URL "https://github.com/tree-sitter/tree-sitter-PARSER_NAME"
+        VERSION "PARSER_VERSION"
+        DESCRIPTION "PARSER_DESCRIPTION"
+        HOMEPAGE_URL "PARSER_URL"
         LANGUAGES C)
 
 option(BUILD_SHARED_LIBS "Build using shared libraries" ON)
+option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF)
 
 set(TREE_SITTER_ABI_VERSION ABI_VERSION_MAX CACHE STRING "Tree-sitter ABI version")
 if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$")
@@ -24,16 +25,21 @@ add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c"
                    COMMENT "Generating parser.c")
 
 add_library(tree-sitter-PARSER_NAME src/parser.c)
-if(EXISTS src/scanner.c)
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c)
   target_sources(tree-sitter-PARSER_NAME PRIVATE src/scanner.c)
 endif()
 target_include_directories(tree-sitter-PARSER_NAME PRIVATE src)
 
+target_compile_definitions(tree-sitter-PARSER_NAME PRIVATE
+                           $<$<BOOL:${TREE_SITTER_REUSE_ALLOCATOR}>:TREE_SITTER_REUSE_ALLOCATOR>
+                           $<$<CONFIG:Debug>:TREE_SITTER_DEBUG>)
+
 set_target_properties(tree-sitter-PARSER_NAME
                       PROPERTIES
                       C_STANDARD 11
                       POSITION_INDEPENDENT_CODE ON
-                      SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}")
+                      SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}"
+                      DEFINE_SYMBOL "")
 
 configure_file(bindings/c/tree-sitter-PARSER_NAME.pc.in
                "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc" @ONLY)
@@ -47,8 +53,6 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc"
 install(TARGETS tree-sitter-PARSER_NAME
         LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")
 
-add_custom_target(test "${TREE_SITTER_CLI}" test
+add_custom_target(ts-test "${TREE_SITTER_CLI}" test
                   WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
                   COMMENT "tree-sitter test")
-
-# vim:ft=cmake:

cli/src/templates/gitattributes

@@ -1,13 +1,37 @@
 * text=auto eol=lf
 
+# Generated source files
 src/*.json linguist-generated
 src/parser.c linguist-generated
 src/tree_sitter/* linguist-generated
 
-bindings/** linguist-generated
-binding.gyp linguist-generated
-setup.py linguist-generated
-Makefile linguist-generated
+# C bindings
+bindings/c/* linguist-generated
 CMakeLists.txt linguist-generated
-Package.swift linguist-generated
+Makefile linguist-generated
+
+# Rust bindings
+bindings/rust/* linguist-generated
+Cargo.toml linguist-generated
+Cargo.lock linguist-generated
+
+# Node.js bindings
+bindings/node/* linguist-generated
+binding.gyp linguist-generated
+package.json linguist-generated
+package-lock.json linguist-generated
+
+# Python bindings
+bindings/python/** linguist-generated
+setup.py linguist-generated
+pyproject.toml linguist-generated
+
+# Go bindings
+bindings/go/* linguist-generated
 go.mod linguist-generated
+go.sum linguist-generated
+
+# Swift bindings
+bindings/swift/** linguist-generated
+Package.swift linguist-generated
+Package.resolved linguist-generated

cli/src/templates/gitignore

@@ -5,11 +5,9 @@ target/
 build/
 prebuilds/
 node_modules/
-*.tgz
 
 # Swift artifacts
 .build/
-Package.resolved
 
 # Go artifacts
 _obj/
@@ -35,3 +33,8 @@ dist/
 *.wasm
 *.obj
 *.o
+
+# Archives
+*.tar.gz
+*.tgz
+*.zip

cli/src/templates/go.mod

@@ -2,4 +2,4 @@ module PARSER_URL_STRIPPED
 
 go 1.22
 
-require github.com/tree-sitter/go-tree-sitter v0.23.1
+require github.com/tree-sitter/go-tree-sitter v0.24.0

cli/src/templates/index.js

@@ -1,6 +1,10 @@
 const root = require("path").join(__dirname, "..", "..");
 
-module.exports = require("node-gyp-build")(root);
+module.exports =
+  typeof process.versions.bun === "string"
+    // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time
+    ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-PARSER_NAME.node`)
+    : require("node-gyp-build")(root);
 
 try {
   module.exports.nodeTypeInfo = require("../../src/node-types.json");

cli/src/templates/makefile

@@ -59,8 +59,8 @@ endif
 
 $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in
 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
-		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR)|' \
-		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR)|' \
+		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
+		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \
 		-e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \
 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@

cli/src/templates/package.json

@@ -2,7 +2,7 @@
   "name": "tree-sitter-PARSER_NAME",
   "version": "PARSER_VERSION",
   "description": "PARSER_DESCRIPTION",
-  "repository": "github:tree-sitter/tree-sitter-PARSER_NAME",
+  "repository": "PARSER_URL",
   "license": "PARSER_LICENSE",
   "author": {
     "name": "PARSER_AUTHOR_NAME",
@@ -19,6 +19,7 @@
   ],
   "files": [
     "grammar.js",
+    "tree-sitter.json",
     "binding.gyp",
     "prebuilds/**",
     "bindings/node/*",
@@ -27,8 +28,8 @@
     "*.wasm"
   ],
   "dependencies": {
-    "node-addon-api": "^8.0.0",
-    "node-gyp-build": "^4.8.1"
+    "node-addon-api": "^8.2.1",
+    "node-gyp-build": "^4.8.2"
   },
   "devDependencies": {
     "prebuildify": "^6.0.1",
@@ -47,11 +48,5 @@
     "prestart": "tree-sitter build --wasm",
     "start": "tree-sitter playground",
     "test": "node --test bindings/node/*_test.js"
-  },
-  "tree-sitter": [
-    {
-      "scope": "source.LOWER_PARSER_NAME",
-      "injection-regex": "^LOWER_PARSER_NAME$"
-    }
-  ]
+  }
 }

cli/src/templates/pyproject.toml

@@ -9,7 +9,6 @@ version = "PARSER_VERSION"
 keywords = ["incremental", "parsing", "tree-sitter", "PARSER_NAME"]
 classifiers = [
   "Intended Audience :: Developers",
-  "License :: OSI Approved :: MIT License",
   "Topic :: Software Development :: Compilers",
   "Topic :: Text Processing :: Linguistic",
   "Typing :: Typed",

cli/src/tests/detect_language.rs

@@ -32,7 +32,7 @@ fn detect_language_by_first_line_regex() {
         .find_language_configurations_at_path(strace_dir.path(), false)
         .unwrap();
 
-    // this is just to validate that we can read the package.json correctly
+    // this is just to validate that we can read the tree-sitter.json correctly
     assert_eq!(config[0].scope.as_ref().unwrap(), "source.strace");
 
     let file_name = strace_dir.path().join("strace.log");

cli/src/tests/node_test.rs

@@ -1026,6 +1026,31 @@ fn test_node_numeric_symbols_respect_simple_aliases() {
     assert_eq!(unary_minus_node.kind_id(), binary_minus_node.kind_id());
 }
 
+#[test]
+fn test_hidden_zero_width_node_with_visible_child() {
+    let code = r"
+class Foo {
+  std::
+private:
+  std::string s;
+};
+";
+
+    let mut parser = Parser::new();
+    parser.set_language(&get_language("cpp")).unwrap();
+    let tree = parser.parse(code, None).unwrap();
+    let root = tree.root_node();
+
+    let class_specifier = root.child(0).unwrap();
+    let field_decl_list = class_specifier.child_by_field_name("body").unwrap();
+    let field_decl = field_decl_list.named_child(0).unwrap();
+    let field_ident = field_decl.child_by_field_name("declarator").unwrap();
+    assert_eq!(
+        field_decl.child_with_descendant(field_ident).unwrap(),
+        field_ident
+    );
+}
+
 fn get_all_nodes(tree: &Tree) -> Vec<Node> {
     let mut result = Vec::new();
     let mut visited_children = false;

cli/src/tests/parser_test.rs

@@ -1507,6 +1507,20 @@ fn test_parsing_with_scanner_logging() {
     assert!(found);
 }
 
+#[test]
+fn test_parsing_get_column_at_eof() {
+    let dir = fixtures_dir().join("test_grammars").join("get_col_eof");
+    let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap();
+    let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap();
+
+    let mut parser = Parser::new();
+    parser
+        .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir)))
+        .unwrap();
+
+    parser.parse("a", None).unwrap();
+}
+
 const fn simple_range(start: usize, end: usize) -> Range {
     Range {
         start_byte: start,

lib/CMakeLists.txt

@@ -1,16 +1,21 @@
 cmake_minimum_required(VERSION 3.13)
 
 project(tree-sitter
-        VERSION "0.24.1"
+        VERSION "0.24.4"
         DESCRIPTION "An incremental parsing system for programming tools"
         HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
         LANGUAGES C)
 
 option(BUILD_SHARED_LIBS "Build using shared libraries" ON)
 option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF)
+option(AMALGAMATED "Build using an amalgamated source" OFF)
 
-file(GLOB TS_SOURCE_FILES src/*.c)
-list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c")
+if(AMALGAMATED)
+  set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c")
+else()
+  file(GLOB TS_SOURCE_FILES src/*.c)
+  list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c")
+endif()
 
 add_library(tree-sitter ${TS_SOURCE_FILES})
 
@@ -20,41 +25,45 @@ if(NOT MSVC)
   target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic)
 endif()
 
-if(NOT BUILD_SHARED_LIBS)
-  if(WIN32)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)
-  else()
-    set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
-  endif()
-endif()
-
 if(TREE_SITTER_FEATURE_WASM)
   if(NOT DEFINED CACHE{WASMTIME_INCLUDE_DIR})
     message(CHECK_START "Looking for wasmtime headers")
     find_path(WASMTIME_INCLUDE_DIR wasmtime.h
-              PATHS ENV DEP_WASMTIME_C_API_INCLUDE
-              REQUIRED)
+              PATHS ENV DEP_WASMTIME_C_API_INCLUDE)
+    if(NOT WASMTIME_INCLUDE_DIR)
+      unset(WASMTIME_INCLUDE_DIR CACHE)
+      message(FATAL_ERROR "Could not find wasmtime headers.\nDid you forget to set CMAKE_INCLUDE_PATH?")
+    endif()
     message(CHECK_PASS "found")
   endif()
 
   if(NOT DEFINED CACHE{WASMTIME_LIBRARY})
     message(CHECK_START "Looking for wasmtime library")
-    find_library(WASMTIME_LIBRARY wasmtime
-                 REQUIRED)
+    if(BUILD_SHARED_LIBS)
+      find_library(WASMTIME_LIBRARY wasmtime)
+    elseif(MSVC)
+      find_library(WASMTIME_LIBRARY wasmtime.lib)
+    else()
+      find_library(WASMTIME_LIBRARY libwasmtime.a)
+    endif()
+    if(NOT WASMTIME_LIBRARY)
+      unset(WASMTIME_LIBRARY CACHE)
+      message(FATAL_ERROR "Could not find wasmtime library.\nDid you forget to set CMAKE_LIBRARY_PATH?")
+    endif()
     message(CHECK_PASS "found")
   endif()
 
   target_compile_definitions(tree-sitter PUBLIC TREE_SITTER_FEATURE_WASM)
   target_include_directories(tree-sitter SYSTEM PRIVATE "${WASMTIME_INCLUDE_DIR}")
-  target_link_libraries(tree-sitter PRIVATE "${WASMTIME_LIBRARY}")
+  target_link_libraries(tree-sitter PUBLIC "${WASMTIME_LIBRARY}")
   set_property(TARGET tree-sitter PROPERTY C_STANDARD_REQUIRED ON)
 
   if(NOT BUILD_SHARED_LIBS)
     if(WIN32)
       target_compile_definitions(tree-sitter PRIVATE WASM_API_EXTERN= WASI_API_EXTERN=)
-      target_link_libraries(tree-sitter PRIVATE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt)
+      target_link_libraries(tree-sitter INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt)
     elseif(NOT APPLE)
-      target_link_libraries(tree-sitter PRIVATE pthread dl m)
+      target_link_libraries(tree-sitter INTERFACE pthread dl m)
     endif()
   endif()
 endif()
@@ -64,7 +73,8 @@ set_target_properties(tree-sitter
                       C_STANDARD 11
                       C_VISIBILITY_PRESET hidden
                       POSITION_INDEPENDENT_CODE ON
-                      SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}")
+                      SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
+                      DEFINE_SYMBOL "")
 
 configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
 

lib/Cargo.toml

@@ -37,7 +37,7 @@ tree-sitter-language = { version = "0.1", path = "language" }
 streaming-iterator = "0.1.9"
 
 [dependencies.wasmtime-c-api]
-version = "25.0.1"
+version = "25.0.2"
 optional = true
 package = "wasmtime-c-api-impl"
 default-features = false

lib/binding_rust/lib.rs

@@ -8,7 +8,6 @@ extern crate alloc;
 #[cfg(not(feature = "std"))]
 use alloc::{boxed::Box, format, string::String, string::ToString, vec::Vec};
 use core::{
-    char,
     ffi::{c_char, c_void, CStr},
     fmt::{self, Write},
     hash, iter,
@@ -489,9 +488,9 @@ impl Parser {
     /// Get the parser's current language.
     #[doc(alias = "ts_parser_language")]
     #[must_use]
-    pub fn language(&self) -> Option<Language> {
+    pub fn language(&self) -> Option<LanguageRef<'_>> {
         let ptr = unsafe { ffi::ts_parser_language(self.0.as_ptr()) };
-        (!ptr.is_null()).then(|| Language(ptr))
+        (!ptr.is_null()).then_some(LanguageRef(ptr, PhantomData))
     }
 
     /// Get the parser's current logger.
@@ -1854,9 +1853,28 @@ impl Query {
                 // Error types that report names
                 ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => {
                     let suffix = source.split_at(offset).1;
-                    let end_offset = suffix
-                        .find(|c| !char::is_alphanumeric(c) && c != '_' && c != '-')
-                        .unwrap_or(suffix.len());
+                    let in_quotes = source.as_bytes()[offset - 1] == b'"';
+                    let mut end_offset = suffix.len();
+                    if let Some(pos) = suffix
+                        .char_indices()
+                        .take_while(|(_, c)| *c != '\n')
+                        .find_map(|(i, c)| match c {
+                            '"' if in_quotes
+                                && i > 0
+                                && suffix.chars().nth(i - 1) != Some('\\') =>
+                            {
+                                Some(i)
+                            }
+                            c if !in_quotes
+                                && (c.is_whitespace() || c == '(' || c == ')' || c == ':') =>
+                            {
+                                Some(i)
+                            }
+                            _ => None,
+                        })
+                    {
+                        end_offset = pos;
+                    }
                     message = suffix.split_at(end_offset).0.to_string();
                     kind = match error_type {
                         ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType,

lib/binding_web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-tree-sitter",
-  "version": "0.24.1",
+  "version": "0.24.4",
   "description": "Tree-sitter bindings for the web",
   "main": "tree-sitter.js",
   "types": "tree-sitter-web.d.ts",

lib/src/lexer.c

@@ -252,12 +252,12 @@ static uint32_t ts_lexer__get_column(TSLexer *_self) {
   uint32_t goal_byte = self->current_position.bytes;
 
   self->did_get_column = true;
-  self->current_position.bytes -= self->current_position.extent.column;
-  self->current_position.extent.column = 0;
-
-  if (self->current_position.bytes < self->chunk_start) {
-    ts_lexer__get_chunk(self);
-  }
+  Length start_of_col = {
+    self->current_position.bytes - self->current_position.extent.column,
+    {self->current_position.extent.row, 0},
+  };
+  ts_lexer_goto(self, start_of_col);
+  ts_lexer__get_chunk(self);
 
   uint32_t result = 0;
   if (!ts_lexer__eof(_self)) {

lib/src/node.c

@@ -103,21 +103,6 @@ static inline bool ts_node_child_iterator_next(
   return true;
 }
 
-// This will return true if the next sibling is a zero-width token that is adjacent to the current node and is relevant
-static inline bool ts_node_child_iterator_next_sibling_is_empty_adjacent(NodeChildIterator *self, TSNode previous) {
-  if (!self->parent.ptr || ts_node_child_iterator_done(self)) return false;
-  if (self->child_index == 0) return false;
-  const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
-  TSSymbol alias = 0;
-  if (!ts_subtree_extra(*child)) {
-    if (self->alias_sequence) {
-      alias = self->alias_sequence[self->structural_child_index];
-    }
-  }
-  TSNode next = ts_node_new(self->tree, child, self->position, alias);
-  return ts_node_end_byte(previous) == ts_node_end_byte(next) && ts_node__is_relevant(next, true);
-}
-
 // TSNode - private
 
 static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
@@ -549,9 +534,9 @@ TSNode ts_node_parent(TSNode self) {
   if (node.id == self.id) return ts_node__null();
 
   while (true) {
-   TSNode next_node = ts_node_child_containing_descendant(node, self);
-   if (ts_node_is_null(next_node)) break;
-   node = next_node;
+    TSNode next_node = ts_node_child_containing_descendant(node, self);
+    if (ts_node_is_null(next_node)) break;
+    node = next_node;
   }
 
   return node;
@@ -560,6 +545,7 @@ TSNode ts_node_parent(TSNode self) {
 TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) {
   uint32_t start_byte = ts_node_start_byte(descendant);
   uint32_t end_byte = ts_node_end_byte(descendant);
+  bool is_empty = start_byte == end_byte;
 
   do {
     NodeChildIterator iter = ts_node_iterate_children(&self);
@@ -572,24 +558,16 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) {
         return ts_node__null();
       }
 
-      // Here we check the current self node and *all* of its zero-width token siblings that follow.
-      // If any of these nodes contain the target subnode, we return that node. Otherwise, we restore the node we started at
-      // for the loop condition, and that will continue with the next *non-zero-width* sibling.
-      TSNode old = self;
-      // While the next sibling is a zero-width token
-      while (ts_node_child_iterator_next_sibling_is_empty_adjacent(&iter, self)) {
-        TSNode current_node = ts_node_child_containing_descendant(self, descendant);
-        // If the target child is in self, return it
-        if (!ts_node_is_null(current_node)) {
-          return current_node;
-        }
-        ts_node_child_iterator_next(&iter, &self);
-        if (self.id == descendant.id) {
-          return ts_node__null();
+      // If the descendant is empty, and the end byte is within `self`,
+      // we check whether `self` contains it or not.
+      if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) {
+        TSNode child = ts_node_child_with_descendant(self, descendant);
+        // If the child is not null, return self if it's relevant, else return the child
+        if (!ts_node_is_null(child)) {
+          return ts_node__is_relevant(self, true) ? self : child;
         }
       }
-      self = old;
-    } while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0);
+    } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0);
   } while (!ts_node__is_relevant(self, true));
 
   return self;
@@ -598,6 +576,7 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) {
 TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) {
   uint32_t start_byte = ts_node_start_byte(descendant);
   uint32_t end_byte = ts_node_end_byte(descendant);
+  bool is_empty = start_byte == end_byte;
 
   do {
     NodeChildIterator iter = ts_node_iterate_children(&self);
@@ -612,24 +591,16 @@ TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) {
         return self;
       }
 
-      // Here we check the current self node and *all* of its zero-width token siblings that follow.
-      // If any of these nodes contain the target subnode, we return that node. Otherwise, we restore the node we started at
-      // for the loop condition, and that will continue with the next *non-zero-width* sibling.
-      TSNode old = self;
-      // While the next sibling is a zero-width token
-      while (ts_node_child_iterator_next_sibling_is_empty_adjacent(&iter, self)) {
-        TSNode current_node = ts_node_child_with_descendant(self, descendant);
-        // If the target child is in self, return it
-        if (!ts_node_is_null(current_node)) {
-          return current_node;
-        }
-        ts_node_child_iterator_next(&iter, &self);
-        if (self.id == descendant.id) {
-          return self;
+      // If the descendant is empty, and the end byte is within `self`,
+      // we check whether `self` contains it or not.
+      if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) {
+        TSNode child = ts_node_child_with_descendant(self, descendant);
+        // If the child is not null, return self if it's relevant, else return the child
+        if (!ts_node_is_null(child)) {
+          return ts_node__is_relevant(self, true) ? self : child;
         }
       }
-      self = old;
-    } while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0);
+    } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0);
   } while (!ts_node__is_relevant(self, true));
 
   return self;

test/fixtures/test_grammars/get_col_eof/grammar.js

@@ -0,0 +1,11 @@
+module.exports = grammar({
+  name: "get_col_eof",
+
+  externals: $ => [
+    $.char
+  ],
+
+  rules: {
+    source_file: $ => repeat($.char),
+  }
+});

test/fixtures/test_grammars/get_col_eof/scanner.c

@@ -0,0 +1,34 @@
+#include "tree_sitter/parser.h"
+
+enum TokenType { CHAR };
+
+void *tree_sitter_get_col_eof_external_scanner_create(void) { return NULL; }
+
+void tree_sitter_get_col_eof_external_scanner_destroy(void *scanner) {}
+
+unsigned tree_sitter_get_col_eof_external_scanner_serialize(void *scanner,
+                                                            char *buffer) {
+  return 0;
+}
+
+void tree_sitter_get_col_eof_external_scanner_deserialize(void *scanner,
+                                                          const char *buffer,
+                                                          unsigned length) {}
+
+bool tree_sitter_get_col_eof_external_scanner_scan(void *scanner,
+                                                   TSLexer *lexer,
+                                                   const bool *valid_symbols) {
+  if (lexer->eof(lexer)) {
+    return false;
+  }
+
+  if (valid_symbols[CHAR]) {
+    lexer->advance(lexer, false);
+    lexer->get_column(lexer);
+    lexer->result_symbol = CHAR;
+    lexer->mark_end(lexer);
+    return true;
+  }
+
+  return false;
+}