feat(lib): add ranged S-expression display

The core C lib has access to hidden MISSING nodes that bindings cannot
see via a tree walker. We also have several re-implementations of this
logic within the cli crate. Centralize on a single, improved implementation.

crates/cli/src/parse.rs

@@ -14,8 +14,8 @@ use log::info;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use tree_sitter::{
-    InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree, TreeCursor,
-    ffi,
+    InputEdit, Language, LogType, Node, ParseOptions, ParseState, Parser, Point, Range, Tree,
+    TreeCursor, ffi, format_sexp,
 };
 
 use crate::{fuzz::edits::Edit, paint::paint, util};
@@ -278,6 +278,35 @@ pub struct ParseResult {
     pub duration: Option<Duration>,
 }
 
+#[must_use]
+pub fn first_error_or_container(root: Node<'_>) -> Option<Node<'_>> {
+    if !root.has_error() {
+        return None;
+    }
+
+    let mut cursor = root.walk();
+    'descend: loop {
+        let node = cursor.node();
+        if node.is_error() || node.is_missing() {
+            return Some(node);
+        }
+
+        if cursor.goto_first_child() {
+            loop {
+                if cursor.node().has_error() {
+                    continue 'descend;
+                }
+                if !cursor.goto_next_sibling() {
+                    cursor.goto_parent();
+                    break;
+                }
+            }
+        }
+
+        return Some(node);
+    }
+}
+
 pub fn parse_file_at_path(
     parser: &mut Parser,
     language: &Language,
@@ -457,58 +486,13 @@ pub fn parse_file_at_path(
         let mut cursor = tree.walk();
 
         if opts.output == ParseOutput::Normal {
-            let mut needs_newline = false;
-            let mut indent_level = 0;
-            let mut did_visit_children = false;
-            loop {
-                let node = cursor.node();
-                let is_named = node.is_named();
-                if did_visit_children {
-                    if is_named {
-                        stdout.write_all(b")")?;
-                        needs_newline = true;
-                    }
-                    if cursor.goto_next_sibling() {
-                        did_visit_children = false;
-                    } else if cursor.goto_parent() {
-                        did_visit_children = true;
-                        indent_level -= 1;
-                    } else {
-                        break;
-                    }
-                } else {
-                    if is_named {
-                        if needs_newline {
-                            stdout.write_all(b"\n")?;
-                        }
-                        for _ in 0..indent_level {
-                            stdout.write_all(b"  ")?;
-                        }
-                        let start = node.start_position();
-                        let end = node.end_position();
-                        if let Some(field_name) = cursor.field_name() {
-                            write!(&mut stdout, "{field_name}: ")?;
-                        }
-                        write!(&mut stdout, "({}", node.kind())?;
-                        if !opts.no_ranges {
-                            write!(
-                                &mut stdout,
-                                " [{}, {}] - [{}, {}]",
-                                start.row, start.column, end.row, end.column
-                            )?;
-                        }
-                        needs_newline = true;
-                    }
-                    if cursor.goto_first_child() {
-                        did_visit_children = false;
-                        indent_level += 1;
-                    } else {
-                        did_visit_children = true;
-                    }
-                }
-            }
-            cursor.reset(tree.root_node());
-            writeln!(&mut stdout)?;
+            let root = tree.root_node();
+            let sexp = if opts.no_ranges {
+                root.to_sexp()
+            } else {
+                root.to_sexp_with_ranges()
+            };
+            writeln!(stdout, "{}", format_sexp(&sexp, 0))?;
         }
 
         if opts.output == ParseOutput::Cst {
@@ -621,47 +605,7 @@ pub fn parse_file_at_path(
             util::print_tree_graph(&tree, "log.html", opts.open_log).unwrap();
         }
 
-        let mut first_error = None;
-        let mut earliest_node_with_error = None;
-        'outer: loop {
-            let node = cursor.node();
-            if node.has_error() {
-                if earliest_node_with_error.is_none() {
-                    earliest_node_with_error = Some(node);
-                }
-                if node.is_error() || node.is_missing() {
-                    first_error = Some(node);
-                    break;
-                }
-
-                // If there's no more children, even though some outer node has an error,
-                // then that means that the first error is hidden, but the later error could be
-                // visible. So, we walk back up to the child of the first node with an error,
-                // and then check its siblings for errors.
-                if !cursor.goto_first_child() {
-                    let earliest = earliest_node_with_error.unwrap();
-                    while cursor.goto_parent() {
-                        if cursor.node().parent().is_some_and(|p| p == earliest) {
-                            while cursor.goto_next_sibling() {
-                                let sibling = cursor.node();
-                                if sibling.is_error() || sibling.is_missing() {
-                                    first_error = Some(sibling);
-                                    break 'outer;
-                                }
-                                if sibling.has_error() && cursor.goto_first_child() {
-                                    continue 'outer;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                    break;
-                }
-            } else if !cursor.goto_next_sibling() {
-                break;
-            }
-        }
-
+        let first_error = first_error_or_container(tree.root_node());
         if first_error.is_some() || opts.print_time {
             let path = path.to_string_lossy();
             write!(
@@ -672,33 +616,7 @@ pub fn parse_file_at_path(
                 width = max_path_length
             )?;
             if let Some(node) = first_error {
-                let node_kind = node.kind();
-                let mut node_text = String::with_capacity(node_kind.len());
-                for c in node_kind.chars() {
-                    if let Some(escaped) = escape_invisible(c) {
-                        node_text += escaped;
-                    } else {
-                        node_text.push(c);
-                    }
-                }
-                write!(&mut stdout, "\t(")?;
-                if node.is_missing() {
-                    if node.is_named() {
-                        write!(&mut stdout, "MISSING {node_text}")?;
-                    } else {
-                        write!(&mut stdout, "MISSING \"{node_text}\"")?;
-                    }
-                } else {
-                    write!(&mut stdout, "{node_text}")?;
-                }
-
-                let start = node.start_position();
-                let end = node.end_position();
-                write!(
-                    &mut stdout,
-                    " [{}, {}] - [{}, {}])",
-                    start.row, start.column, end.row, end.column
-                )?;
+                write!(stdout, "\t{}", node.to_sexp_with_ranges())?;
             }
             if !opts.edits.is_empty() {
                 write!(

crates/cli/src/test.rs

@@ -1914,6 +1914,17 @@ b
     (g
       (h
         (MISSING i)))))
+"
+            .trim()
+        );
+        assert_eq!(
+            format_sexp(
+                "(source_file [0, 0] - [1, 0] (MISSING _hidden [0, 1] - [0, 1]))",
+                0,
+            ),
+            r"
+(source_file [0, 0] - [1, 0]
+  (MISSING _hidden [0, 1] - [0, 1]))
 "
             .trim()
         );

crates/cli/src/tests/node_test.rs

@@ -946,6 +946,11 @@ fn test_node_sexp() {
     assert_eq!(paren_node.to_sexp(), "(\"(\")");
     assert_eq!(identifier_node.kind(), "identifier");
     assert_eq!(identifier_node.to_sexp(), "(identifier)");
+    assert_eq!(
+        identifier_node.to_sexp_with_ranges(),
+        "(identifier [0, 4] - [0, 5])"
+    );
+    assert_eq!(if_node.to_sexp_with_ranges(), "(\"if\" [0, 0] - [0, 2])");
 }
 
 #[test]

crates/cli/src/tests/parser_test.rs

@@ -21,7 +21,7 @@ use super::helpers::{
 };
 use crate::{
     fuzz::edits::Edit,
-    parse::perform_edit,
+    parse::{first_error_or_container, perform_edit},
     tests::{
         generate_parser,
         helpers::fixtures::{fixtures_dir, get_test_fixture_language},
@@ -404,6 +404,10 @@ fn test_parsing_invalid_chars_at_eof() {
         tree.root_node().to_sexp(),
         "(document (ERROR (UNEXPECTED INVALID)))"
     );
+    assert_eq!(
+        tree.root_node().to_sexp_with_ranges(),
+        "(document [0, 0] - [0, 1] (ERROR [0, 0] - [0, 1] (UNEXPECTED INVALID [0, 0] - [0, 1])))"
+    );
 }
 
 #[test]
@@ -2243,3 +2247,82 @@ fn test_grammar_that_should_hang_and_not_segfault() {
         }
     }
 }
+
+#[test]
+fn test_hidden_missing_node_ranges() {
+    let (parser_name, parser_code) = generate_parser(
+        r#"{
+              "name": "test_hidden_missing_node_ranges",
+              "rules": {
+                  "source_file": {
+                      "type": "SEQ",
+                      "members": [
+                          {"type": "STRING", "value": "."},
+                          {"type": "SYMBOL", "name": "id"}
+                      ]
+                  },
+                  "id": {"type": "SYMBOL", "name": "_hidden"},
+                  "_hidden": {"type": "PATTERN", "value": "[A-Za-z0-9_]+"}
+              },
+              "extras": [{"type": "PATTERN", "value": "\\s"}]
+          }"#,
+    )
+    .unwrap();
+
+    let mut parser = Parser::new();
+    parser
+        .set_language(&get_test_language(&parser_name, &parser_code, None))
+        .unwrap();
+
+    let tree = parser.parse(".\n", None).unwrap();
+    let root = tree.root_node();
+    assert!(root.has_error());
+    assert_eq!(
+        root.to_sexp_with_ranges(),
+        "(source_file [0, 0] - [1, 0] (id [0, 1] - [0, 1] (MISSING _hidden [0, 1] - [0, 1])))"
+    );
+
+    let container = first_error_or_container(root).unwrap();
+    assert_eq!(container.kind(), "id");
+    assert_eq!(
+        container.to_sexp_with_ranges(),
+        "(id [0, 1] - [0, 1] (MISSING _hidden [0, 1] - [0, 1]))"
+    );
+
+    let tree = parser.parse(".abc\n", None).unwrap();
+    assert_eq!(first_error_or_container(tree.root_node()), None);
+}
+
+#[test]
+fn test_direct_hidden_missing_node_ranges() {
+    let (parser_name, parser_code) = generate_parser(
+        r#"{
+              "name": "test_direct_hidden_missing_node_ranges",
+              "rules": {
+                  "source_file": {
+                      "type": "SEQ",
+                      "members": [
+                          {"type": "STRING", "value": "."},
+                          {"type": "SYMBOL", "name": "_hidden"}
+                      ]
+                  },
+                  "_hidden": {"type": "PATTERN", "value": "[A-Za-z0-9_]+"}
+              },
+              "extras": [{"type": "PATTERN", "value": "\\s"}]
+          }"#,
+    )
+    .unwrap();
+
+    let mut parser = Parser::new();
+    parser
+        .set_language(&get_test_language(&parser_name, &parser_code, None))
+        .unwrap();
+
+    let tree = parser.parse(".\n", None).unwrap();
+    let root = tree.root_node();
+    assert_eq!(first_error_or_container(root), Some(root));
+    assert_eq!(
+        root.to_sexp_with_ranges(),
+        "(source_file [0, 0] - [1, 0] (MISSING _hidden [0, 1] - [0, 1]))"
+    );
+}

crates/xtask/src/check_wasm_exports.rs

@@ -16,10 +16,12 @@ use notify_debouncer_full::new_debouncer;
 
 use crate::{CheckWasmExports, bail_on_err, watch_wasm};
 
-const EXCLUDES: [&str; 25] = [
+const EXCLUDES: [&str; 26] = [
     // Unneeded because the JS side has its own way of implementing it
     "ts_node_child_by_field_name",
     "ts_node_edit",
+    // Serialized point columns use bytes, unlike the Web binding's UTF-16 columns
+    "ts_node_string_with_ranges",
     // Precomputed and stored in the JS side
     "ts_node_type",
     "ts_node_grammar_type",

lib/binding_rust/bindings.rs

@@ -354,6 +354,10 @@ unsafe extern "C" {
     #[doc = " Get an S-expression representing the node as a string.\n\n This string is allocated with `malloc` and the caller is responsible for\n freeing it using `free`."]
     pub fn ts_node_string(self_: TSNode) -> *mut ::core::ffi::c_char;
 }
+unsafe extern "C" {
+    #[doc = " Get an S-expression representing the node as a string with start and end\n positions included.\n\n This string is allocated with `malloc` and the caller is responsible for\n freeing it using `free`."]
+    pub fn ts_node_string_with_ranges(self_: TSNode) -> *mut ::core::ffi::c_char;
+}
 unsafe extern "C" {
     #[doc = " Check if the node is null. Functions like [`ts_node_child`] and\n [`ts_node_next_sibling`] will return a null node to indicate that no such node\n was found."]
     pub fn ts_node_is_null(self_: TSNode) -> bool;

lib/binding_rust/lib.rs

@@ -2109,6 +2109,19 @@ impl<'tree> Node<'tree> {
         result
     }
 
+    /// Get an S-expression representing the node with start and end positions included.
+    #[doc(alias = "ts_node_string_with_ranges")]
+    #[must_use]
+    pub fn to_sexp_with_ranges(&self) -> String {
+        let c_string = unsafe { ffi::ts_node_string_with_ranges(self.0) };
+        let result = unsafe { CStr::from_ptr(c_string) }
+            .to_str()
+            .unwrap()
+            .to_string();
+        unsafe { ts_free(c_string.cast::<c_void>()) };
+        result
+    }
+
     pub fn utf8_text<'a>(&self, source: &'a [u8]) -> Result<&'a str, str::Utf8Error> {
         str::from_utf8(&source[self.start_byte()..self.end_byte()])
     }
@@ -4022,6 +4035,9 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
             write!(formatted, "{scratch} ").unwrap();
             has_field = true;
             indent_level += 1;
+        } else if scratch.starts_with('[') || scratch.ends_with(']') || scratch == "-" {
+            // "[row," "column]" "-" "[row," "column]"
+            write!(formatted, " {scratch}").unwrap();
         }
     }
 

lib/include/tree_sitter/api.h

@@ -560,6 +560,15 @@ TSPoint ts_node_end_point(TSNode self);
  */
 char *ts_node_string(TSNode self);
 
+/**
+ * Get an S-expression representing the node as a string with start and end
+ * positions included.
+ *
+ * This string is allocated with `malloc` and the caller is responsible for
+ * freeing it using `free`.
+ */
+char *ts_node_string_with_ranges(TSNode self);
+
 /**
  * Check if the node is null. Functions like [`ts_node_child`] and
  * [`ts_node_next_sibling`] will return a null node to indicate that no such node

lib/src/node.c

@@ -486,7 +486,22 @@ char *ts_node_string(TSNode self) {
     alias_symbol,
     ts_language_symbol_metadata(self.tree->language, alias_symbol).visible,
     self.tree->language,
-    false
+    false,
+    false,
+    (Length) {ts_node_start_byte(self), ts_node_start_point(self)}
+  );
+}
+
+char *ts_node_string_with_ranges(TSNode self) {
+  TSSymbol alias_symbol = ts_node__alias(&self);
+  return ts_subtree_string(
+    ts_node__subtree(self),
+    alias_symbol,
+    ts_language_symbol_metadata(self.tree->language, alias_symbol).visible,
+    self.tree->language,
+    false,
+    true,
+    (Length) {ts_node_start_byte(self), ts_node_start_point(self)}
   );
 }
 

lib/src/subtree.c

@@ -832,6 +832,8 @@ static const char *const ROOT_FIELD = "__ROOT__";
 
 typedef struct {
   Subtree subtree;
+  Length position;
+  Length child_position;
   TSSymbol alias_symbol;
   bool alias_is_named;
   const char *field_name;
@@ -846,9 +848,28 @@ typedef struct {
   const TSFieldMapEntry *field_map_end;
 } WriteToStringFrame;
 
+static size_t ts_subtree__write_range_to_string(
+  char *string,
+  size_t limit,
+  Length start,
+  Length size
+) {
+  Length end = length_add(start, size);
+  return snprintf(
+    string,
+    limit,
+    " [%u, %u] - [%u, %u]",
+    start.extent.row,
+    start.extent.column,
+    end.extent.row,
+    end.extent.column
+  );
+}
+
 static size_t ts_subtree__write_to_string(
   Subtree self, char *string, size_t limit,
   const TSLanguage *language, bool include_all,
+  bool include_ranges, Length root_position,
   TSSymbol root_alias_symbol, bool root_alias_is_named, const char *root_field_name
 ) {
   char *cursor = string;
@@ -857,6 +878,8 @@ static size_t ts_subtree__write_to_string(
   Array(WriteToStringFrame) stack = array_new();
   array_push(&stack, ((WriteToStringFrame) {
     .subtree = self,
+    .position = root_position,
+    .child_position = root_position,
     .alias_symbol = root_alias_symbol,
     .alias_is_named = root_alias_is_named,
     .field_name = root_field_name,
@@ -914,15 +937,35 @@ static size_t ts_subtree__write_to_string(
             cursor += snprintf(*writer, limit, "(%s", symbol_name);
           }
         }
+
+        if (include_ranges) {
+          cursor += ts_subtree__write_range_to_string(
+            *writer,
+            limit,
+            frame->position,
+            ts_subtree_size(node)
+          );
+        }
       } else if (frame->is_root) {
         TSSymbol symbol = frame->alias_symbol ? frame->alias_symbol : ts_subtree_symbol(node);
         const char *symbol_name = ts_language_symbol_name(language, symbol);
         if (ts_subtree_child_count(node) > 0) {
           cursor += snprintf(*writer, limit, "(%s", symbol_name);
         } else if (ts_subtree_named(node)) {
-          cursor += snprintf(*writer, limit, "(%s)", symbol_name);
+          cursor += snprintf(*writer, limit, "(%s", symbol_name);
         } else {
-          cursor += snprintf(*writer, limit, "(\"%s\")", symbol_name);
+          cursor += snprintf(*writer, limit, "(\"%s\"", symbol_name);
+        }
+        if (include_ranges) {
+          cursor += ts_subtree__write_range_to_string(
+            *writer,
+            limit,
+            frame->position,
+            ts_subtree_size(node)
+          );
+        }
+        if (ts_subtree_child_count(node) == 0) {
+          cursor += snprintf(*writer, limit, ")");
         }
       }
 
@@ -942,8 +985,13 @@ static size_t ts_subtree__write_to_string(
 
     if (frame->child_index < ts_subtree_child_count(node)) {
       Subtree child = ts_subtree_children(node)[frame->child_index];
+      if (frame->child_index > 0) {
+        frame->child_position = length_add(frame->child_position, ts_subtree_padding(child));
+      }
       WriteToStringFrame child_frame = {
         .subtree = child,
+        .position = frame->child_position,
+        .child_position = frame->child_position,
         .is_root = false,
       };
 
@@ -972,6 +1020,7 @@ static size_t ts_subtree__write_to_string(
       }
 
       frame->child_index++;
+      frame->child_position = length_add(frame->child_position, ts_subtree_size(child));
       // After this push, `frame` may be invalidated by a realloc.
       array_push(&stack, child_frame);
       continue;
@@ -990,18 +1039,20 @@ char *ts_subtree_string(
   TSSymbol alias_symbol,
   bool alias_is_named,
   const TSLanguage *language,
-  bool include_all
+  bool include_all,
+  bool include_ranges,
+  Length position
 ) {
   char scratch_string[1];
   size_t size = ts_subtree__write_to_string(
     self, scratch_string, 1,
-    language, include_all,
+    language, include_all, include_ranges, position,
     alias_symbol, alias_is_named, ROOT_FIELD
   ) + 1;
   char *result = ts_malloc(size * sizeof(char));
   ts_subtree__write_to_string(
     self, result, size,
-    language, include_all,
+    language, include_all, include_ranges, position,
     alias_symbol, alias_is_named, ROOT_FIELD
   );
   return result;

lib/src/subtree.h

@@ -224,7 +224,15 @@ void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLangua
 void ts_subtree_compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack);
 void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *language);
 Subtree ts_subtree_edit(Subtree self, const TSInputEdit *edit, SubtreePool *pool);
-char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, bool include_all);
+char *ts_subtree_string(
+  Subtree self,
+  TSSymbol alias_symbol,
+  bool alias_is_named,
+  const TSLanguage *language,
+  bool include_all,
+  bool include_ranges,
+  Length position
+);
 void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f);
 Subtree ts_subtree_last_external_token(Subtree tree);
 const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);