diff --git a/crates/cli/src/parse.rs b/crates/cli/src/parse.rs index 78aff0608..e7e95b39d 100644 --- a/crates/cli/src/parse.rs +++ b/crates/cli/src/parse.rs @@ -14,8 +14,8 @@ use log::info; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use tree_sitter::{ - InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree, TreeCursor, - ffi, + InputEdit, Language, LogType, Node, ParseOptions, ParseState, Parser, Point, Range, Tree, + TreeCursor, ffi, format_sexp, }; use crate::{fuzz::edits::Edit, paint::paint, util}; @@ -278,6 +278,35 @@ pub struct ParseResult { pub duration: Option, } +#[must_use] +pub fn first_error_or_container(root: Node<'_>) -> Option> { + if !root.has_error() { + return None; + } + + let mut cursor = root.walk(); + 'descend: loop { + let node = cursor.node(); + if node.is_error() || node.is_missing() { + return Some(node); + } + + if cursor.goto_first_child() { + loop { + if cursor.node().has_error() { + continue 'descend; + } + if !cursor.goto_next_sibling() { + cursor.goto_parent(); + break; + } + } + } + + return Some(node); + } +} + pub fn parse_file_at_path( parser: &mut Parser, language: &Language, @@ -457,58 +486,13 @@ pub fn parse_file_at_path( let mut cursor = tree.walk(); if opts.output == ParseOutput::Normal { - let mut needs_newline = false; - let mut indent_level = 0; - let mut did_visit_children = false; - loop { - let node = cursor.node(); - let is_named = node.is_named(); - if did_visit_children { - if is_named { - stdout.write_all(b")")?; - needs_newline = true; - } - if cursor.goto_next_sibling() { - did_visit_children = false; - } else if cursor.goto_parent() { - did_visit_children = true; - indent_level -= 1; - } else { - break; - } - } else { - if is_named { - if needs_newline { - stdout.write_all(b"\n")?; - } - for _ in 0..indent_level { - stdout.write_all(b" ")?; - } - let start = node.start_position(); - let end = node.end_position(); - if let Some(field_name) = cursor.field_name() { - write!(&mut stdout, "{field_name}: ")?; - } - write!(&mut stdout, "({}", node.kind())?; - if !opts.no_ranges { - write!( - &mut stdout, - " [{}, {}] - [{}, {}]", - start.row, start.column, end.row, end.column - )?; - } - needs_newline = true; - } - if cursor.goto_first_child() { - did_visit_children = false; - indent_level += 1; - } else { - did_visit_children = true; - } - } - } - cursor.reset(tree.root_node()); - writeln!(&mut stdout)?; + let root = tree.root_node(); + let sexp = if opts.no_ranges { + root.to_sexp() + } else { + root.to_sexp_with_ranges() + }; + writeln!(stdout, "{}", format_sexp(&sexp, 0))?; } if opts.output == ParseOutput::Cst { @@ -621,47 +605,7 @@ pub fn parse_file_at_path( util::print_tree_graph(&tree, "log.html", opts.open_log).unwrap(); } - let mut first_error = None; - let mut earliest_node_with_error = None; - 'outer: loop { - let node = cursor.node(); - if node.has_error() { - if earliest_node_with_error.is_none() { - earliest_node_with_error = Some(node); - } - if node.is_error() || node.is_missing() { - first_error = Some(node); - break; - } - - // If there's no more children, even though some outer node has an error, - // then that means that the first error is hidden, but the later error could be - // visible. So, we walk back up to the child of the first node with an error, - // and then check its siblings for errors. - if !cursor.goto_first_child() { - let earliest = earliest_node_with_error.unwrap(); - while cursor.goto_parent() { - if cursor.node().parent().is_some_and(|p| p == earliest) { - while cursor.goto_next_sibling() { - let sibling = cursor.node(); - if sibling.is_error() || sibling.is_missing() { - first_error = Some(sibling); - break 'outer; - } - if sibling.has_error() && cursor.goto_first_child() { - continue 'outer; - } - } - break; - } - } - break; - } - } else if !cursor.goto_next_sibling() { - break; - } - } - + let first_error = first_error_or_container(tree.root_node()); if first_error.is_some() || opts.print_time { let path = path.to_string_lossy(); write!( @@ -672,33 +616,7 @@ pub fn parse_file_at_path( width = max_path_length )?; if let Some(node) = first_error { - let node_kind = node.kind(); - let mut node_text = String::with_capacity(node_kind.len()); - for c in node_kind.chars() { - if let Some(escaped) = escape_invisible(c) { - node_text += escaped; - } else { - node_text.push(c); - } - } - write!(&mut stdout, "\t(")?; - if node.is_missing() { - if node.is_named() { - write!(&mut stdout, "MISSING {node_text}")?; - } else { - write!(&mut stdout, "MISSING \"{node_text}\"")?; - } - } else { - write!(&mut stdout, "{node_text}")?; - } - - let start = node.start_position(); - let end = node.end_position(); - write!( - &mut stdout, - " [{}, {}] - [{}, {}])", - start.row, start.column, end.row, end.column - )?; + write!(stdout, "\t{}", node.to_sexp_with_ranges())?; } if !opts.edits.is_empty() { write!( diff --git a/crates/cli/src/test.rs b/crates/cli/src/test.rs index fb1d5f3de..c73f1b585 100644 --- a/crates/cli/src/test.rs +++ b/crates/cli/src/test.rs @@ -1914,6 +1914,17 @@ b (g (h (MISSING i))))) +" + .trim() + ); + assert_eq!( + format_sexp( + "(source_file [0, 0] - [1, 0] (MISSING _hidden [0, 1] - [0, 1]))", + 0, + ), + r" +(source_file [0, 0] - [1, 0] + (MISSING _hidden [0, 1] - [0, 1])) " .trim() ); diff --git a/crates/cli/src/tests/node_test.rs b/crates/cli/src/tests/node_test.rs index 88a09c733..29fdbbb2e 100644 --- a/crates/cli/src/tests/node_test.rs +++ b/crates/cli/src/tests/node_test.rs @@ -946,6 +946,11 @@ fn test_node_sexp() { assert_eq!(paren_node.to_sexp(), "(\"(\")"); assert_eq!(identifier_node.kind(), "identifier"); assert_eq!(identifier_node.to_sexp(), "(identifier)"); + assert_eq!( + identifier_node.to_sexp_with_ranges(), + "(identifier [0, 4] - [0, 5])" + ); + assert_eq!(if_node.to_sexp_with_ranges(), "(\"if\" [0, 0] - [0, 2])"); } #[test] diff --git a/crates/cli/src/tests/parser_test.rs b/crates/cli/src/tests/parser_test.rs index 1278d4f47..ce39dee54 100644 --- a/crates/cli/src/tests/parser_test.rs +++ b/crates/cli/src/tests/parser_test.rs @@ -21,7 +21,7 @@ use super::helpers::{ }; use crate::{ fuzz::edits::Edit, - parse::perform_edit, + parse::{first_error_or_container, perform_edit}, tests::{ generate_parser, helpers::fixtures::{fixtures_dir, get_test_fixture_language}, @@ -404,6 +404,10 @@ fn test_parsing_invalid_chars_at_eof() { tree.root_node().to_sexp(), "(document (ERROR (UNEXPECTED INVALID)))" ); + assert_eq!( + tree.root_node().to_sexp_with_ranges(), + "(document [0, 0] - [0, 1] (ERROR [0, 0] - [0, 1] (UNEXPECTED INVALID [0, 0] - [0, 1])))" + ); } #[test] @@ -2243,3 +2247,82 @@ fn test_grammar_that_should_hang_and_not_segfault() { } } } + +#[test] +fn test_hidden_missing_node_ranges() { + let (parser_name, parser_code) = generate_parser( + r#"{ + "name": "test_hidden_missing_node_ranges", + "rules": { + "source_file": { + "type": "SEQ", + "members": [ + {"type": "STRING", "value": "."}, + {"type": "SYMBOL", "name": "id"} + ] + }, + "id": {"type": "SYMBOL", "name": "_hidden"}, + "_hidden": {"type": "PATTERN", "value": "[A-Za-z0-9_]+"} + }, + "extras": [{"type": "PATTERN", "value": "\\s"}] + }"#, + ) + .unwrap(); + + let mut parser = Parser::new(); + parser + .set_language(&get_test_language(&parser_name, &parser_code, None)) + .unwrap(); + + let tree = parser.parse(".\n", None).unwrap(); + let root = tree.root_node(); + assert!(root.has_error()); + assert_eq!( + root.to_sexp_with_ranges(), + "(source_file [0, 0] - [1, 0] (id [0, 1] - [0, 1] (MISSING _hidden [0, 1] - [0, 1])))" + ); + + let container = first_error_or_container(root).unwrap(); + assert_eq!(container.kind(), "id"); + assert_eq!( + container.to_sexp_with_ranges(), + "(id [0, 1] - [0, 1] (MISSING _hidden [0, 1] - [0, 1]))" + ); + + let tree = parser.parse(".abc\n", None).unwrap(); + assert_eq!(first_error_or_container(tree.root_node()), None); +} + +#[test] +fn test_direct_hidden_missing_node_ranges() { + let (parser_name, parser_code) = generate_parser( + r#"{ + "name": "test_direct_hidden_missing_node_ranges", + "rules": { + "source_file": { + "type": "SEQ", + "members": [ + {"type": "STRING", "value": "."}, + {"type": "SYMBOL", "name": "_hidden"} + ] + }, + "_hidden": {"type": "PATTERN", "value": "[A-Za-z0-9_]+"} + }, + "extras": [{"type": "PATTERN", "value": "\\s"}] + }"#, + ) + .unwrap(); + + let mut parser = Parser::new(); + parser + .set_language(&get_test_language(&parser_name, &parser_code, None)) + .unwrap(); + + let tree = parser.parse(".\n", None).unwrap(); + let root = tree.root_node(); + assert_eq!(first_error_or_container(root), Some(root)); + assert_eq!( + root.to_sexp_with_ranges(), + "(source_file [0, 0] - [1, 0] (MISSING _hidden [0, 1] - [0, 1]))" + ); +} diff --git a/crates/xtask/src/check_wasm_exports.rs b/crates/xtask/src/check_wasm_exports.rs index 228081311..b501c0773 100644 --- a/crates/xtask/src/check_wasm_exports.rs +++ b/crates/xtask/src/check_wasm_exports.rs @@ -16,10 +16,12 @@ use notify_debouncer_full::new_debouncer; use crate::{CheckWasmExports, bail_on_err, watch_wasm}; -const EXCLUDES: [&str; 25] = [ +const EXCLUDES: [&str; 26] = [ // Unneeded because the JS side has its own way of implementing it "ts_node_child_by_field_name", "ts_node_edit", + // Serialized point columns use bytes, unlike the Web binding's UTF-16 columns + "ts_node_string_with_ranges", // Precomputed and stored in the JS side "ts_node_type", "ts_node_grammar_type", diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 092e15a73..216b1e01d 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -354,6 +354,10 @@ unsafe extern "C" { #[doc = " Get an S-expression representing the node as a string.\n\n This string is allocated with `malloc` and the caller is responsible for\n freeing it using `free`."] pub fn ts_node_string(self_: TSNode) -> *mut ::core::ffi::c_char; } +unsafe extern "C" { + #[doc = " Get an S-expression representing the node as a string with start and end\n positions included.\n\n This string is allocated with `malloc` and the caller is responsible for\n freeing it using `free`."] + pub fn ts_node_string_with_ranges(self_: TSNode) -> *mut ::core::ffi::c_char; +} unsafe extern "C" { #[doc = " Check if the node is null. Functions like [`ts_node_child`] and\n [`ts_node_next_sibling`] will return a null node to indicate that no such node\n was found."] pub fn ts_node_is_null(self_: TSNode) -> bool; diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 90b3b5a41..72b18b37a 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -2109,6 +2109,19 @@ impl<'tree> Node<'tree> { result } + /// Get an S-expression representing the node with start and end positions included. + #[doc(alias = "ts_node_string_with_ranges")] + #[must_use] + pub fn to_sexp_with_ranges(&self) -> String { + let c_string = unsafe { ffi::ts_node_string_with_ranges(self.0) }; + let result = unsafe { CStr::from_ptr(c_string) } + .to_str() + .unwrap() + .to_string(); + unsafe { ts_free(c_string.cast::()) }; + result + } + pub fn utf8_text<'a>(&self, source: &'a [u8]) -> Result<&'a str, str::Utf8Error> { str::from_utf8(&source[self.start_byte()..self.end_byte()]) } @@ -4022,6 +4035,9 @@ pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String { write!(formatted, "{scratch} ").unwrap(); has_field = true; indent_level += 1; + } else if scratch.starts_with('[') || scratch.ends_with(']') || scratch == "-" { + // "[row," "column]" "-" "[row," "column]" + write!(formatted, " {scratch}").unwrap(); } } diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 12055f0c3..aa5b3970e 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -560,6 +560,15 @@ TSPoint ts_node_end_point(TSNode self); */ char *ts_node_string(TSNode self); +/** + * Get an S-expression representing the node as a string with start and end + * positions included. + * + * This string is allocated with `malloc` and the caller is responsible for + * freeing it using `free`. + */ +char *ts_node_string_with_ranges(TSNode self); + /** * Check if the node is null. Functions like [`ts_node_child`] and * [`ts_node_next_sibling`] will return a null node to indicate that no such node diff --git a/lib/src/node.c b/lib/src/node.c index ec7fee4cf..1714d091b 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -486,7 +486,22 @@ char *ts_node_string(TSNode self) { alias_symbol, ts_language_symbol_metadata(self.tree->language, alias_symbol).visible, self.tree->language, - false + false, + false, + (Length) {ts_node_start_byte(self), ts_node_start_point(self)} + ); +} + +char *ts_node_string_with_ranges(TSNode self) { + TSSymbol alias_symbol = ts_node__alias(&self); + return ts_subtree_string( + ts_node__subtree(self), + alias_symbol, + ts_language_symbol_metadata(self.tree->language, alias_symbol).visible, + self.tree->language, + false, + true, + (Length) {ts_node_start_byte(self), ts_node_start_point(self)} ); } diff --git a/lib/src/subtree.c b/lib/src/subtree.c index 20e3e4a8b..a74d82cc8 100644 --- a/lib/src/subtree.c +++ b/lib/src/subtree.c @@ -832,6 +832,8 @@ static const char *const ROOT_FIELD = "__ROOT__"; typedef struct { Subtree subtree; + Length position; + Length child_position; TSSymbol alias_symbol; bool alias_is_named; const char *field_name; @@ -846,9 +848,28 @@ typedef struct { const TSFieldMapEntry *field_map_end; } WriteToStringFrame; +static size_t ts_subtree__write_range_to_string( + char *string, + size_t limit, + Length start, + Length size +) { + Length end = length_add(start, size); + return snprintf( + string, + limit, + " [%u, %u] - [%u, %u]", + start.extent.row, + start.extent.column, + end.extent.row, + end.extent.column + ); +} + static size_t ts_subtree__write_to_string( Subtree self, char *string, size_t limit, const TSLanguage *language, bool include_all, + bool include_ranges, Length root_position, TSSymbol root_alias_symbol, bool root_alias_is_named, const char *root_field_name ) { char *cursor = string; @@ -857,6 +878,8 @@ static size_t ts_subtree__write_to_string( Array(WriteToStringFrame) stack = array_new(); array_push(&stack, ((WriteToStringFrame) { .subtree = self, + .position = root_position, + .child_position = root_position, .alias_symbol = root_alias_symbol, .alias_is_named = root_alias_is_named, .field_name = root_field_name, @@ -914,15 +937,35 @@ static size_t ts_subtree__write_to_string( cursor += snprintf(*writer, limit, "(%s", symbol_name); } } + + if (include_ranges) { + cursor += ts_subtree__write_range_to_string( + *writer, + limit, + frame->position, + ts_subtree_size(node) + ); + } } else if (frame->is_root) { TSSymbol symbol = frame->alias_symbol ? frame->alias_symbol : ts_subtree_symbol(node); const char *symbol_name = ts_language_symbol_name(language, symbol); if (ts_subtree_child_count(node) > 0) { cursor += snprintf(*writer, limit, "(%s", symbol_name); } else if (ts_subtree_named(node)) { - cursor += snprintf(*writer, limit, "(%s)", symbol_name); + cursor += snprintf(*writer, limit, "(%s", symbol_name); } else { - cursor += snprintf(*writer, limit, "(\"%s\")", symbol_name); + cursor += snprintf(*writer, limit, "(\"%s\"", symbol_name); + } + if (include_ranges) { + cursor += ts_subtree__write_range_to_string( + *writer, + limit, + frame->position, + ts_subtree_size(node) + ); + } + if (ts_subtree_child_count(node) == 0) { + cursor += snprintf(*writer, limit, ")"); } } @@ -942,8 +985,13 @@ static size_t ts_subtree__write_to_string( if (frame->child_index < ts_subtree_child_count(node)) { Subtree child = ts_subtree_children(node)[frame->child_index]; + if (frame->child_index > 0) { + frame->child_position = length_add(frame->child_position, ts_subtree_padding(child)); + } WriteToStringFrame child_frame = { .subtree = child, + .position = frame->child_position, + .child_position = frame->child_position, .is_root = false, }; @@ -972,6 +1020,7 @@ static size_t ts_subtree__write_to_string( } frame->child_index++; + frame->child_position = length_add(frame->child_position, ts_subtree_size(child)); // After this push, `frame` may be invalidated by a realloc. array_push(&stack, child_frame); continue; @@ -990,18 +1039,20 @@ char *ts_subtree_string( TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, - bool include_all + bool include_all, + bool include_ranges, + Length position ) { char scratch_string[1]; size_t size = ts_subtree__write_to_string( self, scratch_string, 1, - language, include_all, + language, include_all, include_ranges, position, alias_symbol, alias_is_named, ROOT_FIELD ) + 1; char *result = ts_malloc(size * sizeof(char)); ts_subtree__write_to_string( self, result, size, - language, include_all, + language, include_all, include_ranges, position, alias_symbol, alias_is_named, ROOT_FIELD ); return result; diff --git a/lib/src/subtree.h b/lib/src/subtree.h index 07e0059b5..50f6dade0 100644 --- a/lib/src/subtree.h +++ b/lib/src/subtree.h @@ -224,7 +224,15 @@ void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLangua void ts_subtree_compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack); void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *language); Subtree ts_subtree_edit(Subtree self, const TSInputEdit *edit, SubtreePool *pool); -char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, bool include_all); +char *ts_subtree_string( + Subtree self, + TSSymbol alias_symbol, + bool alias_is_named, + const TSLanguage *language, + bool include_all, + bool include_ranges, + Length position +); void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f); Subtree ts_subtree_last_external_token(Subtree tree); const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);