diff --git a/crates/cli/src/tests/tree_test.rs b/crates/cli/src/tests/tree_test.rs index a4941bb66..c2be487f0 100644 --- a/crates/cli/src/tests/tree_test.rs +++ b/crates/cli/src/tests/tree_test.rs @@ -795,3 +795,44 @@ fn get_changed_ranges( *tree = new_tree; result } + +// Regression test for an incremental reparse bug where an external +// scanner's choice depends on lexer->eof() (and thus on the parser's +// current included ranges). The cached token at byte 0 was emitted when +// the included range stopped just after the opener. Widening the range +// to include a later matching delimiter must invalidate that cached +// token so the scanner re-runs and emits the open form instead of the +// unclosed form. +#[test] +fn test_reuse_invalidates_scanner_token_when_included_range_expands() { + let language = get_test_fixture_language("external_lookahead_eof_boundary"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let source = "``"; + + parser + .set_included_ranges(&[Range { + start_byte: 0, + end_byte: 1, + start_point: Point::new(0, 0), + end_point: Point::new(0, 1), + }]) + .unwrap(); + let tree1 = parser.parse(source, None).unwrap(); + assert_eq!(tree1.root_node().to_sexp(), "(document (unclosed_delim))"); + + parser + .set_included_ranges(&[Range { + start_byte: 0, + end_byte: 2, + start_point: Point::new(0, 0), + end_point: Point::new(0, 2), + }]) + .unwrap(); + let tree2 = parser.parse(source, Some(&tree1)).unwrap(); + assert_eq!( + tree2.root_node().to_sexp(), + "(document (span (open_delim) (close_delim)))" + ); +} diff --git a/lib/src/parser.c b/lib/src/parser.c index 714c2a19a..50cb29215 100644 --- a/lib/src/parser.c +++ b/lib/src/parser.c @@ -795,7 +795,13 @@ static Subtree ts_parser__reuse_node( reason = "is_missing"; } else if (ts_subtree_is_fragile(result)) { reason = "is_fragile"; - } else if (ts_parser__has_included_range_difference(self, byte_offset, end_byte_offset)) { + } else if (ts_parser__has_included_range_difference( + self, + byte_offset, + ts_subtree_is_eof(result) + ? end_byte_offset + : end_byte_offset + ts_subtree_lookahead_bytes(result) + )) { reason = "contains_different_included_range"; } diff --git a/test/fixtures/test_grammars/external_lookahead_eof_boundary/corpus.txt b/test/fixtures/test_grammars/external_lookahead_eof_boundary/corpus.txt new file mode 100644 index 000000000..a1b3179e5 --- /dev/null +++ b/test/fixtures/test_grammars/external_lookahead_eof_boundary/corpus.txt @@ -0,0 +1,13 @@ +===================== +opened and closed span +===================== +`` +--- +(document (span (open_delim) (close_delim))) + +================== +unclosed delimiter +================== +` +--- +(document (unclosed_delim)) diff --git a/test/fixtures/test_grammars/external_lookahead_eof_boundary/grammar.js b/test/fixtures/test_grammars/external_lookahead_eof_boundary/grammar.js new file mode 100644 index 000000000..762f357c2 --- /dev/null +++ b/test/fixtures/test_grammars/external_lookahead_eof_boundary/grammar.js @@ -0,0 +1,15 @@ +// External scanner whose token choice depends on `lexer->eof()`: at a +// '`' it peeks past mark_end for a matching close, emitting open_delim +// if one is found or unclosed_delim if eof is reached first. Inside an +// open span the same '`' is emitted as close_delim. + +export default grammar({ + name: 'external_lookahead_eof_boundary', + + externals: $ => [$.open_delim, $.close_delim, $.unclosed_delim], + + rules: { + document: $ => repeat(choice($.span, $.unclosed_delim)), + span: $ => seq($.open_delim, $.close_delim), + } +}); diff --git a/test/fixtures/test_grammars/external_lookahead_eof_boundary/scanner.c b/test/fixtures/test_grammars/external_lookahead_eof_boundary/scanner.c new file mode 100644 index 000000000..9e241e822 --- /dev/null +++ b/test/fixtures/test_grammars/external_lookahead_eof_boundary/scanner.c @@ -0,0 +1,54 @@ +#include "tree_sitter/parser.h" + +enum { + OPEN_DELIM, + CLOSE_DELIM, + UNCLOSED_DELIM, +}; + +void *tree_sitter_external_lookahead_eof_boundary_external_scanner_create(void) { return NULL; } + +void tree_sitter_external_lookahead_eof_boundary_external_scanner_destroy(void *payload) {} + +unsigned tree_sitter_external_lookahead_eof_boundary_external_scanner_serialize( + void *payload, char *buffer +) { + return 0; +} + +void tree_sitter_external_lookahead_eof_boundary_external_scanner_deserialize( + void *payload, const char *buffer, unsigned length +) {} + +bool tree_sitter_external_lookahead_eof_boundary_external_scanner_scan( + void *payload, TSLexer *lexer, const bool *valid_symbols +) { + if (lexer->lookahead != '`') return false; + + lexer->advance(lexer, false); + lexer->mark_end(lexer); + + // Mid-span: emit the closer. + if (valid_symbols[CLOSE_DELIM]) { + lexer->result_symbol = CLOSE_DELIM; + return true; + } + + // At an opener: peek past mark_end for a matching '`'. The eof() + // probe makes the result depend on the current included ranges. + if (valid_symbols[OPEN_DELIM]) { + while (!lexer->eof(lexer)) { + if (lexer->lookahead == '`') { + lexer->result_symbol = OPEN_DELIM; + return true; + } + lexer->advance(lexer, false); + } + if (valid_symbols[UNCLOSED_DELIM]) { + lexer->result_symbol = UNCLOSED_DELIM; + return true; + } + } + + return false; +}