fix(lib): Consider subtree lookahead bytes when determining whether the

parser can reuse a node.

Lookahead bytes can be used to decide what a node is parsed as, so it's
resaonable to consider this as part of a node's "range" when deciding
which edits affect it.

(cherry picked from commit 15ddfb21ed)

crates/cli/src/tests/tree_test.rs

@@ -795,3 +795,44 @@ fn get_changed_ranges(
     *tree = new_tree;
     result
 }
+
+// Regression test for an incremental reparse bug where an external
+// scanner's choice depends on lexer->eof() (and thus on the parser's
+// current included ranges). The cached token at byte 0 was emitted when
+// the included range stopped just after the opener. Widening the range
+// to include a later matching delimiter must invalidate that cached
+// token so the scanner re-runs and emits the open form instead of the
+// unclosed form.
+#[test]
+fn test_reuse_invalidates_scanner_token_when_included_range_expands() {
+    let language = get_test_fixture_language("external_lookahead_eof_boundary");
+    let mut parser = Parser::new();
+    parser.set_language(&language).unwrap();
+
+    let source = "``";
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 1,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 1),
+        }])
+        .unwrap();
+    let tree1 = parser.parse(source, None).unwrap();
+    assert_eq!(tree1.root_node().to_sexp(), "(document (unclosed_delim))");
+
+    parser
+        .set_included_ranges(&[Range {
+            start_byte: 0,
+            end_byte: 2,
+            start_point: Point::new(0, 0),
+            end_point: Point::new(0, 2),
+        }])
+        .unwrap();
+    let tree2 = parser.parse(source, Some(&tree1)).unwrap();
+    assert_eq!(
+        tree2.root_node().to_sexp(),
+        "(document (span (open_delim) (close_delim)))"
+    );
+}

lib/src/parser.c

@@ -795,7 +795,13 @@ static Subtree ts_parser__reuse_node(
       reason = "is_missing";
     } else if (ts_subtree_is_fragile(result)) {
       reason = "is_fragile";
-    } else if (ts_parser__has_included_range_difference(self, byte_offset, end_byte_offset)) {
+    } else if (ts_parser__has_included_range_difference(
+                 self,
+                 byte_offset,
+                 ts_subtree_is_eof(result)
+                   ? end_byte_offset
+                   : end_byte_offset + ts_subtree_lookahead_bytes(result)
+               )) {
       reason = "contains_different_included_range";
     }
 

test/fixtures/test_grammars/external_lookahead_eof_boundary/corpus.txt

@@ -0,0 +1,13 @@
+=====================
+opened and closed span
+=====================
+``
+---
+(document (span (open_delim) (close_delim)))
+
+==================
+unclosed delimiter
+==================
+`
+---
+(document (unclosed_delim))

test/fixtures/test_grammars/external_lookahead_eof_boundary/grammar.js

@@ -0,0 +1,15 @@
+// External scanner whose token choice depends on `lexer->eof()`: at a
+// '`' it peeks past mark_end for a matching close, emitting open_delim
+// if one is found or unclosed_delim if eof is reached first. Inside an
+// open span the same '`' is emitted as close_delim.
+
+export default grammar({
+    name: 'external_lookahead_eof_boundary',
+
+    externals: $ => [$.open_delim, $.close_delim, $.unclosed_delim],
+
+    rules: {
+        document: $ => repeat(choice($.span, $.unclosed_delim)),
+        span: $ => seq($.open_delim, $.close_delim),
+    }
+});

test/fixtures/test_grammars/external_lookahead_eof_boundary/scanner.c

@@ -0,0 +1,54 @@
+#include "tree_sitter/parser.h"
+
+enum {
+    OPEN_DELIM,
+    CLOSE_DELIM,
+    UNCLOSED_DELIM,
+};
+
+void *tree_sitter_external_lookahead_eof_boundary_external_scanner_create(void) { return NULL; }
+
+void tree_sitter_external_lookahead_eof_boundary_external_scanner_destroy(void *payload) {}
+
+unsigned tree_sitter_external_lookahead_eof_boundary_external_scanner_serialize(
+    void *payload, char *buffer
+) {
+    return 0;
+}
+
+void tree_sitter_external_lookahead_eof_boundary_external_scanner_deserialize(
+    void *payload, const char *buffer, unsigned length
+) {}
+
+bool tree_sitter_external_lookahead_eof_boundary_external_scanner_scan(
+    void *payload, TSLexer *lexer, const bool *valid_symbols
+) {
+    if (lexer->lookahead != '`') return false;
+
+    lexer->advance(lexer, false);
+    lexer->mark_end(lexer);
+
+    // Mid-span: emit the closer.
+    if (valid_symbols[CLOSE_DELIM]) {
+        lexer->result_symbol = CLOSE_DELIM;
+        return true;
+    }
+
+    // At an opener: peek past mark_end for a matching '`'. The eof()
+    // probe makes the result depend on the current included ranges.
+    if (valid_symbols[OPEN_DELIM]) {
+        while (!lexer->eof(lexer)) {
+            if (lexer->lookahead == '`') {
+                lexer->result_symbol = OPEN_DELIM;
+                return true;
+            }
+            lexer->advance(lexer, false);
+        }
+        if (valid_symbols[UNCLOSED_DELIM]) {
+            lexer->result_symbol = UNCLOSED_DELIM;
+            return true;
+        }
+    }
+
+    return false;
+}