fix(lib)!: make lookahead iterator exhaustion sticky and retain its language

Track the iterator's phase so exhaustion is sticky, and gate the symbol name on
it, so `NULL` means "not positioned on a symbol".

`ts_lookahead_iterator_new` was also the only language consuming constructor
that did not `ts_language_copy`, so an iterator outliving a wasm language read
freed memory. Retain in `_new` and `_reset`, release in `_delete`.

Previously:
- `ts_lookahead_iterator__next` left a small parse state's cursor one past its
group end, so re-advancing an exhausted iterator resumed returning `true` and
walked through the rest of `ts_small_parse_table` and off the end of it.

- `ts_lookahead_iterator_current_symbol_name` returned `NULL` only when the
exhausted symbol index happened to fall outside the names table, so a grammar
with aliases returned a real but wrong name instead.

lib/binding_rust/bindings.rs

@@ -841,7 +841,7 @@ unsafe extern "C" {
     pub fn ts_language_name(self_: *const TSLanguage) -> *const ::core::ffi::c_char;
 }
 unsafe extern "C" {
-    #[doc = " Create a new lookahead iterator for the given language and parse state.\n\n This returns `NULL` if state is invalid for the language.\n\n Repeatedly using [`ts_lookahead_iterator_next`] and\n [`ts_lookahead_iterator_current_symbol`] will generate valid symbols in the\n given parse state. Newly created lookahead iterators will contain the `ERROR`\n symbol.\n\n Lookahead iterators can be useful to generate suggestions and improve syntax\n error diagnostics. To get symbols valid in an ERROR node, use the lookahead\n iterator on its first leaf node state. For `MISSING` nodes, a lookahead\n iterator created on the previous non-extra leaf node, or using the node's\n parse state may be appropriate."]
+    #[doc = " Create a new lookahead iterator for the given language and parse state.\n\n This returns `NULL` if state is invalid for the language.\n\n Repeatedly using [`ts_lookahead_iterator_next`] and\n [`ts_lookahead_iterator_current_symbol`] will generate valid symbols in the\n given parse state. A newly created iterator is not positioned on a symbol\n until [`ts_lookahead_iterator_next`] is called.\n\n The iterator retains the language, so the language may be deleted while the\n iterator is still in use.\n\n Lookahead iterators can be useful to generate suggestions and improve syntax\n error diagnostics. To get symbols valid in an ERROR node, use the lookahead\n iterator on its first leaf node state. For `MISSING` nodes, a lookahead\n iterator created on the previous non-extra leaf node, or using the node's\n parse state may be appropriate."]
     pub fn ts_lookahead_iterator_new(
         self_: *const TSLanguage,
         state: TSStateId,
@@ -852,14 +852,14 @@ unsafe extern "C" {
     pub fn ts_lookahead_iterator_delete(self_: *mut TSLookaheadIterator);
 }
 unsafe extern "C" {
-    #[doc = " Reset the lookahead iterator to another state.\n\n This returns `true` if the iterator was reset to the given state and `false`\n otherwise."]
+    #[doc = " Reset the lookahead iterator to another state.\n\n This returns `true` if the iterator was reset to the given state and `false`\n otherwise. A reset iterator is not positioned on a symbol."]
     pub fn ts_lookahead_iterator_reset_state(
         self_: *mut TSLookaheadIterator,
         state: TSStateId,
     ) -> bool;
 }
 unsafe extern "C" {
-    #[doc = " Reset the lookahead iterator.\n\n This returns `true` if the language was set successfully and `false`\n otherwise."]
+    #[doc = " Reset the lookahead iterator.\n\n This returns `true` if the language was set successfully and `false`\n otherwise. A reset iterator is not positioned on a symbol."]
     pub fn ts_lookahead_iterator_reset(
         self_: *mut TSLookaheadIterator,
         language: *const TSLanguage,
@@ -875,11 +875,11 @@ unsafe extern "C" {
     pub fn ts_lookahead_iterator_next(self_: *mut TSLookaheadIterator) -> bool;
 }
 unsafe extern "C" {
-    #[doc = " Get the current symbol of the lookahead iterator;"]
+    #[doc = " Get the current symbol of the lookahead iterator.\n\n This is only meaningful when the most recent call to\n [`ts_lookahead_iterator_next`] on `self` returned `true`."]
     pub fn ts_lookahead_iterator_current_symbol(self_: *const TSLookaheadIterator) -> TSSymbol;
 }
 unsafe extern "C" {
-    #[doc = " Get the current symbol type of the lookahead iterator as a null terminated\n string."]
+    #[doc = " Get the current symbol type of the lookahead iterator as a null terminated\n string.\n\n This returns `NULL` unless the most recent call to\n [`ts_lookahead_iterator_next`] on `self` returned `true`."]
     pub fn ts_lookahead_iterator_current_symbol_name(
         self_: *const TSLookaheadIterator,
     ) -> *const ::core::ffi::c_char;

lib/include/tree_sitter/api.h

@@ -1327,8 +1327,11 @@ const char *ts_language_name(const TSLanguage *self);
  *
  * Repeatedly using [`ts_lookahead_iterator_next`] and
  * [`ts_lookahead_iterator_current_symbol`] will generate valid symbols in the
- * given parse state. Newly created lookahead iterators will contain the `ERROR`
- * symbol.
+ * given parse state. A newly created iterator is not positioned on a symbol
+ * until [`ts_lookahead_iterator_next`] is called.
+ *
+ * The iterator retains the language, so the language may be deleted while the
+ * iterator is still in use.
  *
  * Lookahead iterators can be useful to generate suggestions and improve syntax
  * error diagnostics. To get symbols valid in an ERROR node, use the lookahead
@@ -1347,7 +1350,7 @@ void ts_lookahead_iterator_delete(TSLookaheadIterator *self);
  * Reset the lookahead iterator to another state.
  *
  * This returns `true` if the iterator was reset to the given state and `false`
- * otherwise.
+ * otherwise. A reset iterator is not positioned on a symbol.
 */
 bool ts_lookahead_iterator_reset_state(TSLookaheadIterator *self, TSStateId state);
 
@@ -1355,7 +1358,7 @@ bool ts_lookahead_iterator_reset_state(TSLookaheadIterator *self, TSStateId stat
  * Reset the lookahead iterator.
  *
  * This returns `true` if the language was set successfully and `false`
- * otherwise.
+ * otherwise. A reset iterator is not positioned on a symbol.
 */
 bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state);
 
@@ -1372,13 +1375,19 @@ const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self
 bool ts_lookahead_iterator_next(TSLookaheadIterator *self);
 
 /**
- * Get the current symbol of the lookahead iterator;
+ * Get the current symbol of the lookahead iterator.
+ *
+ * This is only meaningful when the most recent call to
+ * [`ts_lookahead_iterator_next`] on `self` returned `true`.
 */
 TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self);
 
 /**
  * Get the current symbol type of the lookahead iterator as a null terminated
  * string.
+ *
+ * This returns `NULL` unless the most recent call to
+ * [`ts_lookahead_iterator_next`] on `self` returned `true`.
 */
 const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self);
 

lib/src/language.c

@@ -246,12 +246,15 @@ TSFieldId ts_language_field_id_for_name(
 TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) {
   if (state >= self->state_count) return NULL;
   LookaheadIterator *iterator = ts_malloc(sizeof(LookaheadIterator));
-  *iterator = ts_language_lookaheads(self, state);
+  *iterator = ts_language_lookaheads(ts_language_copy(self), state);
   return (TSLookaheadIterator *)iterator;
 }
 
 void ts_lookahead_iterator_delete(TSLookaheadIterator *self) {
-  ts_free(self);
+  if (!self) return;
+  LookaheadIterator *iterator = (LookaheadIterator *)self;
+  ts_language_delete(iterator->language);
+  ts_free(iterator);
 }
 
 bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) {
@@ -269,7 +272,9 @@ const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self
 bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) {
   if (state >= language->state_count) return false;
   LookaheadIterator *iterator = (LookaheadIterator *)self;
-  *iterator = ts_language_lookaheads(language, state);
+  const TSLanguage *previous = iterator->language;
+  *iterator = ts_language_lookaheads(ts_language_copy(language), state);
+  ts_language_delete(previous);
   return true;
 }
 
@@ -285,5 +290,6 @@ TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
 
 const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) {
   const LookaheadIterator *iterator = (const LookaheadIterator *)self;
+  if (iterator->phase != LookaheadPositioned) return NULL;
   return ts_language_symbol_name(iterator->language, iterator->symbol);
 }

lib/src/language.h

@@ -19,15 +19,20 @@ typedef struct {
   bool is_reusable;
 } TableEntry;
 
+typedef enum {
+  LookaheadFresh,      // no `next()` yet
+  LookaheadPositioned, // last `next()` returned true
+  LookaheadDone,       // last `next()` returned false
+} LookaheadPhase;
+
 typedef struct {
   const TSLanguage *language;
   const uint16_t *data;
   const uint16_t *group_end;
-  TSStateId state;
   uint16_t table_value;
-  uint16_t section_index;
   uint16_t group_count;
   bool is_small_state;
+  LookaheadPhase phase;
 
   const TSParseAction *actions;
   TSSymbol symbol;
@@ -120,7 +125,7 @@ static inline LookaheadIterator ts_language_lookaheads(
     group_end = data + 1;
     group_count = *data;
   } else {
-    data = &self->parse_table[state * self->symbol_count] - 1;
+    data = &self->parse_table[state * self->symbol_count];
   }
   return (LookaheadIterator) {
     .language = self,
@@ -128,19 +133,25 @@ static inline LookaheadIterator ts_language_lookaheads(
     .group_end = group_end,
     .group_count = group_count,
     .is_small_state = is_small_state,
+    .phase = LookaheadFresh,
     .symbol = UINT16_MAX,
     .next_state = 0,
   };
 }
 
 static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) {
+  if (self->phase == LookaheadDone) return false;
+
   // For small parse states, valid symbols are listed explicitly,
   // grouped by their value. There's no need to look up the actions
   // again until moving to the next group.
   if (self->is_small_state) {
     self->data++;
     if (self->data == self->group_end) {
-      if (self->group_count == 0) return false;
+      if (self->group_count == 0) {
+        self->phase = LookaheadDone;
+        return false;
+      }
       self->group_count--;
       self->table_value = *(self->data++);
       unsigned symbol_count = *(self->data++);
@@ -148,6 +159,7 @@ static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) {
       self->symbol = *self->data;
     } else {
       self->symbol = *self->data;
+      self->phase = LookaheadPositioned;
       return true;
     }
   }
@@ -155,12 +167,15 @@ static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) {
   // For large parse states, iterate through every symbol until one
   // is found that has valid actions.
   else {
-    do {
-      self->data++;
-      self->symbol++;
-      if (self->symbol >= self->language->symbol_count) return false;
-      self->table_value = *self->data;
-    } while (!self->table_value);
+    const uint16_t *row = self->data;
+    TSSymbol symbol = self->phase == LookaheadFresh ? 0 : self->symbol + 1;
+    while (symbol < self->language->symbol_count && !row[symbol]) symbol++;
+    if (symbol >= self->language->symbol_count) {
+      self->phase = LookaheadDone;
+      return false;
+    }
+    self->symbol = symbol;
+    self->table_value = row[symbol];
   }
 
   // Depending on if the symbol is terminal or non-terminal, the table value either
@@ -174,6 +189,7 @@ static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) {
     self->action_count = 0;
     self->next_state = self->table_value;
   }
+  self->phase = LookaheadPositioned;
   return true;
 }