feat(lib): add state to missing nodes.

This allows them to return possible missing values from a `LookaheadIterator`.

lib/binding_rust/bindings.rs

@@ -383,7 +383,7 @@ unsafe extern "C" {
     pub fn ts_node_is_error(self_: TSNode) -> bool;
 }
 unsafe extern "C" {
-    #[doc = " Get this node's parse state."]
+    #[doc = " Get this node's parse state.\n\n For a missing node, this is the state from the recovery path that was\n selected by the parser. It can be used with [`ts_lookahead_iterator_new`] to\n inspect the symbols that are valid in that state. This does not necessarily\n include every symbol that could be recovered by inserting a missing node."]
     pub fn ts_node_parse_state(self_: TSNode) -> TSStateId;
 }
 unsafe extern "C" {
@@ -841,7 +841,7 @@ unsafe extern "C" {
     pub fn ts_language_name(self_: *const TSLanguage) -> *const ::core::ffi::c_char;
 }
 unsafe extern "C" {
-    #[doc = " Create a new lookahead iterator for the given language and parse state.\n\n This returns `NULL` if state is invalid for the language.\n\n Repeatedly using [`ts_lookahead_iterator_next`] and\n [`ts_lookahead_iterator_current_symbol`] will generate valid symbols in the\n given parse state. Newly created lookahead iterators will contain the `ERROR`\n symbol.\n\n Lookahead iterators can be useful to generate suggestions and improve syntax\n error diagnostics. To get symbols valid in an ERROR node, use the lookahead\n iterator on its first leaf node state. For `MISSING` nodes, a lookahead\n iterator created on the previous non-extra leaf node may be appropriate."]
+    #[doc = " Create a new lookahead iterator for the given language and parse state.\n\n This returns `NULL` if state is invalid for the language.\n\n Repeatedly using [`ts_lookahead_iterator_next`] and\n [`ts_lookahead_iterator_current_symbol`] will generate valid symbols in the\n given parse state. Newly created lookahead iterators will contain the `ERROR`\n symbol.\n\n Lookahead iterators can be useful to generate suggestions and improve syntax\n error diagnostics. To get symbols valid in an ERROR node, use the lookahead\n iterator on its first leaf node state. For `MISSING` nodes, a lookahead\n iterator created on the previous non-extra leaf node, or using the node's\n parse state may be appropriate."]
     pub fn ts_lookahead_iterator_new(
         self_: *const TSLanguage,
         state: TSStateId,

lib/binding_rust/lib.rs

@@ -675,9 +675,11 @@ impl Language {
     ///
     /// Lookahead iterators can be useful to generate suggestions and improve
     /// syntax error diagnostics. To get symbols valid in an `ERROR` node, use the
-    /// lookahead iterator on its first leaf node state. For `MISSING` nodes, a
-    /// lookahead iterator created on the previous non-extra leaf node may be
-    /// appropriate.
+    /// lookahead iterator on its first leaf node state. For a missing node, use
+    /// the node's [`parse_state`](Node::parse_state). Keep in mind that lookahead
+    /// symbols are valid in that parse state, but are not necessarily valid
+    /// continuations in the context of the actual following token or guaranteed
+    /// to be considered during error recovery.
     #[doc(alias = "ts_lookahead_iterator_new")]
     #[must_use]
     pub fn lookahead_iterator(&self, state: u16) -> Option<LookaheadIterator> {
@@ -1689,7 +1691,14 @@ impl<'tree> Node<'tree> {
         unsafe { ffi::ts_node_is_error(self.0) }
     }
 
-    /// Get this node's parse state.
+    /// Get the parse state immediately before this node.
+    ///
+    /// For a missing node, this is the state from the recovery path that was
+    /// selected by the parser. It can be used with
+    /// [`Language::lookahead_iterator`] to inspect the symbols that are valid in
+    /// that state. This does not necessarily include every symbol that could be
+    /// recovered by inserting a missing node, because the recovery process can
+    /// consider multiple stack versions.
     #[doc(alias = "ts_node_parse_state")]
     #[must_use]
     pub fn parse_state(&self) -> u16 {

lib/include/tree_sitter/api.h

@@ -591,6 +591,11 @@ bool ts_node_is_error(TSNode self);
 
 /**
  * Get this node's parse state.
+ *
+ * For a missing node, this is the state from the recovery path that was
+ * selected by the parser. It can be used with [`ts_lookahead_iterator_new`] to
+ * inspect the symbols that are valid in that state. This does not necessarily
+ * include every symbol that could be recovered by inserting a missing node.
 */
 TSStateId ts_node_parse_state(TSNode self);
 
@@ -1328,7 +1333,8 @@ const char *ts_language_name(const TSLanguage *self);
  * Lookahead iterators can be useful to generate suggestions and improve syntax
  * error diagnostics. To get symbols valid in an ERROR node, use the lookahead
  * iterator on its first leaf node state. For `MISSING` nodes, a lookahead
- * iterator created on the previous non-extra leaf node may be appropriate.
+ * iterator created on the previous non-extra leaf node, or using the node's
+ * parse state may be appropriate.
 */
 TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state);
 

lib/src/parser.c

@@ -1514,7 +1514,7 @@ static void ts_parser__handle_error(
 
           StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v);
           Subtree missing_tree = ts_subtree_new_missing_leaf(
-            &self->tree_pool, missing_symbol,
+            &self->tree_pool, missing_symbol, state,
             padding, lookahead_bytes,
             self->language
           );

lib/src/subtree.c

@@ -539,13 +539,14 @@ Subtree ts_subtree_new_error_node(
 Subtree ts_subtree_new_missing_leaf(
   SubtreePool *pool,
   TSSymbol symbol,
+  TSStateId state,
   Length padding,
   uint32_t lookahead_bytes,
   const TSLanguage *language
 ) {
   Subtree result = ts_subtree_new_leaf(
     pool, symbol, padding, length_zero(), lookahead_bytes,
-    0, false, false, false, language
+    state, false, false, false, language
   );
   if (result.data.is_inline) {
     result.data.is_missing = true;

lib/src/subtree.h

@@ -211,6 +211,7 @@ Subtree ts_subtree_new_error_node(
 Subtree ts_subtree_new_missing_leaf(
   SubtreePool *pool,
   TSSymbol symbol,
+  TSStateId state,
   Length padding,
   uint32_t lookahead_bytes,
   const TSLanguage *language