p_parse_inner: do not consume the follow token

assets/parser.c.erb

@@ -1106,7 +1106,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                     (statevalues.length == 2u) &&
                     (last_shifted_rule_set_id == start_rule_set_id))
                 {
-                    /* Successful parse via follow token. */
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context->input_index -= token_info.length;
+                    context->text_position = token_info.position;
 <% if @grammar.tree %>
                     context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
 <% else %>

assets/parser.d.erb

@@ -1171,7 +1171,13 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                     (statevalues.length == 2u) &&
                     (last_shifted_rule_set_id == start_rule_set_id))
                 {
-                    /* Successful parse via follow token. */
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context.input_index -= token_info.length;
+                    context.text_position = token_info.position;
 <% if @grammar.tree %>
                     context.parse_result = statevalues[$-1].tree_node;
 <% else %>

doc/user_guide.md

@@ -1397,6 +1397,12 @@ size_t p_parse_inner_Statement(p_context_t * context,
 Passing `null` for the slice makes the function behave identically to
 `p_parse_Statement()`.
 
+When the parse is completed via a non-EOF follow token, that follow token is
+**not** consumed from the input stream.
+The parser rewinds the input index and text position to the start of the follow
+token so that a subsequent call to `p_lex()` or another parse function sees the
+same token.
+
 ### `p_position_valid`
 
 The `p_position_valid()` function is only generated for C targets.

spec/test_parse_inner_shared.c

@@ -50,13 +50,26 @@ int main()
     /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
      * that parse_Start uses to select R2 over R1 in the ambiguous state, from
      * R1's start state the reduce to R1 is unconditional, and the follow-
-     * token shift retry at the R1-accepting state completes the parse. */
+     * token shift retry at the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed from
+     * the input: p_position() should point to the follow token, and a
+     * subsequent p_lex() should return it. */
     {
         input = "abb";
         context = p_context_new((uint8_t const *)input, strlen(input));
         p_token_t follow_tokens[] = { TOKEN_b };
         assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
         assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `b` is at column 3 (1-based). */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
         p_context_delete(context);
     }
 
@@ -69,6 +82,14 @@ int main()
         p_token_t follow_tokens[] = { TOKEN_a };
         assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
         assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `a` is at column 3 (1-based) and remains in the
+         * input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
         p_context_delete(context);
     }
 

spec/test_parse_inner_shared.d

@@ -34,12 +34,23 @@ unittest
     /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
      * parse_Start would use to select R2 over R1, but from R1's own start
      * state R1 reduces unconditionally, and the follow-token shift retry at
-     * the R1-accepting state completes the parse. */
+     * the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed: a
+     * subsequent p_lex() should return it. */
     input = "abb";
     context = p_context_new(input);
     p_token_t[] follow_tokens_b = [TOKEN_b];
     assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
     assert(p_result_R1(context) == 11);
+    p_position_t pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    p_token_info_t token_info;
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_b);
+    assert(token_info.position.row == 1);
+    assert(token_info.position.col == 3);
 
     /* parse_inner_R1("aba", [a]) also succeeds. */
     input = "aba";
@@ -47,6 +58,11 @@ unittest
     p_token_t[] follow_tokens_a = [TOKEN_a];
     assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
     assert(p_result_R1(context) == 11);
+    pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_a);
 
     /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
     input = "ab";