p_parse_inner: do not consume the follow token

spec/test_parse_inner_shared.c

@@ -50,13 +50,26 @@ int main()
     /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
      * that parse_Start uses to select R2 over R1 in the ambiguous state, from
      * R1's start state the reduce to R1 is unconditional, and the follow-
-     * token shift retry at the R1-accepting state completes the parse. */
+     * token shift retry at the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed from
+     * the input: p_position() should point to the follow token, and a
+     * subsequent p_lex() should return it. */
     {
         input = "abb";
         context = p_context_new((uint8_t const *)input, strlen(input));
         p_token_t follow_tokens[] = { TOKEN_b };
         assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
         assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `b` is at column 3 (1-based). */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
         p_context_delete(context);
     }
 
@@ -69,6 +82,14 @@ int main()
         p_token_t follow_tokens[] = { TOKEN_a };
         assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
         assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `a` is at column 3 (1-based) and remains in the
+         * input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
         p_context_delete(context);
     }
 

spec/test_parse_inner_shared.d

@@ -34,12 +34,23 @@ unittest
     /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
      * parse_Start would use to select R2 over R1, but from R1's own start
      * state R1 reduces unconditionally, and the follow-token shift retry at
-     * the R1-accepting state completes the parse. */
+     * the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed: a
+     * subsequent p_lex() should return it. */
     input = "abb";
     context = p_context_new(input);
     p_token_t[] follow_tokens_b = [TOKEN_b];
     assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
     assert(p_result_R1(context) == 11);
+    p_position_t pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    p_token_info_t token_info;
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_b);
+    assert(token_info.position.row == 1);
+    assert(token_info.position.col == 3);
 
     /* parse_inner_R1("aba", [a]) also succeeds. */
     input = "aba";
@@ -47,6 +58,11 @@ unittest
     p_token_t[] follow_tokens_a = [TOKEN_a];
     assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
     assert(p_result_R1(context) == 11);
+    pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_a);
 
     /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
     input = "ab";