Add p_parse_inner_XXX() APIs

spec/propane_spec.rb

@@ -1589,6 +1589,56 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "supports parse_inner APIs that treat provided tokens as follow tokens" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+Start -> Y << $$ = $1; >>
+Y -> a << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs block success when the outer rule is unfinished" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> a Start b << $$ = $2; >>
+Start -> c << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_recursive.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
+    write_grammar <<EOF
+ptype int;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b << $$ = 11; >>
+R2 -> a b << $$ = 22; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_shared.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows multiple starting rules in tree mode" do
     write_grammar <<EOF
 tree;

spec/test_parse_inner.c

@@ -0,0 +1,73 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (chain reduce):
+     *   Start -> Y      << $$ = $1; >>
+     *   Y     -> a      << $$ = $1; >>
+     *   token a         << $$ = 1; >>
+     *
+     * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF,
+     * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with
+     * `b` as a follow token should succeed via the reduce-side retry chain
+     * (Y then Start) followed by the shift-side retry hitting $EOF at the
+     * final state. */
+
+    /* Standard parse succeeds on complete input. */
+    char const * input = "a";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(1u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with an empty (NULL) follow-token vector behaves like a
+     * standard parse. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    {
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+    }
+    p_context_delete(context);
+
+    /* parse_inner with a non-matching follow token still fails. The grammar
+     * can't consume `b` and it isn't listed as a follow token, so the retries
+     * do not fire. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner.c for details on the grammar and cases. */
+
+    /* Standard parse succeeds on complete input. */
+    string input = "a";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a null follow-token slice behaves like a standard
+     * parse. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a non-matching follow token still fails. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_eof = [TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN);
+}

spec/test_parse_inner_recursive.c

@@ -0,0 +1,77 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (recursive):
+     *   Start -> a Start b   << $$ = $2; >>
+     *   Start -> c           << $$ = $1; >>
+     *   token a << $$ = 1; >>
+     *   token b << $$ = 2; >>
+     *   token c << $$ = 3; >>
+     *
+     * Here `Start` can appear in the middle of another `Start` rule, so the
+     * inner-parse follow-token success must be blocked whenever an unfinished
+     * outer `Start -> a Start b` remains on the parse stack (i.e. the parse
+     * stack contains more than just the initial state and the reduced start
+     * rule set). */
+
+    /* Standard parse of `c` succeeds. */
+    char const * input = "c";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `acb` succeeds (full outer rule). */
+    input = "acb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `ac` fails (`b` missing). */
+    input = "ac";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner with `ac` and follow token `b` also fails: even though the
+     * inner `Start -> c` reduces and `Start` is shifted, the outer
+     * `Start -> a Start . b` is still on the stack (stack length > 2), so the
+     * "reduced start rule is the only thing on the parse stack" invariant
+     * blocks the shift-side follow-token success. */
+    {
+        input = "ac";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    /* parse_inner with `acb` (complete outer rule) succeeds via the standard
+     * path. */
+    {
+        input = "acb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with just `c` succeeds via the standard path even when a
+     * follow-token vector is supplied. */
+    {
+        input = "c";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_recursive.d

@@ -0,0 +1,49 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_recursive.c for details on the grammar. */
+
+    /* Standard parse of `c` succeeds. */
+    string input = "c";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `acb` succeeds. */
+    input = "acb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `ac` fails. */
+    input = "ac";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `ac` fails: outer rule still on the stack. */
+    input = "ac";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `acb` succeeds via the standard path. */
+    input = "acb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* parse_inner with just `c` succeeds via the standard path. */
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+}

spec/test_parse_inner_shared.c

@@ -0,0 +1,83 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar:
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b   << $$ = 11; >>
+     *   R2 -> a b   << $$ = 22; >>
+     *   token a; token b;
+     *
+     * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within
+     * parse_Start, the generated parser differentiates the reduce by
+     * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2
+     * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional
+     * on any lookahead. This test exercises p_parse_inner_R1() to confirm
+     * that reductions to R1 succeed even when the incoming follow token is
+     * not the natural lookahead used by parse_Start's disambiguation. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the
+     * shared "a b" state. */
+    char const * input = "aba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    /* Standard parse of R1 fails on "abb" (unexpected trailing token). */
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
+     * that parse_Start uses to select R2 over R1 in the ambiguous state, from
+     * R1's start state the reduce to R1 is unconditional, and the follow-
+     * token shift retry at the R1-accepting state completes the parse. */
+    {
+        input = "abb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token
+     * parse_Start uses to select R1, and it works here as a follow token
+     * too. */
+    {
+        input = "aba";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_a };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_shared.d

@@ -0,0 +1,56 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_shared.c for details on the grammar. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */
+    string input = "aba";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* Standard parse of R1 fails on "abb". */
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
+     * parse_Start would use to select R2 over R1, but from R1's own start
+     * state R1 reduces unconditionally, and the follow-token shift retry at
+     * the R1-accepting state completes the parse. */
+    input = "abb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* parse_inner_R1("aba", [a]) also succeeds. */
+    input = "aba";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_a = [TOKEN_a];
+    assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_R1(context, null) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+}