From 946eb4eef7a2839ef3e2982ecf5dc7bc6f6b7852 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Sat, 11 Jul 2026 20:08:56 -0400 Subject: [PATCH] Add p_parse_inner_XXX() APIs --- CHANGELOG.md | 10 ++++ assets/parser.c.erb | 75 ++++++++++++++++++++++++++-- assets/parser.d.erb | 73 +++++++++++++++++++++++++-- assets/parser.h.erb | 2 + doc/user_guide.md | 30 +++++++++++ spec/propane_spec.rb | 50 +++++++++++++++++++ spec/test_parse_inner.c | 73 +++++++++++++++++++++++++++ spec/test_parse_inner.d | 51 +++++++++++++++++++ spec/test_parse_inner_recursive.c | 77 ++++++++++++++++++++++++++++ spec/test_parse_inner_recursive.d | 49 ++++++++++++++++++ spec/test_parse_inner_shared.c | 83 +++++++++++++++++++++++++++++++ spec/test_parse_inner_shared.d | 56 +++++++++++++++++++++ 12 files changed, 619 insertions(+), 10 deletions(-) create mode 100644 spec/test_parse_inner.c create mode 100644 spec/test_parse_inner.d create mode 100644 spec/test_parse_inner_recursive.c create mode 100644 spec/test_parse_inner_recursive.d create mode 100644 spec/test_parse_inner_shared.c create mode 100644 spec/test_parse_inner_shared.d diff --git a/CHANGELOG.md b/CHANGELOG.md index 09168d7..7f7fe62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Unreleased + +### New Features + +- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow + tokens. These behave the same as `p_parse_XXX()` by parsing starting at the + given start rule, but instead of expecting the rest of the input to match + the start rule they allow specifying a set of tokens that may follow the + start rule. + ## v4.7.0 ### New Features diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 32b4fca..eb6e647 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -1008,8 +1008,17 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token) * * @param context * Lexer/parser context structure. - * @start_state_id + * @param start_state_id * ID of the state in which to start. + * @param start_rule_set_id + * Rule set ID for the requested start rule. Only used when + * @p follow_tokens is non-NULL, to gate follow-token shift success. + * @param follow_tokens + * Optional array of caller-provided follow tokens (tokens expected to + * appear immediately after the start rule in some outer context). Used to + * drive the "parse inner" retry logic. May be NULL for a standard parse. + * @param n_follow_tokens + * Number of entries in @p follow_tokens. * * @retval P_SUCCESS * The parser successfully matched the input text. The parse result value @@ -1022,12 +1031,15 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token) * @reval P_UNEXPECTED_INPUT * Input text does not match any lexer pattern. */ -static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id) +static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id, + size_t start_rule_set_id, + <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens) { <%= @grammar.prefix %>token_info_t token_info; <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID; state_values_stack_t statevalues; size_t reduced_rule_set = INVALID_ID; + size_t last_shifted_rule_set_id = INVALID_ID; <% if @grammar.tree %> void * reduced_parser_node; <% else %> @@ -1051,6 +1063,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start } token = token_info.token; } + /* For a "parse inner" operation, determine once per iteration whether + * the current token is a member of the caller-provided follow token + * set. Used by both the shift-side and reduce-side retries below. */ + bool token_is_follow = false; + for (size_t i = 0u; i < n_follow_tokens; i++) + { + if (token == follow_tokens[i]) + { + token_is_follow = true; + break; + } + } size_t shift_state = INVALID_ID; if (reduced_rule_set != INVALID_ID) { @@ -1070,10 +1094,36 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start result = P_SUCCESS; break; } + if ((shift_state == INVALID_ID) && token_is_follow) + { + /* For a "parse inner" operation, if the incoming token is one + * of the caller's follow tokens, retry the shift as + * TOKEN___EOF. Only consider the parse complete if the reduced + * start rule is the only thing on the parse stack (i.e. the + * initial state plus a single shifted start rule set entry). */ + size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF); + if ((retry_shift_state != INVALID_ID) && + (statevalues.length == 2u) && + (last_shifted_rule_set_id == start_rule_set_id)) + { + /* Successful parse via follow token. */ +<% if @grammar.tree %> + context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node; +<% else %> + context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue; +<% end %> + result = P_SUCCESS; + break; + } + } } if (shift_state != INVALID_ID) { - /* We have something to shift. */ + /* We have something to shift. Track the last shifted rule set ID + * (INVALID_ID if we just shifted a token) so the follow-token + * shift retry can gate success on the reduced start rule being the + * only thing on top of the initial state. */ + last_shifted_rule_set_id = reduced_rule_set; state_values_stack_push(&statevalues); state_value_t * new_state_info = state_values_stack_index(&statevalues, -1); new_state_info->state_id = shift_state; @@ -1120,6 +1170,15 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start } size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token); + if ((reduce_index == INVALID_ID) && token_is_follow) + { + /* For a "parse inner" operation, if the incoming token is one of + * the caller's follow tokens, retry the reduce lookup as + * TOKEN___EOF. Whatever reduce_index results (if any) is used + * regardless of which rule set it reduces to; this allows chains + * of reductions leading up to the start rule. */ + reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF); + } if (reduce_index != INVALID_ID) { /* We have something to reduce. */ @@ -1219,14 +1278,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context) { - return parse_from(context, 0u); + return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u); } <% @grammar.start_rules.each_with_index do |start_rule, i| %> size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context) { - return parse_from(context, <%= i %>u); + return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u); +} + +size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context, + <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens) +{ + return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens); } <% end %> diff --git a/assets/parser.d.erb b/assets/parser.d.erb index c2d4aba..d22ba6b 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -1079,8 +1079,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token * * @param context * Lexer/parser context structure. - * @start_state_id + * @param start_state_id * ID of the state in which to start. + * @param start_rule_set_id + * Rule set ID for the requested start rule. Only used when + * @p follow_tokens is non-empty, to gate follow-token shift success. + * @param follow_tokens + * Optional slice of caller-provided follow tokens (tokens expected to + * appear immediately after the start rule in some outer context). Used to + * drive the "parse inner" retry logic. May be null/empty for a standard + * parse. * * @retval P_SUCCESS * The parser successfully matched the input text. The parse result value @@ -1093,13 +1101,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token * @reval P_UNEXPECTED_INPUT * Input text does not match any lexer pattern. */ -private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id) +private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id, + size_t start_rule_set_id, + const(<%= @grammar.prefix %>token_t)[] follow_tokens) { <%= @grammar.prefix %>token_info_t token_info; <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID; state_value_t[] statevalues = new state_value_t[](1); statevalues[0].state_id = start_state_id; size_t reduced_rule_set = INVALID_ID; + size_t last_shifted_rule_set_id = INVALID_ID; <% if @grammar.tree %> void * reduced_parser_node; <% else %> @@ -1118,6 +1129,18 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star } token = token_info.token; } + /* For a "parse inner" operation, determine once per iteration whether + * the current token is a member of the caller-provided follow token + * set. Used by both the shift-side and reduce-side retries below. */ + bool token_is_follow = false; + foreach (eof_token; follow_tokens) + { + if (token == eof_token) + { + token_is_follow = true; + break; + } + } size_t shift_state = INVALID_ID; if (reduced_rule_set != INVALID_ID) { @@ -1136,10 +1159,35 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star <% end %> return P_SUCCESS; } + if ((shift_state == INVALID_ID) && token_is_follow) + { + /* For a "parse inner" operation, if the incoming token is one + * of the caller's follow tokens, retry the shift as + * TOKEN___EOF. Only consider the parse complete if the reduced + * start rule is the only thing on the parse stack (i.e. the + * initial state plus a single shifted start rule set entry). */ + size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF); + if ((retry_shift_state != INVALID_ID) && + (statevalues.length == 2u) && + (last_shifted_rule_set_id == start_rule_set_id)) + { + /* Successful parse via follow token. */ +<% if @grammar.tree %> + context.parse_result = statevalues[$-1].tree_node; +<% else %> + context.parse_result = statevalues[$-1].pvalue; +<% end %> + return P_SUCCESS; + } + } } if (shift_state != INVALID_ID) { - /* We have something to shift. */ + /* We have something to shift. Track the last shifted rule set ID + * (INVALID_ID if we just shifted a token) so the follow-token + * shift retry can gate success on the reduced start rule being the + * only thing on top of the initial state. */ + last_shifted_rule_set_id = reduced_rule_set; statevalues ~= state_value_t(shift_state); if (reduced_rule_set == INVALID_ID) { @@ -1173,6 +1221,15 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star } size_t reduce_index = check_reduce(statevalues[$-1].state_id, token); + if ((reduce_index == INVALID_ID) && token_is_follow) + { + /* For a "parse inner" operation, if the incoming token is one of + * the caller's follow tokens, retry the reduce lookup as + * TOKEN___EOF. Whatever reduce_index results (if any) is used + * regardless of which rule set it reduces to; this allows chains + * of reductions leading up to the start rule. */ + reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF); + } if (reduce_index != INVALID_ID) { /* We have something to reduce. */ @@ -1271,14 +1328,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context) { - return parse_from(context, 0u); + return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null); } <% @grammar.start_rules.each_with_index do |start_rule, i| %> public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context) { - return parse_from(context, <%= i %>u); + return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null); +} + +public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context, + const(<%= @grammar.prefix %>token_t)[] follow_tokens) +{ + return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens); } <% end %> diff --git a/assets/parser.h.erb b/assets/parser.h.erb index b54ca81..8f9dfca 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -212,6 +212,8 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context); <% @grammar.start_rules.each_with_index do |start_rule, i| %> size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context); +size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context, + <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens); <% end %> <% if @grammar.tree %> diff --git a/doc/user_guide.md b/doc/user_guide.md index a5afb82..2858871 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -1367,6 +1367,36 @@ size_t result = p_parse_Statement(context); In this case, the parser will start parsing with the `Statement` rule. +### `p_parse_inner_XXX` + +For each start rule, a `p_parse_inner_XXX()` function is also generated. +This variant of the parser entry point accepts a caller-provided array of +"follow tokens" — tokens the caller allows to appear immediately after the +start rule in some outer grammar context. +It is useful when embedding a Propane-generated sub-parser within an outer +parser and the outer parser knows which tokens naturally terminate the +sub-parse. + +For C targets, the signature is (example for a rule named `Statement`): + +``` +size_t p_parse_inner_Statement(p_context_t * context, + p_token_t const * follow_tokens, size_t n_follow_tokens); +``` + +Passing a `NULL` pointer (or a count of zero) makes the function behave +identically to `p_parse_Statement()`. + +For D targets, the signature accepts a slice: + +``` +size_t p_parse_inner_Statement(p_context_t * context, + const(p_token_t)[] follow_tokens); +``` + +Passing `null` for the slice makes the function behave identically to +`p_parse_Statement()`. + ### `p_position_valid` The `p_position_valid()` function is only generated for C targets. diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 26076fe..edd9f15 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1589,6 +1589,56 @@ EOF expect(results.status).to eq 0 end + it "supports parse_inner APIs that treat provided tokens as follow tokens" do + write_grammar <> +token b << $$ = 2; >> +Start -> Y << $$ = $1; >> +Y -> a << $$ = $1; >> +EOF + run_propane(language: language) + compile("spec/test_parse_inner.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + + it "parse_inner APIs block success when the outer rule is unfinished" do + write_grammar <> +token b << $$ = 2; >> +token c << $$ = 3; >> +Start -> a Start b << $$ = $2; >> +Start -> c << $$ = $1; >> +EOF + run_propane(language: language) + compile("spec/test_parse_inner_recursive.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + + it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do + write_grammar < R1 a; +Start -> R2 b; +R1 -> a b << $$ = 11; >> +R2 -> a b << $$ = 22; >> +EOF + run_propane(language: language) + compile("spec/test_parse_inner_shared.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + it "allows multiple starting rules in tree mode" do write_grammar < +#include +#include "testutils.h" + +int main() +{ + /* Grammar (chain reduce): + * Start -> Y << $$ = $1; >> + * Y -> a << $$ = $1; >> + * token a << $$ = 1; >> + * + * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF, + * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with + * `b` as a follow token should succeed via the reduce-side retry chain + * (Y then Start) followed by the shift-side retry hitting $EOF at the + * final state. */ + + /* Standard parse succeeds on complete input. */ + char const * input = "a"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_SUCCESS); + assert_eq(1u, (size_t)p_result_Start(context)); + p_context_delete(context); + + /* Standard parse fails when there's an unexpected trailing token. */ + input = "ab"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN); + p_context_delete(context); + + /* parse_inner succeeds via a chain of reduce retries (Y, then Start), + * followed by the shift-side retry hitting $EOF at the final state. */ + { + input = "ab"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_b }; + assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(1u, (size_t)p_result_Start(context)); + p_context_delete(context); + } + + /* parse_inner with an empty (NULL) follow-token vector behaves like a + * standard parse. */ + input = "ab"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN); + p_context_delete(context); + + /* parse_inner behaves like a standard parse when the input matches the + * grammar fully. */ + input = "a"; + context = p_context_new((uint8_t const *)input, strlen(input)); + { + p_token_t follow_tokens[] = { TOKEN_b }; + assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(1u, (size_t)p_result_Start(context)); + } + p_context_delete(context); + + /* parse_inner with a non-matching follow token still fails. The grammar + * can't consume `b` and it isn't listed as a follow token, so the retries + * do not fire. */ + { + input = "ab"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN___EOF }; + assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN); + p_context_delete(context); + } + + return 0; +} diff --git a/spec/test_parse_inner.d b/spec/test_parse_inner.d new file mode 100644 index 0000000..7dc8ac0 --- /dev/null +++ b/spec/test_parse_inner.d @@ -0,0 +1,51 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + /* See test_parse_inner.c for details on the grammar and cases. */ + + /* Standard parse succeeds on complete input. */ + string input = "a"; + p_context_t * context = p_context_new(input); + assert(p_parse_Start(context) == P_SUCCESS); + assert(p_result_Start(context) == 1); + + /* Standard parse fails when there's an unexpected trailing token. */ + input = "ab"; + context = p_context_new(input); + assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN); + + /* parse_inner succeeds via a chain of reduce retries (Y, then Start), + * followed by the shift-side retry hitting $EOF at the final state. */ + input = "ab"; + context = p_context_new(input); + p_token_t[] follow_tokens_b = [TOKEN_b]; + assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS); + assert(p_result_Start(context) == 1); + + /* parse_inner with a null follow-token slice behaves like a standard + * parse. */ + input = "ab"; + context = p_context_new(input); + assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN); + + /* parse_inner behaves like a standard parse when the input matches the + * grammar fully. */ + input = "a"; + context = p_context_new(input); + assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS); + assert(p_result_Start(context) == 1); + + /* parse_inner with a non-matching follow token still fails. */ + input = "ab"; + context = p_context_new(input); + p_token_t[] follow_tokens_eof = [TOKEN___EOF]; + assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN); +} diff --git a/spec/test_parse_inner_recursive.c b/spec/test_parse_inner_recursive.c new file mode 100644 index 0000000..26dde26 --- /dev/null +++ b/spec/test_parse_inner_recursive.c @@ -0,0 +1,77 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +int main() +{ + /* Grammar (recursive): + * Start -> a Start b << $$ = $2; >> + * Start -> c << $$ = $1; >> + * token a << $$ = 1; >> + * token b << $$ = 2; >> + * token c << $$ = 3; >> + * + * Here `Start` can appear in the middle of another `Start` rule, so the + * inner-parse follow-token success must be blocked whenever an unfinished + * outer `Start -> a Start b` remains on the parse stack (i.e. the parse + * stack contains more than just the initial state and the reduced start + * rule set). */ + + /* Standard parse of `c` succeeds. */ + char const * input = "c"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_SUCCESS); + assert_eq(3u, (size_t)p_result_Start(context)); + p_context_delete(context); + + /* Standard parse of `acb` succeeds (full outer rule). */ + input = "acb"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_SUCCESS); + assert_eq(3u, (size_t)p_result_Start(context)); + p_context_delete(context); + + /* Standard parse of `ac` fails (`b` missing). */ + input = "ac"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN); + p_context_delete(context); + + /* parse_inner with `ac` and follow token `b` also fails: even though the + * inner `Start -> c` reduces and `Start` is shifted, the outer + * `Start -> a Start . b` is still on the stack (stack length > 2), so the + * "reduced start rule is the only thing on the parse stack" invariant + * blocks the shift-side follow-token success. */ + { + input = "ac"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF }; + assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN); + p_context_delete(context); + } + + /* parse_inner with `acb` (complete outer rule) succeeds via the standard + * path. */ + { + input = "acb"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_b }; + assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(3u, (size_t)p_result_Start(context)); + p_context_delete(context); + } + + /* parse_inner with just `c` succeeds via the standard path even when a + * follow-token vector is supplied. */ + { + input = "c"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_b }; + assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(3u, (size_t)p_result_Start(context)); + p_context_delete(context); + } + + return 0; +} diff --git a/spec/test_parse_inner_recursive.d b/spec/test_parse_inner_recursive.d new file mode 100644 index 0000000..fecda60 --- /dev/null +++ b/spec/test_parse_inner_recursive.d @@ -0,0 +1,49 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + /* See test_parse_inner_recursive.c for details on the grammar. */ + + /* Standard parse of `c` succeeds. */ + string input = "c"; + p_context_t * context = p_context_new(input); + assert(p_parse_Start(context) == P_SUCCESS); + assert(p_result_Start(context) == 3); + + /* Standard parse of `acb` succeeds. */ + input = "acb"; + context = p_context_new(input); + assert(p_parse_Start(context) == P_SUCCESS); + assert(p_result_Start(context) == 3); + + /* Standard parse of `ac` fails. */ + input = "ac"; + context = p_context_new(input); + assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN); + + /* parse_inner with `ac` fails: outer rule still on the stack. */ + input = "ac"; + context = p_context_new(input); + p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF]; + assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN); + + /* parse_inner with `acb` succeeds via the standard path. */ + input = "acb"; + context = p_context_new(input); + p_token_t[] follow_tokens_b = [TOKEN_b]; + assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS); + assert(p_result_Start(context) == 3); + + /* parse_inner with just `c` succeeds via the standard path. */ + input = "c"; + context = p_context_new(input); + assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS); + assert(p_result_Start(context) == 3); +} diff --git a/spec/test_parse_inner_shared.c b/spec/test_parse_inner_shared.c new file mode 100644 index 0000000..67329d4 --- /dev/null +++ b/spec/test_parse_inner_shared.c @@ -0,0 +1,83 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +int main() +{ + /* Grammar: + * start Start; + * start R1; + * Start -> R1 a; + * Start -> R2 b; + * R1 -> a b << $$ = 11; >> + * R2 -> a b << $$ = 22; >> + * token a; token b; + * + * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within + * parse_Start, the generated parser differentiates the reduce by + * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2 + * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional + * on any lookahead. This test exercises p_parse_inner_R1() to confirm + * that reductions to R1 succeed even when the incoming follow token is + * not the natural lookahead used by parse_Start's disambiguation. */ + + /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the + * shared "a b" state. */ + char const * input = "aba"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_SUCCESS); + p_context_delete(context); + + input = "abb"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_SUCCESS); + p_context_delete(context); + + /* Standard parse of R1 succeeds on "ab". */ + input = "ab"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_R1(context) == P_SUCCESS); + assert_eq(11u, (size_t)p_result_R1(context)); + p_context_delete(context); + + /* Standard parse of R1 fails on "abb" (unexpected trailing token). */ + input = "abb"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN); + p_context_delete(context); + + /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead + * that parse_Start uses to select R2 over R1 in the ambiguous state, from + * R1's start state the reduce to R1 is unconditional, and the follow- + * token shift retry at the R1-accepting state completes the parse. */ + { + input = "abb"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_b }; + assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(11u, (size_t)p_result_R1(context)); + p_context_delete(context); + } + + /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token + * parse_Start uses to select R1, and it works here as a follow token + * too. */ + { + input = "aba"; + context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_a }; + assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(11u, (size_t)p_result_R1(context)); + p_context_delete(context); + } + + /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */ + input = "ab"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS); + assert_eq(11u, (size_t)p_result_R1(context)); + p_context_delete(context); + + return 0; +} diff --git a/spec/test_parse_inner_shared.d b/spec/test_parse_inner_shared.d new file mode 100644 index 0000000..fc6870a --- /dev/null +++ b/spec/test_parse_inner_shared.d @@ -0,0 +1,56 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + /* See test_parse_inner_shared.c for details on the grammar. */ + + /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */ + string input = "aba"; + p_context_t * context = p_context_new(input); + assert(p_parse_Start(context) == P_SUCCESS); + + input = "abb"; + context = p_context_new(input); + assert(p_parse_Start(context) == P_SUCCESS); + + /* Standard parse of R1 succeeds on "ab". */ + input = "ab"; + context = p_context_new(input); + assert(p_parse_R1(context) == P_SUCCESS); + assert(p_result_R1(context) == 11); + + /* Standard parse of R1 fails on "abb". */ + input = "abb"; + context = p_context_new(input); + assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN); + + /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that + * parse_Start would use to select R2 over R1, but from R1's own start + * state R1 reduces unconditionally, and the follow-token shift retry at + * the R1-accepting state completes the parse. */ + input = "abb"; + context = p_context_new(input); + p_token_t[] follow_tokens_b = [TOKEN_b]; + assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS); + assert(p_result_R1(context) == 11); + + /* parse_inner_R1("aba", [a]) also succeeds. */ + input = "aba"; + context = p_context_new(input); + p_token_t[] follow_tokens_a = [TOKEN_a]; + assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS); + assert(p_result_R1(context) == 11); + + /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */ + input = "ab"; + context = p_context_new(input); + assert(p_parse_inner_R1(context, null) == P_SUCCESS); + assert(p_result_R1(context) == 11); +}