diff --git a/assets/parser.c.erb b/assets/parser.c.erb index eb6e647..a27d686 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -1106,7 +1106,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start (statevalues.length == 2u) && (last_shifted_rule_set_id == start_rule_set_id)) { - /* Successful parse via follow token. */ + /* Successful parse via follow token. Rewind the input + * position so that the follow token is not consumed from + * the input stream and remains available for a subsequent + * call to <%= @grammar.prefix %>lex() or a + * <%= @grammar.prefix %>parse*() function. */ + context->input_index -= token_info.length; + context->text_position = token_info.position; <% if @grammar.tree %> context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node; <% else %> diff --git a/assets/parser.d.erb b/assets/parser.d.erb index d22ba6b..a33e6bf 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -1171,7 +1171,13 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star (statevalues.length == 2u) && (last_shifted_rule_set_id == start_rule_set_id)) { - /* Successful parse via follow token. */ + /* Successful parse via follow token. Rewind the input + * position so that the follow token is not consumed from + * the input stream and remains available for a subsequent + * call to <%= @grammar.prefix %>lex() or a + * <%= @grammar.prefix %>parse*() function. */ + context.input_index -= token_info.length; + context.text_position = token_info.position; <% if @grammar.tree %> context.parse_result = statevalues[$-1].tree_node; <% else %> diff --git a/doc/user_guide.md b/doc/user_guide.md index 2858871..0477558 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -1397,6 +1397,12 @@ size_t p_parse_inner_Statement(p_context_t * context, Passing `null` for the slice makes the function behave identically to `p_parse_Statement()`. +When the parse is completed via a non-EOF follow token, that follow token is +**not** consumed from the input stream. +The parser rewinds the input index and text position to the start of the follow +token so that a subsequent call to `p_lex()` or another parse function sees the +same token. + ### `p_position_valid` The `p_position_valid()` function is only generated for C targets. diff --git a/spec/test_parse_inner_shared.c b/spec/test_parse_inner_shared.c index 67329d4..072f516 100644 --- a/spec/test_parse_inner_shared.c +++ b/spec/test_parse_inner_shared.c @@ -50,13 +50,26 @@ int main() /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead * that parse_Start uses to select R2 over R1 in the ambiguous state, from * R1's start state the reduce to R1 is unconditional, and the follow- - * token shift retry at the R1-accepting state completes the parse. */ + * token shift retry at the R1-accepting state completes the parse. + * + * The follow token that completed the parse must not be consumed from + * the input: p_position() should point to the follow token, and a + * subsequent p_lex() should return it. */ { input = "abb"; context = p_context_new((uint8_t const *)input, strlen(input)); p_token_t follow_tokens[] = { TOKEN_b }; assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS); assert_eq(11u, (size_t)p_result_R1(context)); + /* Follow token `b` is at column 3 (1-based). */ + p_position_t pos = p_position(context); + assert_eq(1u, (size_t)pos.row); + assert_eq(3u, (size_t)pos.col); + p_token_info_t token_info; + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_b, (size_t)token_info.token); + assert_eq(1u, (size_t)token_info.position.row); + assert_eq(3u, (size_t)token_info.position.col); p_context_delete(context); } @@ -69,6 +82,14 @@ int main() p_token_t follow_tokens[] = { TOKEN_a }; assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS); assert_eq(11u, (size_t)p_result_R1(context)); + /* Follow token `a` is at column 3 (1-based) and remains in the + * input. */ + p_position_t pos = p_position(context); + assert_eq(1u, (size_t)pos.row); + assert_eq(3u, (size_t)pos.col); + p_token_info_t token_info; + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_a, (size_t)token_info.token); p_context_delete(context); } diff --git a/spec/test_parse_inner_shared.d b/spec/test_parse_inner_shared.d index fc6870a..60f2208 100644 --- a/spec/test_parse_inner_shared.d +++ b/spec/test_parse_inner_shared.d @@ -34,12 +34,23 @@ unittest /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that * parse_Start would use to select R2 over R1, but from R1's own start * state R1 reduces unconditionally, and the follow-token shift retry at - * the R1-accepting state completes the parse. */ + * the R1-accepting state completes the parse. + * + * The follow token that completed the parse must not be consumed: a + * subsequent p_lex() should return it. */ input = "abb"; context = p_context_new(input); p_token_t[] follow_tokens_b = [TOKEN_b]; assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS); assert(p_result_R1(context) == 11); + p_position_t pos = p_position(context); + assert(pos.row == 1); + assert(pos.col == 3); + p_token_info_t token_info; + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_b); + assert(token_info.position.row == 1); + assert(token_info.position.col == 3); /* parse_inner_R1("aba", [a]) also succeeds. */ input = "aba"; @@ -47,6 +58,11 @@ unittest p_token_t[] follow_tokens_a = [TOKEN_a]; assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS); assert(p_result_R1(context) == 11); + pos = p_position(context); + assert(pos.row == 1); + assert(pos.col == 3); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_a); /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */ input = "ab";