josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

p_parse_inner: do not consume the follow token

Browse Source
This commit is contained in:
Josh Holtrop 2026-07-11 21:16:59 -04:00
parent 946eb4eef7
commit 2f9a29f7c5
5 changed files with 59 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
assets/parser.c.erb
View File

@ -1106,7 +1106,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
(statevalues.length == 2u) &&
(last_shifted_rule_set_id == start_rule_set_id))
{
/* Successful parse via follow token. */
/* Successful parse via follow token. Rewind the input
* position so that the follow token is not consumed from
* the input stream and remains available for a subsequent
* call to <%= @grammar.prefix %>lex() or a
* <%= @grammar.prefix %>parse*() function. */
context->input_index -= token_info.length;
context->text_position = token_info.position;
<% if @grammar.tree %>
context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
<% else %>

8
assets/parser.d.erb
View File

@ -1171,7 +1171,13 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
(statevalues.length == 2u) &&
(last_shifted_rule_set_id == start_rule_set_id))
{
/* Successful parse via follow token. */
/* Successful parse via follow token. Rewind the input
* position so that the follow token is not consumed from
* the input stream and remains available for a subsequent
* call to <%= @grammar.prefix %>lex() or a
* <%= @grammar.prefix %>parse*() function. */
context.input_index -= token_info.length;
context.text_position = token_info.position;
<% if @grammar.tree %>
context.parse_result = statevalues[$-1].tree_node;
<% else %>

6
doc/user_guide.md
View File

@ -1397,6 +1397,12 @@ size_t p_parse_inner_Statement(p_context_t * context,
Passing `null` for the slice makes the function behave identically to
`p_parse_Statement()`.
When the parse is completed via a non-EOF follow token, that follow token is
**not** consumed from the input stream.
The parser rewinds the input index and text position to the start of the follow
token so that a subsequent call to `p_lex()` or another parse function sees the
same token.
### `p_position_valid`
The `p_position_valid()` function is only generated for C targets.

23
spec/test_parse_inner_shared.c
View File

@ -50,13 +50,26 @@ int main()
/* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
* that parse_Start uses to select R2 over R1 in the ambiguous state, from
* R1's start state the reduce to R1 is unconditional, and the follow-
* token shift retry at the R1-accepting state completes the parse. */
* token shift retry at the R1-accepting state completes the parse.
*
* The follow token that completed the parse must not be consumed from
* the input: p_position() should point to the follow token, and a
* subsequent p_lex() should return it. */
{
input = "abb";
context = p_context_new((uint8_t const *)input, strlen(input));
p_token_t follow_tokens[] = { TOKEN_b };
assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
assert_eq(11u, (size_t)p_result_R1(context));
/* Follow token `b` is at column 3 (1-based). */
p_position_t pos = p_position(context);
assert_eq(1u, (size_t)pos.row);
assert_eq(3u, (size_t)pos.col);
p_token_info_t token_info;
assert(p_lex(context, &token_info) == P_SUCCESS);
assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
assert_eq(1u, (size_t)token_info.position.row);
assert_eq(3u, (size_t)token_info.position.col);
p_context_delete(context);
}
@ -69,6 +82,14 @@ int main()
p_token_t follow_tokens[] = { TOKEN_a };
assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
assert_eq(11u, (size_t)p_result_R1(context));
/* Follow token `a` is at column 3 (1-based) and remains in the
* input. */
p_position_t pos = p_position(context);
assert_eq(1u, (size_t)pos.row);
assert_eq(3u, (size_t)pos.col);
p_token_info_t token_info;
assert(p_lex(context, &token_info) == P_SUCCESS);
assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
p_context_delete(context);
}

18
spec/test_parse_inner_shared.d
View File

@ -34,12 +34,23 @@ unittest
/* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
* parse_Start would use to select R2 over R1, but from R1's own start
* state R1 reduces unconditionally, and the follow-token shift retry at
* the R1-accepting state completes the parse. */
* the R1-accepting state completes the parse.
*
* The follow token that completed the parse must not be consumed: a
* subsequent p_lex() should return it. */
input = "abb";
context = p_context_new(input);
p_token_t[] follow_tokens_b = [TOKEN_b];
assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
assert(p_result_R1(context) == 11);
p_position_t pos = p_position(context);
assert(pos.row == 1);
assert(pos.col == 3);
p_token_info_t token_info;
assert(p_lex(context, &token_info) == P_SUCCESS);
assert(token_info.token == TOKEN_b);
assert(token_info.position.row == 1);
assert(token_info.position.col == 3);
/* parse_inner_R1("aba", [a]) also succeeds. */
input = "aba";
@ -47,6 +58,11 @@ unittest
p_token_t[] follow_tokens_a = [TOKEN_a];
assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
assert(p_result_R1(context) == 11);
pos = p_position(context);
assert(pos.row == 1);
assert(pos.col == 3);
assert(p_lex(context, &token_info) == P_SUCCESS);
assert(token_info.token == TOKEN_a);
/* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
input = "ab";