Add lex_fn documentation about custom lex function contract

CHANGELOG.md

@@ -1,3 +1,17 @@
+## Unreleased
+
+### New Features
+
+- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow
+  tokens. These behave the same as `p_parse_XXX()` by parsing starting at the
+  given start rule, but instead of expecting the rest of the input to match
+  the start rule they allow specifying a set of tokens that may follow the
+  start rule.
+- Add `p_set_position()` API to set the current text position stored in the
+  context. Useful for setting the initial text position to something other
+  than `(1, 1)` for a nested parse operation.
+- Add `p_input_index()` API to get the current input text byte offset.
+
 ## v4.7.0
 
 ### New Features

assets/parser.c.erb

@@ -1008,8 +1008,17 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-NULL, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional array of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be NULL for a standard parse.
+ * @param n_follow_tokens
+ *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1022,12 +1031,15 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
@@ -1051,6 +1063,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        for (size_t i = 0u; i < n_follow_tokens; i++)
+        {
+            if (token == follow_tokens[i])
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1070,10 +1094,42 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 result = P_SUCCESS;
                 break;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context->input_index -= token_info.length;
+                    context->text_position = token_info.position;
+<% if @grammar.tree %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+<% else %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
+                    result = P_SUCCESS;
+                    break;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             state_values_stack_push(&statevalues);
             state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
             new_state_info->state_id = shift_state;
@@ -1120,6 +1176,15 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1219,14 +1284,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
+}
+
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
 }
 <% end %>
 
@@ -1275,6 +1346,37 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
     return context->text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context->text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context->input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.d.erb

@@ -1079,8 +1079,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1093,13 +1101,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
@@ -1118,6 +1129,18 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        foreach (eof_token; follow_tokens)
+        {
+            if (token == eof_token)
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1136,10 +1159,41 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% end %>
                 return P_SUCCESS;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context.input_index -= token_info.length;
+                    context.text_position = token_info.position;
+<% if @grammar.tree %>
+                    context.parse_result = statevalues[$-1].tree_node;
+<% else %>
+                    context.parse_result = statevalues[$-1].pvalue;
+<% end %>
+                    return P_SUCCESS;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             statevalues ~= state_value_t(shift_state);
             if (reduced_rule_set == INVALID_ID)
             {
@@ -1173,6 +1227,15 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
         }
 
         size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1271,14 +1334,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 
 public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
+}
+
+public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
 }
 <% end %>
 
@@ -1356,6 +1425,37 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
     return context.text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context.text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context.input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -212,6 +212,8 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
 <%   end %>
 
 <% if @grammar.tree %>
@@ -235,6 +237,10 @@ void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefi
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
+
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
+
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -453,6 +453,41 @@ enabled.
 When tree generation is enabled, the `pvalue` field can be set to an instance
 of whatever type the user has defined as the `ptype` type.
 
+The custom lex function returns a result code to the parser.
+Returning `P_SUCCESS` indicates that the function has produced a token in the
+`token` output field and the parse should proceed.
+Returning any other result code stops the parse immediately; the parser
+propagates that code out of `p_parse()` (and the `p_parse_XXX()` and
+`p_parse_inner_XXX()` functions) unchanged.
+This allows a custom lex function to surface an error condition, for example by
+propagating a `P_UNEXPECTED_TOKEN`, `P_UNEXPECTED_INPUT`, or `P_DECODE_ERROR`
+result from a nested `p_parse_inner_XXX()` or `p_lex()` call.
+
+Observe the following contract when returning a result code other than
+`P_SUCCESS`:
+
+* Do not return `P_DROP`.
+  This code is an internal lexer signal and is never returned to the parser by
+  the generated `p_lex()` function.
+  Returning it from a custom lex function would be reported as a spurious parse
+  failure. If drop functionality is required, the custom lex function should
+  loop and return the next non-drop token.
+* Do not return `P_EOF` to signal the end of the input.
+  The end of input is communicated to the parser by returning `P_SUCCESS` with
+  the `token` field set to the end-of-input token (`TOKEN___EOF`), which is what
+  the generated `p_lex()` function does.
+  Returning `P_EOF` aborts the parse rather than allowing it to complete.
+* The `p_token()` and `p_position()` accessors are populated by the parser only
+  when the parser itself detects an unexpected token.
+  When a custom lex function returns an error code, `p_token()` is not updated
+  and may not reflect a meaningful token, and `p_position()` reflects the
+  lexer's current text position rather than a parser-identified error location.
+  A custom lex function that wants a specific reported position can set it with
+  `p_set_position()` before returning.
+* Returning `P_USER_TERMINATED` does not populate the user terminate code
+  returned by `p_user_terminate_code()`. This user terminate code is normally
+  populated by the `$terminate()` function in the user code block.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -1367,6 +1402,42 @@ size_t result = p_parse_Statement(context);
 
 In this case, the parser will start parsing with the `Statement` rule.
 
+### `p_parse_inner_XXX`
+
+For each start rule, a `p_parse_inner_XXX()` function is also generated.
+This variant of the parser entry point accepts a caller-provided array of
+"follow tokens" — tokens the caller allows to appear immediately after the
+start rule in some outer grammar context.
+It is useful when embedding a Propane-generated sub-parser within an outer
+parser and the outer parser knows which tokens naturally terminate the
+sub-parse.
+
+For C targets, the signature is (example for a rule named `Statement`):
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    p_token_t const * follow_tokens, size_t n_follow_tokens);
+```
+
+Passing a `NULL` pointer (or a count of zero) makes the function behave
+identically to `p_parse_Statement()`.
+
+For D targets, the signature accepts a slice:
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    const(p_token_t)[] follow_tokens);
+```
+
+Passing `null` for the slice makes the function behave identically to
+`p_parse_Statement()`.
+
+When the parse is completed via a non-EOF follow token, that follow token is
+**not** consumed from the input stream.
+The parser rewinds the input index and text position to the start of the follow
+token so that a subsequent call to `p_lex()` or another parse function sees the
+same token.
+
 ### `p_position_valid`
 
 The `p_position_valid()` function is only generated for C targets.
@@ -1438,6 +1509,41 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 }
 ```
 
+### `p_set_position`
+
+The `p_set_position()` function sets the current text position stored in the
+context.
+This is useful when performing a nested parse operation over a slice of a
+larger enclosing document: by setting the initial text position to something
+other than the default `(1, 1)`, any token positions or error positions
+subsequently reported by the lexer/parser are relative to the enclosing
+document rather than the slice.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_position_t start = { .row = 5, .col = 20 };
+p_set_position(context, start);
+p_parse(context);
+```
+
+### `p_input_index`
+
+The `p_input_index()` function returns the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This is useful for slicing out the remaining input after a partial parse,
+or for locating tokens in the original input buffer.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_parse_inner_Statement(context, follow_tokens, n_follow_tokens);
+size_t offset = p_input_index(context);
+/* Remaining input starts at `input + offset`. */
+```
+
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user

spec/macros.c.propane

@@ -0,0 +1,25 @@
+<<
+#include <stdlib.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int v);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/macros.d.propane

@@ -0,0 +1,31 @@
+<<
+import test_macros;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+    int n = 0;
+    foreach (c; match)
+    {
+        n *= 10;
+        n += (c - '0');
+    }
+    $$ = n;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -0,0 +1,19 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.c.propane

@@ -0,0 +1,17 @@
+<<
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/parse_inner_nested_tree.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested_tree;
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/propane_spec.rb

@@ -938,6 +938,34 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows setting the text position via p_set_position()" do
+        write_grammar <<EOF
+token a;
+token b;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_set_position.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "exposes the current input byte offset via p_input_index()" do
+        write_grammar <<EOF
+drop /\\s+/;
+token a;
+token b;
+start Start;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_input_index.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows creating a JSON parser" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/json_parser.#{ext}.propane"))
@@ -1589,6 +1617,95 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "supports parse_inner APIs that treat provided tokens as follow tokens" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+Start -> Y << $$ = $1; >>
+Y -> a << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs block success when the outer rule is unfinished" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> a Start b << $$ = $2; >>
+Start -> c << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_recursive.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
+    write_grammar <<EOF
+ptype int;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b << $$ = 11; >>
+R2 -> a b << $$ = 22; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_shared.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work in tree generation mode with follow tokens" do
+    write_grammar <<EOF
+tree;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b;
+R2 -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "supports a reentrant nested parse driven from a custom lex function" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/parse_inner_nested.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_parse_inner_nested.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "tracks positions across a lex-function nested parse in tree mode" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/parse_inner_nested_tree.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_parse_inner_nested_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows multiple starting rules in tree mode" do
     write_grammar <<EOF
 tree;
@@ -2067,6 +2184,16 @@ StartR start: 1, 3
 StartR end: 1, 6
 EOF
       end
+
+      it "allows nested parses for macro expansions" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/macros.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_macros.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_input_index.c

@@ -0,0 +1,60 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   drop /\\s+/;
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_input_index() reports the parser/lexer's current byte
+     * offset into the input text. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert_eq(0u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        char const * input = "a b";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        /* The dropped space between `a` and `b` advances input_index too. */
+        assert_eq(3u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_input_index.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_input_index.c for details on the grammar and cases. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_input_index(context) == 0);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        string input = "a b";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_input_index(context) == 1);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 3);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+}

spec/test_macros.c

@@ -0,0 +1,117 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+#include <assert.h>
+#include <stddef.h>
+
+static p_context_t * context;
+size_t n_tokens;
+p_token_info_t token_infos[10];
+
+/* Capture the macro body tokens (everything up to the closing '}') into
+ * token_infos[]. Called from mylexfn() right after the definition's '{' has
+ * been lexed, so the input cursor is positioned at the first body token. */
+static void capture_macro_body(void)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert_eq(result, P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < sizeof(token_infos) / sizeof(token_infos[0]));
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info->token)
+        {
+        case TOKEN_macro:
+            /* Start of a macro definition: "macro macroname { ... }". */
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                /* Use of a macro: replay its captured body tokens instead of
+                 * returning the macroname to the parser. */
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            /* Definition name: pass through and keep waiting for '{'. */
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                /* Consume and store the macro body now, before the parser gets
+                 * a chance to read its lookahead token (which would otherwise
+                 * swallow the first body token). */
+                capture_macro_body();
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int nums[10];
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    char const * input =
+        "macro @m { 23 + 200 }\n"
+        "66 + 100\n"
+        "@m\n"
+        "33 + 55\n"
+        "@m\n";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(n_nums, 4);
+    assert_eq(nums[0], 166);
+    assert_eq(nums[1], 223);
+    assert_eq(nums[2], 88);
+    assert_eq(nums[3], 223);
+
+    return 0;
+}

spec/test_macros.d

@@ -0,0 +1,116 @@
+import testparser;
+import testutils;
+
+size_t n_tokens;
+p_token_info_t[10] token_infos;
+
+// Capture the macro body tokens (everything up to the closing '}') into
+// token_infos[]. Called from mylexfn() right after the definition's '{' has
+// been lexed, so the input cursor is positioned at the first body token.
+void capture_macro_body(p_context_t * context)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert(result == P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < token_infos.length);
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info.token)
+        {
+        case TOKEN_macro:
+            // Start of a macro definition: "macro macroname { ... }".
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                // Use of a macro: replay its captured body tokens instead of
+                // returning the macroname to the parser.
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            // Definition name: pass through and keep waiting for '{'.
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                // Consume and store the macro body now, before the parser gets
+                // a chance to read its lookahead token (which would otherwise
+                // swallow the first body token).
+                capture_macro_body(context);
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int[10] nums;
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "macro @m { 23 + 200 }\n" ~
+        "66 + 100\n" ~
+        "@m\n" ~
+        "33 + 55\n" ~
+        "@m\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert(n_nums == 4);
+    assert(nums[0] == 166);
+    assert(nums[1] == 223);
+    assert(nums[2] == 88);
+    assert(nums[3] == 223);
+}

spec/test_parse_inner.c

@@ -0,0 +1,73 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (chain reduce):
+     *   Start -> Y      << $$ = $1; >>
+     *   Y     -> a      << $$ = $1; >>
+     *   token a         << $$ = 1; >>
+     *
+     * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF,
+     * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with
+     * `b` as a follow token should succeed via the reduce-side retry chain
+     * (Y then Start) followed by the shift-side retry hitting $EOF at the
+     * final state. */
+
+    /* Standard parse succeeds on complete input. */
+    char const * input = "a";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(1u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with an empty (NULL) follow-token vector behaves like a
+     * standard parse. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    {
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+    }
+    p_context_delete(context);
+
+    /* parse_inner with a non-matching follow token still fails. The grammar
+     * can't consume `b` and it isn't listed as a follow token, so the retries
+     * do not fire. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner.c for details on the grammar and cases. */
+
+    /* Standard parse succeeds on complete input. */
+    string input = "a";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a null follow-token slice behaves like a standard
+     * parse. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a non-matching follow token still fails. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_eof = [TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN);
+}

spec/test_parse_inner_nested.c

@@ -0,0 +1,78 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (integer evaluator; parentheses handled by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/;
+ *   token num /\d+/          << ... atoi ... >>
+ *   Start -> Expr            << $$ = $1; >>
+ *   Expr  -> num             << $$ = $1; >>
+ *   Expr  -> Expr plus num   << $$ = $1 + $3; >>
+ *
+ * The tokens lparen and rparen appear in no grammar rule. Instead, when the
+ * lex function lexes a '(', it performs a nested parse (p_parse_inner_Start)
+ * of the parenthesized sub-expression -- reentrantly, while the outer parse is
+ * still suspended in this callback -- reads the computed value with
+ * p_result_Start, consumes the ')' that p_parse_inner deliberately left in the
+ * input, and hands a single synthesized num token carrying that value back to
+ * the outer parse. Nested groups recurse this process to arbitrary depth. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->pvalue.v_default = value;
+    }
+    return P_SUCCESS;
+}
+
+static int eval(char const * input)
+{
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5u, (size_t)eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3u, (size_t)eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14u, (size_t)eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37u, (size_t)eval("2 + (10 + (20 + 5))"));
+    assert_eq(15u, (size_t)eval("(1 + 2) + (3 + (4 + 5))"));
+
+    return 0;
+}

spec/test_parse_inner_nested.d

@@ -0,0 +1,64 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue.v_default = value;
+    }
+    return P_SUCCESS;
+}
+
+int eval(string input)
+{
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5, eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3, eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14, eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37, eval("2 + (10 + (20 + 5))"));
+    assert_eq(15, eval("(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested_tree.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (tree generation mode; parentheses handled by the lex function):
+ *   tree;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/; token num /\d+/;
+ *   Start -> Expr;
+ *   Expr  -> num;
+ *   Expr  -> Expr plus num;
+ *
+ * The same lexer-driven nested parse as test_parse_inner_nested, but in tree
+ * generation mode. Each "( ... )" group is parsed by a reentrant
+ * p_parse_inner_Start() call from the lex function; the resulting subtree is
+ * discarded and a single synthesized num token is handed to the outer parse.
+ * The synthesized token's position is set to span the whole group ('(' start
+ * through ')' end), so this verifies that positions survive the nested-parse
+ * boundary and land correctly in the outer tree. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info->position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start * inner = p_result_Start(context);
+        assert_not_null(inner);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq((size_t)(start_position.col + 1u), (size_t)inner->position.col);
+        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)inner->end_position.col);
+        p_tree_delete_Start(inner);
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->position = start_position;
+        out_token_info->end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    char const * input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start * tree = p_result(context);
+    assert_not_null(tree);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr * top = tree->pExpr;
+    assert_not_null(top);
+    assert_not_null(top->pExpr);
+    assert_not_null(top->pToken2);
+    assert_not_null(top->pToken3);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, (size_t)top->pToken2->position.row);
+    assert_eq(9u, (size_t)top->pToken2->position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, (size_t)top->pToken3->position.row);
+    assert_eq(11u, (size_t)top->pToken3->position.col);
+    assert_eq(1u, (size_t)top->pToken3->end_position.row);
+    assert_eq(17u, (size_t)top->pToken3->end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr * left = top->pExpr;
+    assert_not_null(left->pToken1);
+    assert_eq(1u, (size_t)left->pToken1->position.row);
+    assert_eq(1u, (size_t)left->pToken1->position.col);
+    assert_eq(1u, (size_t)left->pToken1->end_position.row);
+    assert_eq(7u, (size_t)left->pToken1->end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, (size_t)tree->position.col);
+    assert_eq(17u, (size_t)tree->end_position.col);
+
+    p_tree_delete(tree);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_nested_tree.d

@@ -0,0 +1,91 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested_tree.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info.position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start * inner = p_result_Start(context);
+        assert(inner !is null);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq(start_position.col + 1u, inner.position.col);
+        assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
+        p_tree_delete_Start(inner);
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    string input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start * tree = p_result(context);
+    assert(tree !is null);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr * top = tree.pExpr;
+    assert(top !is null);
+    assert(top.pExpr !is null);
+    assert(top.pToken2 !is null);
+    assert(top.pToken3 !is null);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, top.pToken2.position.row);
+    assert_eq(9u, top.pToken2.position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, top.pToken3.position.row);
+    assert_eq(11u, top.pToken3.position.col);
+    assert_eq(1u, top.pToken3.end_position.row);
+    assert_eq(17u, top.pToken3.end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr * left = top.pExpr;
+    assert(left.pToken1 !is null);
+    assert_eq(1u, left.pToken1.position.row);
+    assert_eq(1u, left.pToken1.position.col);
+    assert_eq(1u, left.pToken1.end_position.row);
+    assert_eq(7u, left.pToken1.end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, tree.position.col);
+    assert_eq(17u, tree.end_position.col);
+
+    p_tree_delete(tree);
+    p_context_delete(context);
+}

spec/test_parse_inner_recursive.c

@@ -0,0 +1,77 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (recursive):
+     *   Start -> a Start b   << $$ = $2; >>
+     *   Start -> c           << $$ = $1; >>
+     *   token a << $$ = 1; >>
+     *   token b << $$ = 2; >>
+     *   token c << $$ = 3; >>
+     *
+     * Here `Start` can appear in the middle of another `Start` rule, so the
+     * inner-parse follow-token success must be blocked whenever an unfinished
+     * outer `Start -> a Start b` remains on the parse stack (i.e. the parse
+     * stack contains more than just the initial state and the reduced start
+     * rule set). */
+
+    /* Standard parse of `c` succeeds. */
+    char const * input = "c";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `acb` succeeds (full outer rule). */
+    input = "acb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `ac` fails (`b` missing). */
+    input = "ac";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner with `ac` and follow token `b` also fails: even though the
+     * inner `Start -> c` reduces and `Start` is shifted, the outer
+     * `Start -> a Start . b` is still on the stack (stack length > 2), so the
+     * "reduced start rule is the only thing on the parse stack" invariant
+     * blocks the shift-side follow-token success. */
+    {
+        input = "ac";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    /* parse_inner with `acb` (complete outer rule) succeeds via the standard
+     * path. */
+    {
+        input = "acb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with just `c` succeeds via the standard path even when a
+     * follow-token vector is supplied. */
+    {
+        input = "c";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_recursive.d

@@ -0,0 +1,49 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_recursive.c for details on the grammar. */
+
+    /* Standard parse of `c` succeeds. */
+    string input = "c";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `acb` succeeds. */
+    input = "acb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `ac` fails. */
+    input = "ac";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `ac` fails: outer rule still on the stack. */
+    input = "ac";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `acb` succeeds via the standard path. */
+    input = "acb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* parse_inner with just `c` succeeds via the standard path. */
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+}

spec/test_parse_inner_shared.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar:
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b   << $$ = 11; >>
+     *   R2 -> a b   << $$ = 22; >>
+     *   token a; token b;
+     *
+     * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within
+     * parse_Start, the generated parser differentiates the reduce by
+     * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2
+     * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional
+     * on any lookahead. This test exercises p_parse_inner_R1() to confirm
+     * that reductions to R1 succeed even when the incoming follow token is
+     * not the natural lookahead used by parse_Start's disambiguation. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the
+     * shared "a b" state. */
+    char const * input = "aba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    /* Standard parse of R1 fails on "abb" (unexpected trailing token). */
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
+     * that parse_Start uses to select R2 over R1 in the ambiguous state, from
+     * R1's start state the reduce to R1 is unconditional, and the follow-
+     * token shift retry at the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed from
+     * the input: p_position() should point to the follow token, and a
+     * subsequent p_lex() should return it. */
+    {
+        input = "abb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `b` is at column 3 (1-based). */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token
+     * parse_Start uses to select R1, and it works here as a follow token
+     * too. */
+    {
+        input = "aba";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_a };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `a` is at column 3 (1-based) and remains in the
+         * input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_shared.d

@@ -0,0 +1,72 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_shared.c for details on the grammar. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */
+    string input = "aba";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* Standard parse of R1 fails on "abb". */
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
+     * parse_Start would use to select R2 over R1, but from R1's own start
+     * state R1 reduces unconditionally, and the follow-token shift retry at
+     * the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed: a
+     * subsequent p_lex() should return it. */
+    input = "abb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    p_position_t pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    p_token_info_t token_info;
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_b);
+    assert(token_info.position.row == 1);
+    assert(token_info.position.col == 3);
+
+    /* parse_inner_R1("aba", [a]) also succeeds. */
+    input = "aba";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_a = [TOKEN_a];
+    assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_a);
+
+    /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_R1(context, null) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+}

spec/test_parse_inner_tree.c

@@ -0,0 +1,89 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (tree generation mode, shared reduce state):
+     *   tree;
+     *   token a; token b;
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b;
+     *   R2 -> a b;
+     *
+     * Exercises p_parse_inner_R1() with a non-EOF follow token in tree
+     * generation mode. Verifies:
+     *   * The reduced tree for R1 is well-formed after a follow-token
+     *     completion.
+     *   * The follow token is not consumed and remains available for a
+     *     subsequent p_lex() call.
+     *   * p_tree_delete_R1() cleans up the returned tree without leaks
+     *     (verified in CI via valgrind). */
+
+    /* Baseline: p_parse_R1 works on "ab" and the returned tree is
+     * well-formed. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 * tree = p_result_R1(context);
+        assert_not_null(tree);
+        assert_not_null(tree->pToken1);
+        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
+        assert_not_null(tree->pToken2);
+        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
+        p_tree_delete_R1(tree);
+        p_context_delete(context);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 * tree = p_result_R1(context);
+        assert_not_null(tree);
+        assert_not_null(tree->pToken1);
+        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
+        assert_eq(1u, (size_t)tree->pToken1->position.row);
+        assert_eq(1u, (size_t)tree->pToken1->position.col);
+        assert_not_null(tree->pToken2);
+        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
+        assert_eq(1u, (size_t)tree->pToken2->position.row);
+        assert_eq(2u, (size_t)tree->pToken2->position.col);
+
+        /* The R1 tree covers positions 1..2 — the third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert_eq(1u, (size_t)tree->position.row);
+        assert_eq(1u, (size_t)tree->position.col);
+        assert_eq(1u, (size_t)tree->end_position.row);
+        assert_eq(2u, (size_t)tree->end_position.col);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+
+        /* p_tree_delete_R1 must free every node reachable from the tree
+         * without leaking anything. valgrind (invoked by the spec runner on
+         * Linux) will detect any missed frees. */
+        p_tree_delete_R1(tree);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_tree.d

@@ -0,0 +1,68 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_tree.c for details on the grammar and cases. */
+
+    /* Baseline: p_parse_R1 works on "ab". */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 * tree = p_result_R1(context);
+        assert(tree !is null);
+        assert(tree.pToken1 !is null);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.token == TOKEN_b);
+        p_tree_delete_R1(tree);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 * tree = p_result_R1(context);
+        assert(tree !is null);
+        assert(tree.pToken1 !is null);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken1.position.row == 1);
+        assert(tree.pToken1.position.col == 1);
+        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.token == TOKEN_b);
+        assert(tree.pToken2.position.row == 1);
+        assert(tree.pToken2.position.col == 2);
+
+        /* The R1 tree covers positions 1..2. The third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert(tree.position.row == 1);
+        assert(tree.position.col == 1);
+        assert(tree.end_position.row == 1);
+        assert(tree.end_position.col == 2);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 3);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 3);
+
+        p_tree_delete_R1(tree);
+    }
+}

spec/test_set_position.c

@@ -0,0 +1,81 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_set_position() overrides the default (1, 1) starting
+     * position so that lexed tokens and error positions are reported
+     * relative to the caller-supplied position. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(1u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() overrides the initial position; subsequent lex calls
+     * report token positions relative to the set position. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {5u, 20u};
+        p_set_position(context, initial);
+        p_position_t pos = p_position(context);
+        assert_eq(5u, (size_t)pos.row);
+        assert_eq(20u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(20u, (size_t)token_info.position.col);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(21u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a full parse: successful parse still works and
+     * text_position tracking is relative to the set starting point. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {3u, 7u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a parse that fails: the reported error
+     * position is relative to the set starting point. */
+    {
+        char const * input = "aa";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {10u, 2u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        /* Error is at the second `a`, which is one column past the initial
+         * column. */
+        assert_eq(10u, (size_t)err_pos.row);
+        assert_eq(3u, (size_t)err_pos.col);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_set_position.d

@@ -0,0 +1,66 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_set_position.c for details. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 1);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+
+    /* p_set_position() overrides the initial position. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(5u, 20u));
+        p_position_t pos = p_position(context);
+        assert(pos.row == 5);
+        assert(pos.col == 20);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 20);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 21);
+    }
+
+    /* p_set_position() before a full parse still parses successfully. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(3u, 7u));
+        assert(p_parse_Start(context) == P_SUCCESS);
+    }
+
+    /* p_set_position() before a parse that fails: error position is
+     * relative to the set starting point. */
+    {
+        string input = "aa";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(10u, 2u));
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        assert(err_pos.row == 10);
+        assert(err_pos.col == 3);
+    }
+}