v4.8.1

.github/workflows/run-tests.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - name: Install dependencies (Linux)
       if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc valgrind
 
     - name: Install dependencies (macOS)
       if: runner.os == 'macOS'

CHANGELOG.md

@@ -1,3 +1,36 @@
+## v4.8.1
+
+### Fixes
+
+- Fix tree node struct type forward-declarations for C/C++
+
+## v4.8.0
+
+### New Features
+
+- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow
+  tokens. These behave the same as `p_parse_XXX()` by parsing starting at the
+  given start rule, but instead of expecting the rest of the input to match
+  the start rule they allow specifying a set of tokens that may follow the
+  start rule.
+- Add `p_set_position()` API to set the current text position stored in the
+  context. Useful for setting the initial text position to something other
+  than `(1, 1)` for a nested parse operation.
+- Add `p_input_index()` API to get the current input text byte offset.
+- Add `p_set_input_index()` API to set the current input text byte offset.
+  Useful together with `p_set_position()` to rewind the input part-way through
+  a parse in order to re-read an earlier section of the input.
+
+## v4.7.0
+
+### New Features
+
+- Support parser rule user code blocks in tree generation mode.
+
+### Fixes
+
+- propane.vim: do not highlight rule components as propane keywords
+
 ## v4.6.0
 
 ### New Features

assets/parser.c.erb

@@ -923,7 +923,9 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
     memset(&empty_pos, 0, sizeof(empty_pos));
     return empty_pos;
 }
+<% end %>
 
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -934,7 +936,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1006,8 +1008,17 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-NULL, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional array of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be NULL for a standard parse.
+ * @param n_follow_tokens
+ *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1020,12 +1031,15 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
@@ -1049,6 +1063,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        for (size_t i = 0u; i < n_follow_tokens; i++)
+        {
+            if (token == follow_tokens[i])
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1068,10 +1094,42 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 result = P_SUCCESS;
                 break;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context->input_index -= token_info.length;
+                    context->text_position = token_info.position;
+<% if @grammar.tree %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+<% else %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
+                    result = P_SUCCESS;
+                    break;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             state_values_stack_push(&statevalues);
             state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
             new_state_info->state_id = shift_state;
@@ -1118,6 +1176,15 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1169,6 +1236,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 reduced_parser_node = NULL;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                state_values_stack_free(&statevalues);
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
@@ -1210,14 +1284,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
+}
+
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
 }
 <% end %>
 
@@ -1266,6 +1346,58 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
     return context->text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context->text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context->input_index;
+}
+
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context->input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.d.erb

@@ -994,7 +994,9 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
     }
     return <%= @grammar.prefix %>position_t.INVALID;
 }
+<% end %>
 
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -1005,7 +1007,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1077,8 +1079,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1091,13 +1101,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
@@ -1116,6 +1129,18 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        foreach (eof_token; follow_tokens)
+        {
+            if (token == eof_token)
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1134,10 +1159,41 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% end %>
                 return P_SUCCESS;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context.input_index -= token_info.length;
+                    context.text_position = token_info.position;
+<% if @grammar.tree %>
+                    context.parse_result = statevalues[$-1].tree_node;
+<% else %>
+                    context.parse_result = statevalues[$-1].pvalue;
+<% end %>
+                    return P_SUCCESS;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             statevalues ~= state_value_t(shift_state);
             if (reduced_rule_set == INVALID_ID)
             {
@@ -1171,6 +1227,15 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
         }
 
         size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1227,6 +1292,12 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 reduced_parser_node = null;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1263,14 +1334,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 
 public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
+}
+
+public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
 }
 <% end %>
 
@@ -1348,6 +1425,58 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
     return context.text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context.text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context.input_index;
+}
+
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context.input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -104,7 +104,7 @@ typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
 <%     next if rule_set.optional? %>
-struct <%= name %>;
+struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
 <%   end %>
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
@@ -212,6 +212,8 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
 <%   end %>
 
 <% if @grammar.tree %>
@@ -235,6 +237,12 @@ void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefi
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
+
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
+
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
+
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -270,9 +270,38 @@ For C targets this can be accomplished with
 `if (p_position_valid(${$.position}))` and for D targets this can be
 accomplished with `if (${$.position}.valid)`.
 
-Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
+Parser rule code blocks are still supported in tree generation mode, but they
+behave differently than when tree generation mode is not active.
+The code block for a rule is executed after the rule has been matched and its
+tree node has been fully formed.
+Within the code block, `$$` refers to the tree node for the reduced rule, typed
+as a pointer to that rule's generated tree node structure.
+The tree nodes for the rule components are accessed positionally with `$1`,
+`$2`, `$3`, etc..., each typed as a pointer to the generated tree node structure
+for that component (a rule node or a `Token` node).
+Field aliases (see the "Specifying parser rules" section) may also be used to
+reference a component tree node by name; a field alias behaves identically to
+the positional reference for that component.
+The positional position expansions (`${$.position}`, `${N.position}`, etc...)
+are not available in tree generation mode; the `position` and `end_position`
+fields of the tree nodes can be accessed directly instead.
+
+Example:
+
+```
+tree;
+
+Assignment -> ident equals Expr <<
+    /* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is
+     * the Expr rule node. */
+    printf("assignment on row %d, col %d\n",
+        $$->position.row, $$->position.col);
+    printf("target identifier ends on row %d, col %d\n",
+        $1->end_position.row, $1->end_position.col);
+>>
+```
 
 ##> `context_user_fields` statement - adding custom fields to the context
 
@@ -424,6 +453,41 @@ enabled.
 When tree generation is enabled, the `pvalue` field can be set to an instance
 of whatever type the user has defined as the `ptype` type.
 
+The custom lex function returns a result code to the parser.
+Returning `P_SUCCESS` indicates that the function has produced a token in the
+`token` output field and the parse should proceed.
+Returning any other result code stops the parse immediately; the parser
+propagates that code out of `p_parse()` (and the `p_parse_XXX()` and
+`p_parse_inner_XXX()` functions) unchanged.
+This allows a custom lex function to surface an error condition, for example by
+propagating a `P_UNEXPECTED_TOKEN`, `P_UNEXPECTED_INPUT`, or `P_DECODE_ERROR`
+result from a nested `p_parse_inner_XXX()` or `p_lex()` call.
+
+Observe the following contract when returning a result code other than
+`P_SUCCESS`:
+
+* Do not return `P_DROP`.
+  This code is an internal lexer signal and is never returned to the parser by
+  the generated `p_lex()` function.
+  Returning it from a custom lex function would be reported as a spurious parse
+  failure. If drop functionality is required, the custom lex function should
+  loop and return the next non-drop token.
+* Do not return `P_EOF` to signal the end of the input.
+  The end of input is communicated to the parser by returning `P_SUCCESS` with
+  the `token` field set to the end-of-input token (`TOKEN___EOF`), which is what
+  the generated `p_lex()` function does.
+  Returning `P_EOF` aborts the parse rather than allowing it to complete.
+* The `p_token()` and `p_position()` accessors are populated by the parser only
+  when the parser itself detects an unexpected token.
+  When a custom lex function returns an error code, `p_token()` is not updated
+  and may not reflect a meaningful token, and `p_position()` reflects the
+  lexer's current text position rather than a parser-identified error location.
+  A custom lex function that wants a specific reported position can set it with
+  `p_set_position()` before returning.
+* Returning `P_USER_TERMINATED` does not populate the user terminate code
+  returned by `p_user_terminate_code()`. This user terminate code is normally
+  populated by the `$terminate()` function in the user code block.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -710,7 +774,9 @@ It is recommended to place this statement early in the grammar.
 In tree generation mode various aspects of propane's behavior are changed:
 
   * Only one `ptype` is allowed.
-  * Parser user code blocks are not supported.
+  * Parser user code blocks execute after the rule's tree node has been formed
+  and access the tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser rule
+  code blocks" section).
   * Structure types are generated to represent the parsed tokens and rules as
   defined in the grammar.
   * The parse result from `p_result()` points to a `Start` struct containing
@@ -1059,8 +1125,12 @@ The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.
 
-Parser rule code blocks are not allowed and not used when tree generation mode
-is active.
+In tree generation mode, parser rule code blocks access the reduced rule tree
+node and its component tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser
+rule code blocks" section).
+Field aliases may still be used in tree generation mode to reference a component
+tree node by name, behaving identically to the corresponding positional
+reference.
 
 ##> User termination of the lexer or parser
 
@@ -1332,6 +1402,42 @@ size_t result = p_parse_Statement(context);
 
 In this case, the parser will start parsing with the `Statement` rule.
 
+### `p_parse_inner_XXX`
+
+For each start rule, a `p_parse_inner_XXX()` function is also generated.
+This variant of the parser entry point accepts a caller-provided array of
+"follow tokens" — tokens the caller allows to appear immediately after the
+start rule in some outer grammar context.
+It is useful when embedding a Propane-generated sub-parser within an outer
+parser and the outer parser knows which tokens naturally terminate the
+sub-parse.
+
+For C targets, the signature is (example for a rule named `Statement`):
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    p_token_t const * follow_tokens, size_t n_follow_tokens);
+```
+
+Passing a `NULL` pointer (or a count of zero) makes the function behave
+identically to `p_parse_Statement()`.
+
+For D targets, the signature accepts a slice:
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    const(p_token_t)[] follow_tokens);
+```
+
+Passing `null` for the slice makes the function behave identically to
+`p_parse_Statement()`.
+
+When the parse is completed via a non-EOF follow token, that follow token is
+**not** consumed from the input stream.
+The parser rewinds the input index and text position to the start of the follow
+token so that a subsequent call to `p_lex()` or another parse function sees the
+same token.
+
 ### `p_position_valid`
 
 The `p_position_valid()` function is only generated for C targets.
@@ -1403,6 +1509,63 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 }
 ```
 
+### `p_set_position`
+
+The `p_set_position()` function sets the current text position stored in the
+context.
+This is useful when performing a nested parse operation over a slice of a
+larger enclosing document: by setting the initial text position to something
+other than the default `(1, 1)`, any token positions or error positions
+subsequently reported by the lexer/parser are relative to the enclosing
+document rather than the slice.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_position_t start = { .row = 5, .col = 20 };
+p_set_position(context, start);
+p_parse(context);
+```
+
+### `p_input_index`
+
+The `p_input_index()` function returns the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This is useful for slicing out the remaining input after a partial parse,
+or for locating tokens in the original input buffer.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_parse_inner_Statement(context, follow_tokens, n_follow_tokens);
+size_t offset = p_input_index(context);
+/* Remaining input starts at `input + offset`. */
+```
+
+### `p_set_input_index`
+
+The `p_set_input_index()` function sets the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This moves the lexer's read cursor, which can be used together with
+`p_set_position()` to rewind the input part-way through a parse in order to
+re-read an earlier section of the input.
+The byte offset is not validated; the caller is responsible for providing an
+offset within the bounds of the input text.
+A value previously returned by `p_input_index()` is a suitable argument.
+
+Example:
+
+```
+/* Save the cursor and text position at the start of a section. */
+size_t saved_index = p_input_index(context);
+p_position_t saved_position = p_position(context);
+/* ... later, rewind to re-read that section. */
+p_set_input_index(context, saved_index);
+p_set_position(context, saved_position);
+```
+
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user

extra/vim/syntax/propane.vim

@@ -20,10 +20,16 @@ exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
 syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
 
 syn match propaneComment "#.*"
-syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
+" The right-hand side of a rule (after '->' up to '<<' or ';') lists symbol
+" names that may coincide with propane keywords (e.g. 'start', 'token',
+" 'tree'). Wrap it in a region that excludes keyword matches so those names
+" are not highlighted as keywords. The '<<' is left unconsumed so the
+" propaneTarget region can still match it.
+syn region propaneRuleRhs matchgroup=propaneOperator start="->" end="\ze<<" end=";" contains=propaneFieldAlias,propaneRuleOperator,propaneComment keepend
+syn match propaneRuleOperator "?" contained
 " Keywords that introduce a user-defined name. The name is consumed by
 " propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
 " is not highlighted as a keyword. These must be a match (not syn keyword)
@@ -39,6 +45,7 @@ hi def link propaneKeyword Keyword
 hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
+hi def link propaneRuleOperator Operator
 hi def link propaneFieldOperator Operator
 hi def link propaneDelimiter Delimiter
 hi def link propaneFieldAlias Identifier

lib/propane/generator.rb

@@ -185,7 +185,8 @@ class Propane
                 end
               end
               @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
-              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optcode = @grammar.tree ? nil : "$$ = $1;\n"
+              @grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number)
               optional_rules_added << component
             end
           end
@@ -296,21 +297,24 @@ class Propane
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "_pvalue->v_#{rule.ptypename}"
-          when "d"
-            "_pvalue.v_#{rule.ptypename}"
+          if @grammar.tree
+            case @language
+            when "c"
+              "((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+            when "d"
+              "(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+            end
+          else
+            case @language
+            when "c"
+              "_pvalue->v_#{rule.ptypename}"
+            when "d"
+              "_pvalue.v_#{rule.ptypename}"
+            end
           end
         end
         code = code.gsub(/\$(\d+)/) do |match|
-          index = $1.to_i
-          case @language
-          when "c"
-            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          when "d"
-            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
-          end
+          parser_component_reference(rule, $1.to_i)
         end
         code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
           index = $1.to_i
@@ -323,12 +327,11 @@ class Propane
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
-            case @language
-            when "c"
-              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
-            when "d"
-              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
-            end
+            # Field aliases are just a named reference to a positional rule
+            # component, so reuse the same expansion as `$1', `$2', etc. Note
+            # that rule.aliases stores a 0-based component index, so add 1 to
+            # convert it to the 1-based index used for positional references.
+            parser_component_reference(rule, index + 1)
           else
             raise Error.new("Field alias '#{aliasname}' not found")
           end
@@ -384,6 +387,45 @@ class Propane
       code
     end
 
+    # Expand a positional reference to a parser rule component.
+    #
+    # This is used to expand `$1', `$2', etc. as well as field aliases (which
+    # are just named references to a positional rule component).
+    #
+    # @param rule [Rule]
+    #   The Rule containing the user code.
+    # @param index [Integer]
+    #   1-based index of the rule component to reference.
+    #
+    # @return [String]
+    #   Expanded rule component reference.
+    def parser_component_reference(rule, index)
+      component = rule.components[index - 1]
+      if @grammar.tree
+        # In tree mode a component reference yields a pointer to that
+        # component's tree node. An optional component propagates its target
+        # node (or null), so use the optional target's node type.
+        if component.is_a?(RuleSet) && component.optional?
+          component = component.option_target
+        end
+        node_name = component.is_a?(Token) ? "Token" : component.name
+        typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
+        case @language
+        when "c"
+          "((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
+        when "d"
+          "(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
+        end
+      else
+        case @language
+        when "c"
+          "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
+        when "d"
+          "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
+        end
+      end
+    end
+
     # Get the lex function to use.
     #
     # @return [String]

lib/propane/grammar.rb

@@ -58,6 +58,10 @@ class Propane
       @tokens.size + 1
     end
 
+    def parser_user_code_used?
+      @rules.any? {|r| r.code}
+    end
+
     private
 
     def parse_grammar!
@@ -263,12 +267,8 @@ class Propane
         end
         md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        if @tree
-          consume!(/;/, "expected `;'")
-        else
-          unless code = parse_code_block!
-            consume!(/;/, "expected `;' or code block")
-          end
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @modeline = nil

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.6.0"
+  VERSION = "4.8.1"
 end

spec/macros.c.propane

@@ -0,0 +1,25 @@
+<<
+#include <stdlib.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int v);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/macros.d.propane

@@ -0,0 +1,31 @@
+<<
+import test_macros;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+    int n = 0;
+    foreach (c; match)
+    {
+        n *= 10;
+        n += (c - '0');
+    }
+    $$ = n;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -0,0 +1,19 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.c.propane

@@ -0,0 +1,17 @@
+<<
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/parse_inner_nested_tree.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested_tree;
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/propane_spec.rb

@@ -95,13 +95,17 @@ EOF
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
     results = Results.new(stdout, stderr, status)
-    stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
-    vgout = stdout + stderr
-    File.binwrite("spec/run/.vgout", vgout)
-    vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
-      bytes = $1.to_i
-      if bytes > 0
-        raise "Valgrind detected memory leak"
+    # Valgrind is only reliably available on Linux, so limit the leak check to
+    # Linux platforms.
+    if RUBY_PLATFORM =~ /linux/
+      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+      vgout = stdout + stderr
+      File.binwrite("spec/run/.vgout", vgout)
+      vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
+        bytes = $1.to_i
+        if bytes > 0
+          raise "Valgrind detected memory leak"
+        end
       end
     end
     results
@@ -745,6 +749,78 @@ EOF
         ])
       end
 
+      it "executes user code associated with a parser rule in tree mode" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$->n_fields;
+  ${context.start_a_value} = $$->pA->pToken1->pvalue;
+  ${context.a_value} = $1->pToken1->pvalue;
+  ${context.b_value} = $2->pToken1->pvalue;
+  ${context.b_token} = $2->pToken1->token;
+  ${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0;
+  ${context.alias_a_value} = ${ay}->pToken1->pvalue;
+  ${context.alias_b_value} = ${bee}->pToken1->pvalue;
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >>
+EOF
+        when "d"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$.n_fields;
+  ${context.start_a_value} = $$.pA.pToken1.pvalue;
+  ${context.a_value} = $1.pToken1.pvalue;
+  ${context.b_value} = $2.pToken1.pvalue;
+  ${context.b_token} = $2.pToken1.token;
+  ${context.c_field_is_null} = ($$.pC is null) ? 1 : 0;
+  ${context.alias_a_value} = ${ay}.pToken1.pvalue;
+  ${context.alias_b_value} = ${bee}.pToken1.pvalue;
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_parser_user_code_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "parses lists" do
         write_grammar <<EOF
 ptype #{language == "d" ? "uint" : "uint32_t"};
@@ -862,6 +938,44 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows setting the text position via p_set_position()" do
+        write_grammar <<EOF
+token a;
+token b;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_set_position.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "exposes the current input byte offset via p_input_index()" do
+        write_grammar <<EOF
+drop /\\s+/;
+token a;
+token b;
+start Start;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_input_index.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows rewinding the input to repeat a section during a parse" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/rewind.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_rewind.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows creating a JSON parser" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/json_parser.#{ext}.propane"))
@@ -1513,6 +1627,95 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "supports parse_inner APIs that treat provided tokens as follow tokens" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+Start -> Y << $$ = $1; >>
+Y -> a << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs block success when the outer rule is unfinished" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> a Start b << $$ = $2; >>
+Start -> c << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_recursive.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
+    write_grammar <<EOF
+ptype int;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b << $$ = 11; >>
+R2 -> a b << $$ = 22; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_shared.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work in tree generation mode with follow tokens" do
+    write_grammar <<EOF
+tree;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b;
+R2 -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "supports a reentrant nested parse driven from a custom lex function" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/parse_inner_nested.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_parse_inner_nested.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "tracks positions across a lex-function nested parse in tree mode" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/parse_inner_nested_tree.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_parse_inner_nested_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows multiple starting rules in tree mode" do
     write_grammar <<EOF
 tree;
@@ -1991,6 +2194,16 @@ StartR start: 1, 3
 StartR end: 1, 6
 EOF
       end
+
+      it "allows nested parses for macro expansions" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/macros.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_macros.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/rewind.c.propane

@@ -0,0 +1,23 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int value);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -0,0 +1,20 @@
+<<
+import test_rewind;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/test_input_index.c

@@ -0,0 +1,60 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   drop /\\s+/;
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_input_index() reports the parser/lexer's current byte
+     * offset into the input text. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert_eq(0u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        char const * input = "a b";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        /* The dropped space between `a` and `b` advances input_index too. */
+        assert_eq(3u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_input_index.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_input_index.c for details on the grammar and cases. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_input_index(context) == 0);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        string input = "a b";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_input_index(context) == 1);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 3);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+}

spec/test_macros.c

@@ -0,0 +1,118 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+#include <assert.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+static p_context_t * context;
+size_t n_tokens;
+p_token_info_t token_infos[10];
+
+/* Capture the macro body tokens (everything up to the closing '}') into
+ * token_infos[]. Called from mylexfn() right after the definition's '{' has
+ * been lexed, so the input cursor is positioned at the first body token. */
+static void capture_macro_body(void)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert_eq(result, P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < sizeof(token_infos) / sizeof(token_infos[0]));
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info->token)
+        {
+        case TOKEN_macro:
+            /* Start of a macro definition: "macro macroname { ... }". */
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                /* Use of a macro: replay its captured body tokens instead of
+                 * returning the macroname to the parser. */
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            /* Definition name: pass through and keep waiting for '{'. */
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                /* Consume and store the macro body now, before the parser gets
+                 * a chance to read its lookahead token (which would otherwise
+                 * swallow the first body token). */
+                capture_macro_body();
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int nums[10];
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    char const * input =
+        "macro @m { 23 + 200 }\n"
+        "66 + 100\n"
+        "@m\n"
+        "33 + 55\n"
+        "@m\n";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(n_nums, 4);
+    assert_eq(nums[0], 166);
+    assert_eq(nums[1], 223);
+    assert_eq(nums[2], 88);
+    assert_eq(nums[3], 223);
+
+    return 0;
+}

spec/test_macros.d

@@ -0,0 +1,116 @@
+import testparser;
+import testutils;
+
+size_t n_tokens;
+p_token_info_t[10] token_infos;
+
+// Capture the macro body tokens (everything up to the closing '}') into
+// token_infos[]. Called from mylexfn() right after the definition's '{' has
+// been lexed, so the input cursor is positioned at the first body token.
+void capture_macro_body(p_context_t * context)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert(result == P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < token_infos.length);
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info.token)
+        {
+        case TOKEN_macro:
+            // Start of a macro definition: "macro macroname { ... }".
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                // Use of a macro: replay its captured body tokens instead of
+                // returning the macroname to the parser.
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            // Definition name: pass through and keep waiting for '{'.
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                // Consume and store the macro body now, before the parser gets
+                // a chance to read its lookahead token (which would otherwise
+                // swallow the first body token).
+                capture_macro_body(context);
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int[10] nums;
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "macro @m { 23 + 200 }\n" ~
+        "66 + 100\n" ~
+        "@m\n" ~
+        "33 + 55\n" ~
+        "@m\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert(n_nums == 4);
+    assert(nums[0] == 166);
+    assert(nums[1] == 223);
+    assert(nums[2] == 88);
+    assert(nums[3] == 223);
+}

spec/test_parse_inner.c

@@ -0,0 +1,73 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (chain reduce):
+     *   Start -> Y      << $$ = $1; >>
+     *   Y     -> a      << $$ = $1; >>
+     *   token a         << $$ = 1; >>
+     *
+     * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF,
+     * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with
+     * `b` as a follow token should succeed via the reduce-side retry chain
+     * (Y then Start) followed by the shift-side retry hitting $EOF at the
+     * final state. */
+
+    /* Standard parse succeeds on complete input. */
+    char const * input = "a";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(1u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with an empty (NULL) follow-token vector behaves like a
+     * standard parse. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    {
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+    }
+    p_context_delete(context);
+
+    /* parse_inner with a non-matching follow token still fails. The grammar
+     * can't consume `b` and it isn't listed as a follow token, so the retries
+     * do not fire. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner.c for details on the grammar and cases. */
+
+    /* Standard parse succeeds on complete input. */
+    string input = "a";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a null follow-token slice behaves like a standard
+     * parse. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a non-matching follow token still fails. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_eof = [TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN);
+}

spec/test_parse_inner_nested.c

@@ -0,0 +1,78 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (integer evaluator; parentheses handled by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/;
+ *   token num /\d+/          << ... atoi ... >>
+ *   Start -> Expr            << $$ = $1; >>
+ *   Expr  -> num             << $$ = $1; >>
+ *   Expr  -> Expr plus num   << $$ = $1 + $3; >>
+ *
+ * The tokens lparen and rparen appear in no grammar rule. Instead, when the
+ * lex function lexes a '(', it performs a nested parse (p_parse_inner_Start)
+ * of the parenthesized sub-expression -- reentrantly, while the outer parse is
+ * still suspended in this callback -- reads the computed value with
+ * p_result_Start, consumes the ')' that p_parse_inner deliberately left in the
+ * input, and hands a single synthesized num token carrying that value back to
+ * the outer parse. Nested groups recurse this process to arbitrary depth. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->pvalue = p_value(value);
+    }
+    return P_SUCCESS;
+}
+
+static int eval(char const * input)
+{
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5u, (size_t)eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3u, (size_t)eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14u, (size_t)eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37u, (size_t)eval("2 + (10 + (20 + 5))"));
+    assert_eq(15u, (size_t)eval("(1 + 2) + (3 + (4 + 5))"));
+
+    return 0;
+}

spec/test_parse_inner_nested.d

@@ -0,0 +1,64 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue = p_value(value);
+    }
+    return P_SUCCESS;
+}
+
+int eval(string input)
+{
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5, eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3, eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14, eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37, eval("2 + (10 + (20 + 5))"));
+    assert_eq(15, eval("(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested_tree.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (tree generation mode; parentheses handled by the lex function):
+ *   tree;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/; token num /\d+/;
+ *   Start -> Expr;
+ *   Expr  -> num;
+ *   Expr  -> Expr plus num;
+ *
+ * The same lexer-driven nested parse as test_parse_inner_nested, but in tree
+ * generation mode. Each "( ... )" group is parsed by a reentrant
+ * p_parse_inner_Start() call from the lex function; the resulting subtree is
+ * discarded and a single synthesized num token is handed to the outer parse.
+ * The synthesized token's position is set to span the whole group ('(' start
+ * through ')' end), so this verifies that positions survive the nested-parse
+ * boundary and land correctly in the outer tree. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info->position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start * inner = p_result_Start(context);
+        assert_not_null(inner);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq((size_t)(start_position.col + 1u), (size_t)inner->position.col);
+        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)inner->end_position.col);
+        p_tree_delete_Start(inner);
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->position = start_position;
+        out_token_info->end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    char const * input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start * tree = p_result(context);
+    assert_not_null(tree);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr * top = tree->pExpr;
+    assert_not_null(top);
+    assert_not_null(top->pExpr);
+    assert_not_null(top->pToken2);
+    assert_not_null(top->pToken3);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, (size_t)top->pToken2->position.row);
+    assert_eq(9u, (size_t)top->pToken2->position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, (size_t)top->pToken3->position.row);
+    assert_eq(11u, (size_t)top->pToken3->position.col);
+    assert_eq(1u, (size_t)top->pToken3->end_position.row);
+    assert_eq(17u, (size_t)top->pToken3->end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr * left = top->pExpr;
+    assert_not_null(left->pToken1);
+    assert_eq(1u, (size_t)left->pToken1->position.row);
+    assert_eq(1u, (size_t)left->pToken1->position.col);
+    assert_eq(1u, (size_t)left->pToken1->end_position.row);
+    assert_eq(7u, (size_t)left->pToken1->end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, (size_t)tree->position.col);
+    assert_eq(17u, (size_t)tree->end_position.col);
+
+    p_tree_delete(tree);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_nested_tree.d

@@ -0,0 +1,91 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested_tree.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info.position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start * inner = p_result_Start(context);
+        assert(inner !is null);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq(start_position.col + 1u, inner.position.col);
+        assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
+        p_tree_delete_Start(inner);
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    string input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start * tree = p_result(context);
+    assert(tree !is null);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr * top = tree.pExpr;
+    assert(top !is null);
+    assert(top.pExpr !is null);
+    assert(top.pToken2 !is null);
+    assert(top.pToken3 !is null);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, top.pToken2.position.row);
+    assert_eq(9u, top.pToken2.position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, top.pToken3.position.row);
+    assert_eq(11u, top.pToken3.position.col);
+    assert_eq(1u, top.pToken3.end_position.row);
+    assert_eq(17u, top.pToken3.end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr * left = top.pExpr;
+    assert(left.pToken1 !is null);
+    assert_eq(1u, left.pToken1.position.row);
+    assert_eq(1u, left.pToken1.position.col);
+    assert_eq(1u, left.pToken1.end_position.row);
+    assert_eq(7u, left.pToken1.end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, tree.position.col);
+    assert_eq(17u, tree.end_position.col);
+
+    p_tree_delete(tree);
+    p_context_delete(context);
+}

spec/test_parse_inner_recursive.c

@@ -0,0 +1,77 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (recursive):
+     *   Start -> a Start b   << $$ = $2; >>
+     *   Start -> c           << $$ = $1; >>
+     *   token a << $$ = 1; >>
+     *   token b << $$ = 2; >>
+     *   token c << $$ = 3; >>
+     *
+     * Here `Start` can appear in the middle of another `Start` rule, so the
+     * inner-parse follow-token success must be blocked whenever an unfinished
+     * outer `Start -> a Start b` remains on the parse stack (i.e. the parse
+     * stack contains more than just the initial state and the reduced start
+     * rule set). */
+
+    /* Standard parse of `c` succeeds. */
+    char const * input = "c";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `acb` succeeds (full outer rule). */
+    input = "acb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `ac` fails (`b` missing). */
+    input = "ac";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner with `ac` and follow token `b` also fails: even though the
+     * inner `Start -> c` reduces and `Start` is shifted, the outer
+     * `Start -> a Start . b` is still on the stack (stack length > 2), so the
+     * "reduced start rule is the only thing on the parse stack" invariant
+     * blocks the shift-side follow-token success. */
+    {
+        input = "ac";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    /* parse_inner with `acb` (complete outer rule) succeeds via the standard
+     * path. */
+    {
+        input = "acb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with just `c` succeeds via the standard path even when a
+     * follow-token vector is supplied. */
+    {
+        input = "c";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_recursive.d

@@ -0,0 +1,49 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_recursive.c for details on the grammar. */
+
+    /* Standard parse of `c` succeeds. */
+    string input = "c";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `acb` succeeds. */
+    input = "acb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `ac` fails. */
+    input = "ac";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `ac` fails: outer rule still on the stack. */
+    input = "ac";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `acb` succeeds via the standard path. */
+    input = "acb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* parse_inner with just `c` succeeds via the standard path. */
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+}

spec/test_parse_inner_shared.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar:
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b   << $$ = 11; >>
+     *   R2 -> a b   << $$ = 22; >>
+     *   token a; token b;
+     *
+     * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within
+     * parse_Start, the generated parser differentiates the reduce by
+     * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2
+     * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional
+     * on any lookahead. This test exercises p_parse_inner_R1() to confirm
+     * that reductions to R1 succeed even when the incoming follow token is
+     * not the natural lookahead used by parse_Start's disambiguation. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the
+     * shared "a b" state. */
+    char const * input = "aba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    /* Standard parse of R1 fails on "abb" (unexpected trailing token). */
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
+     * that parse_Start uses to select R2 over R1 in the ambiguous state, from
+     * R1's start state the reduce to R1 is unconditional, and the follow-
+     * token shift retry at the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed from
+     * the input: p_position() should point to the follow token, and a
+     * subsequent p_lex() should return it. */
+    {
+        input = "abb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `b` is at column 3 (1-based). */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token
+     * parse_Start uses to select R1, and it works here as a follow token
+     * too. */
+    {
+        input = "aba";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_a };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `a` is at column 3 (1-based) and remains in the
+         * input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_shared.d

@@ -0,0 +1,72 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_shared.c for details on the grammar. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */
+    string input = "aba";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* Standard parse of R1 fails on "abb". */
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
+     * parse_Start would use to select R2 over R1, but from R1's own start
+     * state R1 reduces unconditionally, and the follow-token shift retry at
+     * the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed: a
+     * subsequent p_lex() should return it. */
+    input = "abb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    p_position_t pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    p_token_info_t token_info;
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_b);
+    assert(token_info.position.row == 1);
+    assert(token_info.position.col == 3);
+
+    /* parse_inner_R1("aba", [a]) also succeeds. */
+    input = "aba";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_a = [TOKEN_a];
+    assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_a);
+
+    /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_R1(context, null) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+}

spec/test_parse_inner_tree.c

@@ -0,0 +1,89 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (tree generation mode, shared reduce state):
+     *   tree;
+     *   token a; token b;
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b;
+     *   R2 -> a b;
+     *
+     * Exercises p_parse_inner_R1() with a non-EOF follow token in tree
+     * generation mode. Verifies:
+     *   * The reduced tree for R1 is well-formed after a follow-token
+     *     completion.
+     *   * The follow token is not consumed and remains available for a
+     *     subsequent p_lex() call.
+     *   * p_tree_delete_R1() cleans up the returned tree without leaks
+     *     (verified in CI via valgrind). */
+
+    /* Baseline: p_parse_R1 works on "ab" and the returned tree is
+     * well-formed. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 * tree = p_result_R1(context);
+        assert_not_null(tree);
+        assert_not_null(tree->pToken1);
+        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
+        assert_not_null(tree->pToken2);
+        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
+        p_tree_delete_R1(tree);
+        p_context_delete(context);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 * tree = p_result_R1(context);
+        assert_not_null(tree);
+        assert_not_null(tree->pToken1);
+        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
+        assert_eq(1u, (size_t)tree->pToken1->position.row);
+        assert_eq(1u, (size_t)tree->pToken1->position.col);
+        assert_not_null(tree->pToken2);
+        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
+        assert_eq(1u, (size_t)tree->pToken2->position.row);
+        assert_eq(2u, (size_t)tree->pToken2->position.col);
+
+        /* The R1 tree covers positions 1..2 — the third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert_eq(1u, (size_t)tree->position.row);
+        assert_eq(1u, (size_t)tree->position.col);
+        assert_eq(1u, (size_t)tree->end_position.row);
+        assert_eq(2u, (size_t)tree->end_position.col);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+
+        /* p_tree_delete_R1 must free every node reachable from the tree
+         * without leaking anything. valgrind (invoked by the spec runner on
+         * Linux) will detect any missed frees. */
+        p_tree_delete_R1(tree);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_tree.d

@@ -0,0 +1,68 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_tree.c for details on the grammar and cases. */
+
+    /* Baseline: p_parse_R1 works on "ab". */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 * tree = p_result_R1(context);
+        assert(tree !is null);
+        assert(tree.pToken1 !is null);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.token == TOKEN_b);
+        p_tree_delete_R1(tree);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 * tree = p_result_R1(context);
+        assert(tree !is null);
+        assert(tree.pToken1 !is null);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken1.position.row == 1);
+        assert(tree.pToken1.position.col == 1);
+        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.token == TOKEN_b);
+        assert(tree.pToken2.position.row == 1);
+        assert(tree.pToken2.position.col == 2);
+
+        /* The R1 tree covers positions 1..2. The third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert(tree.position.row == 1);
+        assert(tree.position.col == 1);
+        assert(tree.end_position.row == 1);
+        assert(tree.end_position.col == 2);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 3);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 3);
+
+        p_tree_delete_R1(tree);
+    }
+}

spec/test_parser_user_code_tree.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context->start_n_fields);
+    assert_eq(11, context->start_a_value);
+    assert_eq(11, context->a_value);
+    assert_eq(22, context->b_value);
+    assert_eq(TOKEN_b, context->b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context->c_is_null);
+    assert_eq(1, context->c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context->alias_a_value);
+    assert_eq(22, context->alias_b_value);
+
+    Start * start = p_result(context);
+    assert(start->pA != NULL);
+    assert(start->pB != NULL);
+    assert(start->pC == NULL);
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parser_user_code_tree.d

@@ -0,0 +1,39 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context.start_n_fields);
+    assert_eq(11, context.start_a_value);
+    assert_eq(11, context.a_value);
+    assert_eq(22, context.b_value);
+    assert_eq(TOKEN_b, context.b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context.c_is_null);
+    assert_eq(1, context.c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context.alias_a_value);
+    assert_eq(22, context.alias_b_value);
+
+    Start * start = p_result(context);
+    assert(start.pA !is null);
+    assert(start.pB !is null);
+    assert(start.pC is null);
+    p_tree_delete(start);
+}

spec/test_rewind.c

@@ -0,0 +1,124 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (statement list of additions; a "repeat" directive handled entirely
+ * by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token repeat /repeat/; token lbrace /\{/; token rbrace /\}/;
+ *   token plus /\+/; token num /\d+/ << ... atoi ... >>
+ *   Start      -> Statements;
+ *   Statements -> ;
+ *   Statements -> Statement Statements;
+ *   Statement  -> Add;
+ *   Add        -> num plus num  << record($1 + $3); >>
+ *
+ * Scenario: a "repeat <count> { <body> }" directive that expands its body
+ * <count> times, similar to loop unrolling in a configuration DSL. The tokens
+ * repeat, lbrace, and rbrace appear in no grammar rule; the lex function
+ * interprets the directive and feeds the body's tokens to the parser <count>
+ * times. Rather than buffering the body tokens, the lex function records the
+ * input byte offset and text position at the start of the body (with
+ * p_input_index() and p_position()) and, each time it reaches the closing '}',
+ * rewinds the lexer back to that point (with p_set_input_index() and
+ * p_set_position()) to re-read the body from the original input. Rewinding the
+ * text position as well as the byte offset means each expansion reports the
+ * same token positions as the first. */
+
+static int nums[16];
+static size_t n_nums;
+static uint32_t num_cols[16];
+static size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info->token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info->token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info->token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info->position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    char const * input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* The additions were recorded once per body expansion, then once more for
+     * the trailing statement. */
+    assert_eq(4u, n_nums);
+    assert_eq(30u, (size_t)nums[0]);
+    assert_eq(30u, (size_t)nums[1]);
+    assert_eq(30u, (size_t)nums[2]);
+    assert_eq(10u, (size_t)nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, (size_t)num_cols[0]);
+    assert_eq(17u, (size_t)num_cols[1]);
+    assert_eq(12u, (size_t)num_cols[2]);
+    assert_eq(17u, (size_t)num_cols[3]);
+    assert_eq(12u, (size_t)num_cols[4]);
+    assert_eq(17u, (size_t)num_cols[5]);
+    assert_eq(22u, (size_t)num_cols[6]);
+    assert_eq(26u, (size_t)num_cols[7]);
+
+    return 0;
+}

spec/test_rewind.d

@@ -0,0 +1,101 @@
+import testparser;
+import testutils;
+
+/* Grammar and scenario: see test_rewind.c. */
+
+int[16] nums;
+size_t n_nums;
+uint[16] num_cols;
+size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info.token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info.token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info.token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info.position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    string input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(4u, n_nums);
+    assert_eq(30, nums[0]);
+    assert_eq(30, nums[1]);
+    assert_eq(30, nums[2]);
+    assert_eq(10, nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, num_cols[0]);
+    assert_eq(17u, num_cols[1]);
+    assert_eq(12u, num_cols[2]);
+    assert_eq(17u, num_cols[3]);
+    assert_eq(12u, num_cols[4]);
+    assert_eq(17u, num_cols[5]);
+    assert_eq(22u, num_cols[6]);
+    assert_eq(26u, num_cols[7]);
+}

spec/test_set_position.c

@@ -0,0 +1,109 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_set_position() overrides the default (1, 1) starting
+     * position so that lexed tokens and error positions are reported
+     * relative to the caller-supplied position. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(1u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() overrides the initial position; subsequent lex calls
+     * report token positions relative to the set position. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {5u, 20u};
+        p_set_position(context, initial);
+        p_position_t pos = p_position(context);
+        assert_eq(5u, (size_t)pos.row);
+        assert_eq(20u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(20u, (size_t)token_info.position.col);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(21u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a full parse: successful parse still works and
+     * text_position tracking is relative to the set starting point. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {3u, 7u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a parse that fails: the reported error
+     * position is relative to the set starting point. */
+    {
+        char const * input = "aa";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {10u, 2u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        /* Error is at the second `a`, which is one column past the initial
+         * column. */
+        assert_eq(10u, (size_t)err_pos.row);
+        assert_eq(3u, (size_t)err_pos.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input: both
+     * tokens are lexed, then the cursor and text position are rewound to the
+     * start so that the same tokens are produced again with the same reported
+     * positions. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert_eq(0u, start_index);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(2u, p_input_index(context));
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert_eq(0u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_set_position.d

@@ -0,0 +1,90 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_set_position.c for details. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 1);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+
+    /* p_set_position() overrides the initial position. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(5u, 20u));
+        p_position_t pos = p_position(context);
+        assert(pos.row == 5);
+        assert(pos.col == 20);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 20);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 21);
+    }
+
+    /* p_set_position() before a full parse still parses successfully. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(3u, 7u));
+        assert(p_parse_Start(context) == P_SUCCESS);
+    }
+
+    /* p_set_position() before a parse that fails: error position is
+     * relative to the set starting point. */
+    {
+        string input = "aa";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(10u, 2u));
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        assert(err_pos.row == 10);
+        assert(err_pos.col == 3);
+    }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert(start_index == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 2);
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert(p_input_index(context) == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+}