v4.8.1

Allow user code to access specific pvalue values, especially when
multiple ptypes are in use.

.github/workflows/run-tests.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - name: Install dependencies (Linux)
       if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc valgrind
 
     - name: Install dependencies (macOS)
       if: runner.os == 'macOS'

CHANGELOG.md

@@ -1,3 +1,78 @@
+## v4.8.1
+
+### Fixes
+
+- Fix tree node struct type forward-declarations for C/C++
+
+## v4.8.0
+
+### New Features
+
+- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow
+  tokens. These behave the same as `p_parse_XXX()` by parsing starting at the
+  given start rule, but instead of expecting the rest of the input to match
+  the start rule they allow specifying a set of tokens that may follow the
+  start rule.
+- Add `p_set_position()` API to set the current text position stored in the
+  context. Useful for setting the initial text position to something other
+  than `(1, 1)` for a nested parse operation.
+- Add `p_input_index()` API to get the current input text byte offset.
+- Add `p_set_input_index()` API to set the current input text byte offset.
+  Useful together with `p_set_position()` to rewind the input part-way through
+  a parse in order to re-read an earlier section of the input.
+
+## v4.7.0
+
+### New Features
+
+- Support parser rule user code blocks in tree generation mode.
+
+### Fixes
+
+- propane.vim: do not highlight rule components as propane keywords
+
+## v4.6.0
+
+### New Features
+
+- Add lexer user code API to access matched input text positions
+- Track rule component text positions and add parser user code API to access
+
+### Fixes
+
+- Fixed a few user guide and source comments related to text input positions
+
+## v4.5.0
+
+### New Features
+
+- Add `noline` grammar statement to skip emitting `#line` directives
+- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
+
+### Fixes
+
+- Fix #line reset directives
+- Update keyword list in extra/vim/syntax/propane.vim
+- Fix propane.vim keyword detection
+
+## v4.4.0
+
+### New Features
+
+- Add p_value_get() / p_value_get_XXX() accessors
+
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
 ## v4.1.0
 
 ### New Features

Gemfile.lock

@@ -5,11 +5,11 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.1)
-    psych (5.3.1)
+    erb (6.0.4)
+    psych (5.4.0)
       date
       stringio
-    rake (13.3.1)
+    rake (13.4.2)
     rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.14

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.c.erb

@@ -486,6 +486,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     case P_SUCCESS:
         {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
             uint8_t const * match = &context->input[context->input_index];
@@ -524,17 +540,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         }
         return P_SUCCESS;
@@ -707,12 +712,14 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 } state_value_t;
 
@@ -866,6 +873,59 @@ static void state_values_stack_free(state_values_stack_t * stack)
 }
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                int stack_index = -1;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
+                    {
+                        return sv->end_position;
+                    }
+                }
+            }
+            else
+            {
+                int stack_index = -(int)n_states;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->position))
+                    {
+                        return sv->position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
+            }
+            else
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
+            }
+        }
+    }
+    <%= @grammar.prefix %>position_t empty_pos;
+    memset(&empty_pos, 0, sizeof(empty_pos));
+    return empty_pos;
+}
+<% end %>
+
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -876,7 +936,7 @@ static void state_values_stack_free(state_values_stack_t * stack)
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -948,8 +1008,17 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-NULL, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional array of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be NULL for a standard parse.
+ * @param n_follow_tokens
+ *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -962,15 +1031,20 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     state_values_stack_init(&statevalues);
@@ -981,7 +1055,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -989,6 +1063,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        for (size_t i = 0u; i < n_follow_tokens; i++)
+        {
+            if (token == follow_tokens[i])
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1008,12 +1094,45 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 result = P_SUCCESS;
                 break;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context->input_index -= token_info.length;
+                    context->text_position = token_info.position;
+<% if @grammar.tree %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+<% else %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
+                    result = P_SUCCESS;
+                    break;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
@@ -1030,9 +1149,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
+                new_state_info->tree_node = token_tree_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+                new_state_info->position = token_info.position;
+                new_state_info->end_position = token_info.end_position;
+                new_state_info->pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
             }
@@ -1040,9 +1161,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
+                new_state_info->tree_node = reduced_parser_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                new_state_info->pvalue = reduced_parser_value;
+                new_state_info->position = reduced_position;
+                new_state_info->end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
@@ -1053,6 +1176,15 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1104,6 +1236,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 reduced_parser_node = NULL;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                state_values_stack_free(&statevalues);
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
@@ -1113,6 +1252,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                memset(&reduced_position, 0, sizeof(reduced_position));
+                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
@@ -1135,14 +1284,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
+}
+
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
 }
 <% end %>
 
@@ -1191,6 +1346,58 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
     return context->text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context->text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context->input_index;
+}
+
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context->input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.d.erb

@@ -59,10 +59,10 @@ public alias <%= @grammar.prefix %>code_point_t = uint;
  */
 public struct <%= @grammar.prefix %>position_t
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint col;
 
     /** Invalid position value. */
@@ -86,6 +86,22 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
+{
+    return pvalue.v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -649,6 +665,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     {
     case P_SUCCESS:
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
@@ -687,17 +719,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -866,12 +887,14 @@ private struct state_value_t
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** Tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 
     this(size_t state_id)
@@ -925,6 +948,55 @@ private immutable parser_state_t[] parser_state_table = [
 ];
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-1-j];
+                    if (sv.end_position.valid)
+                    {
+                        return sv.end_position;
+                    }
+                }
+            }
+            else
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-n_states+j];
+                    if (sv.position.valid)
+                    {
+                        return sv.position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return statevalues[$-1-n_states+i].end_position;
+            }
+            else
+            {
+                return statevalues[$-1-n_states+i].position;
+            }
+        }
+    }
+    return <%= @grammar.prefix %>position_t.INVALID;
+}
+<% end %>
+
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -935,7 +1007,7 @@ private immutable parser_state_t[] parser_state_table = [
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1007,8 +1079,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1021,29 +1101,46 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        foreach (eof_token; follow_tokens)
+        {
+            if (token == eof_token)
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1062,10 +1159,41 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% end %>
                 return P_SUCCESS;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context.input_index -= token_info.length;
+                    context.text_position = token_info.position;
+<% if @grammar.tree %>
+                    context.parse_result = statevalues[$-1].tree_node;
+<% else %>
+                    context.parse_result = statevalues[$-1].pvalue;
+<% end %>
+                    return P_SUCCESS;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             statevalues ~= state_value_t(shift_state);
             if (reduced_rule_set == INVALID_ID)
             {
@@ -1075,6 +1203,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 statevalues[$-1].tree_node = token_tree_node;
 <% else %>
+                statevalues[$-1].position = token_info.position;
+                statevalues[$-1].end_position = token_info.end_position;
                 statevalues[$-1].pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
@@ -1086,6 +1216,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 statevalues[$-1].tree_node = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
+                statevalues[$-1].position = reduced_position;
+                statevalues[$-1].end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
 <% end %>
@@ -1095,6 +1227,15 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
         }
 
         size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1151,6 +1292,12 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 reduced_parser_node = null;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1158,6 +1305,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                reduced_position = <%= @grammar.prefix %>position_t.INVALID;
+                reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
@@ -1177,14 +1334,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 
 public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
+}
+
+public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
 }
 <% end %>
 
@@ -1262,6 +1425,58 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
     return context.text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context.text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context.input_index;
+}
+
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context.input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -45,10 +45,10 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
  */
 typedef struct
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint32_t row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint32_t col;
 } <%= @grammar.prefix %>position_t;
 
@@ -69,6 +69,22 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
+{
+    return pvalue->v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -88,7 +104,7 @@ typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
 <%     next if rule_set.optional? %>
-struct <%= name %>;
+struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
 <%   end %>
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
@@ -196,6 +212,8 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
 <%   end %>
 
 <% if @grammar.tree %>
@@ -219,6 +237,12 @@ void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefi
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
+
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
+
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
+
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -151,12 +151,37 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
-#### C/C++
+Lexer code blocks appear between `<<` and `>>` markers following a `drop`,
+`token`, or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+The input text positions of the matched token can also be accessed from within
+a lexer code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+The start position of the matched token is accessed with `${position}`, and
+the end position of the matched token is accessed with `${end_position}`.
+
+Example:
+
+```
+token integer /\d+/ <<
+  printf("integer token on row %d, col %d\n",
+      ${position}.row, ${position}.col);
+  $$ = parse_integer(match, match_length);
+>>
+```
+
+#### C/C++ lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a pointer points to the text matched by the lexer pattern.
-  * `match_length` - length of the matched text.
+  * `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern.
+  * `match_length` (`size_t`) - length of the matched text.
 
 Example:
 
@@ -174,11 +199,11 @@ token integer /\d+/ <<
 >>
 ```
 
-#### D
+#### D lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a slice containing the text matched by the lexer pattern.
+  * `match` (`string`) - a slice containing the text matched by the lexer pattern.
 
 ```
 ptype ulong;
@@ -194,13 +219,6 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `drop`, `token`, or pattern expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
 ### Parser rule code blocks
 
 Example:
@@ -217,9 +235,73 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...
 
-Parser rule code blocks are not available in tree generation mode.
+The input text positions for the reduced rule and for the individual rule
+components can also be accessed from within a parser rule code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+
+The start position of the overall reduced rule is accessed with
+`${$.position}`, and the end position of the overall reduced rule is accessed
+with `${$.end_position}`.
+
+The start and end positions of an individual rule component are accessed
+positionally with `${N.position}` and `${N.end_position}`, where `N` is the
+1-based index of the component (`${1.position}` for the first component,
+`${2.position}` for the second, and so on).
+
+Example:
+
+```
+Assignment -> ident equals Expr <<
+    printf("assignment on row %d, col %d\n",
+        ${$.position}.row, ${$.position}.col);
+    printf("target identifier ends on row %d, col %d\n",
+        ${1.end_position}.row, ${1.end_position}.col);
+    printf("expression starts on row %d, col %d\n",
+        ${3.position}.row, ${3.position}.col);
+>>
+```
+
+A rule or rule component that allows for an empty match may not have valid
+positions.
+In this case the position should be checked for validity before its `row` and
+`col` fields are used (see `${#p_position_valid}`).
+For C targets this can be accomplished with
+`if (p_position_valid(${$.position}))` and for D targets this can be
+accomplished with `if (${$.position}.valid)`.
+
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
+Parser rule code blocks are still supported in tree generation mode, but they
+behave differently than when tree generation mode is not active.
+The code block for a rule is executed after the rule has been matched and its
+tree node has been fully formed.
+Within the code block, `$$` refers to the tree node for the reduced rule, typed
+as a pointer to that rule's generated tree node structure.
+The tree nodes for the rule components are accessed positionally with `$1`,
+`$2`, `$3`, etc..., each typed as a pointer to the generated tree node structure
+for that component (a rule node or a `Token` node).
+Field aliases (see the "Specifying parser rules" section) may also be used to
+reference a component tree node by name; a field alias behaves identically to
+the positional reference for that component.
+The positional position expansions (`${$.position}`, `${N.position}`, etc...)
+are not available in tree generation mode; the `position` and `end_position`
+fields of the tree nodes can be accessed directly instead.
+
+Example:
+
+```
+tree;
+
+Assignment -> ident equals Expr <<
+    /* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is
+     * the Expr rule node. */
+    printf("assignment on row %d, col %d\n",
+        $$->position.row, $$->position.col);
+    printf("target identifier ends on row %d, col %d\n",
+        $1->end_position.row, $1->end_position.col);
+>>
+```
 
 ##> `context_user_fields` statement - adding custom fields to the context
 
@@ -316,6 +398,96 @@ free_token_node <<
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
+##> `lex_fn` statement - specifying a custom lexer function
+
+Propane generates both a lexer and a parser.
+By default, the parser uses the generated `p_lex()` function directly to
+return information for the next lexed token from the input stream.
+
+However, the user can specify a custom lex function.
+This function may or may not use the Propane generated `p_lex()` function under
+the hood.
+For example, a token sequence could be injected or repeated from a previously
+saved macro definition.
+
+Example (C/C++):
+
+```
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+lex_fn mylexfn;
+```
+
+The `lex_fn` statement takes one argument specifying the name of the custom
+lexer function.
+The user must supply a value for the `token` field of the `p_token_info_t`
+output structure so that the parser knows what token was lexed.
+
+Additionally, if the parser user code makes use of the token's pvalue, then
+the lexer function must supply a value for the `pvalue` field of the
+`p_token_info_t` structure.
+The `p_value()` generated API function could be useful for specifying
+parser values to associate with the lexed token when tree generation is not
+enabled.
+When tree generation is enabled, the `pvalue` field can be set to an instance
+of whatever type the user has defined as the `ptype` type.
+
+The custom lex function returns a result code to the parser.
+Returning `P_SUCCESS` indicates that the function has produced a token in the
+`token` output field and the parse should proceed.
+Returning any other result code stops the parse immediately; the parser
+propagates that code out of `p_parse()` (and the `p_parse_XXX()` and
+`p_parse_inner_XXX()` functions) unchanged.
+This allows a custom lex function to surface an error condition, for example by
+propagating a `P_UNEXPECTED_TOKEN`, `P_UNEXPECTED_INPUT`, or `P_DECODE_ERROR`
+result from a nested `p_parse_inner_XXX()` or `p_lex()` call.
+
+Observe the following contract when returning a result code other than
+`P_SUCCESS`:
+
+* Do not return `P_DROP`.
+  This code is an internal lexer signal and is never returned to the parser by
+  the generated `p_lex()` function.
+  Returning it from a custom lex function would be reported as a spurious parse
+  failure. If drop functionality is required, the custom lex function should
+  loop and return the next non-drop token.
+* Do not return `P_EOF` to signal the end of the input.
+  The end of input is communicated to the parser by returning `P_SUCCESS` with
+  the `token` field set to the end-of-input token (`TOKEN___EOF`), which is what
+  the generated `p_lex()` function does.
+  Returning `P_EOF` aborts the parse rather than allowing it to complete.
+* The `p_token()` and `p_position()` accessors are populated by the parser only
+  when the parser itself detects an unexpected token.
+  When a custom lex function returns an error code, `p_token()` is not updated
+  and may not reflect a meaningful token, and `p_position()` reflects the
+  lexer's current text position rather than a parser-identified error location.
+  A custom lex function that wants a specific reported position can set it with
+  `p_set_position()` before returning.
+* Returning `P_USER_TERMINATED` does not populate the user terminate code
+  returned by `p_user_terminate_code()`. This user terminate code is normally
+  populated by the `$terminate()` function in the user code block.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -328,6 +500,27 @@ module proj.parser;
 If a module statement is not present, then the generated D module will not
 contain a module statement and the default module name will be used.
 
+##> `noline` statement - disabling `#line` directives
+
+By default, Propane emits `#line` directives into the generated output around
+user code blocks.
+These directives instruct the compiler to report any warnings or errors in the
+user code using the file name and line number of the original grammar file,
+rather than the generated output file.
+This makes it easier to locate the source of a compiler diagnostic in the
+grammar file.
+
+The `noline` statement disables the emission of these `#line` directives.
+
+```
+noline;
+```
+
+When the `noline` statement is present, user code blocks are copied to the
+generated output without any surrounding `#line` directives.
+This can be useful when debugging the generated parser itself, or when the
+`#line` directives interfere with other tooling.
+
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
 The `on_token_node` statement can be used to provide code that initializes
@@ -581,7 +774,9 @@ It is recommended to place this statement early in the grammar.
 In tree generation mode various aspects of propane's behavior are changed:
 
   * Only one `ptype` is allowed.
-  * Parser user code blocks are not supported.
+  * Parser user code blocks execute after the rule's tree node has been formed
+  and access the tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser rule
+  code blocks" section).
   * Structure types are generated to represent the parsed tokens and rules as
   defined in the grammar.
   * The parse result from `p_result()` points to a `Start` struct containing
@@ -930,8 +1125,12 @@ The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.
 
-Parser rule code blocks are not allowed and not used when tree generation mode
-is active.
+In tree generation mode, parser rule code blocks access the reduced rule tree
+node and its component tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser
+rule code blocks" section).
+Field aliases may still be used in tree generation mode to reference a component
+tree node by name, behaving identically to the corresponding positional
+reference.
 
 ##> User termination of the lexer or parser
 
@@ -1007,6 +1206,17 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_value_t`
+
+If tree generation mode is enabled, the `p_value_t` type is defined to be the
+type given to the `ptype` statement in the grammar file.
+
+If tree generation mode is not enabled, there could be more than one `ptype`
+given, so the `p_value_t` type is a union of all possible `ptype` types.
+In this case, the API functions `p_value()` and `p_value_XXX()` for each given
+`ptype` name `XXX` are generated to return `p_value_t` instances holding the
+corresponding `ptype`.
+
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -1016,6 +1226,9 @@ The `p_token_info_t` structure contains the following fields:
 * `length` (`size_t`) holds the number of input bytes used by the token.
 * `token` (`p_token_t`) holds the token ID of the lexed token
 * `pvalue` (`p_value_t`) holds the parser value associated with the token.
+  The actual user value can be extracted with `p_value_get(&token_info.pvalue)`
+  for the default value or `p_value_get_XXX(&token_info.pvalue)` for named
+  `ptype` values.
 
 ### Tree Node Types
 
@@ -1189,6 +1402,42 @@ size_t result = p_parse_Statement(context);
 
 In this case, the parser will start parsing with the `Statement` rule.
 
+### `p_parse_inner_XXX`
+
+For each start rule, a `p_parse_inner_XXX()` function is also generated.
+This variant of the parser entry point accepts a caller-provided array of
+"follow tokens" — tokens the caller allows to appear immediately after the
+start rule in some outer grammar context.
+It is useful when embedding a Propane-generated sub-parser within an outer
+parser and the outer parser knows which tokens naturally terminate the
+sub-parse.
+
+For C targets, the signature is (example for a rule named `Statement`):
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    p_token_t const * follow_tokens, size_t n_follow_tokens);
+```
+
+Passing a `NULL` pointer (or a count of zero) makes the function behave
+identically to `p_parse_Statement()`.
+
+For D targets, the signature accepts a slice:
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    const(p_token_t)[] follow_tokens);
+```
+
+Passing `null` for the slice makes the function behave identically to
+`p_parse_Statement()`.
+
+When the parse is completed via a non-EOF follow token, that follow token is
+**not** consumed from the input stream.
+The parser rewinds the input index and text position to the start of the follow
+token so that a subsequent call to `p_lex()` or another parse function sees the
+same token.
+
 ### `p_position_valid`
 
 The `p_position_valid()` function is only generated for C targets.
@@ -1256,10 +1505,67 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token at row %u column %u\n",
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 
+### `p_set_position`
+
+The `p_set_position()` function sets the current text position stored in the
+context.
+This is useful when performing a nested parse operation over a slice of a
+larger enclosing document: by setting the initial text position to something
+other than the default `(1, 1)`, any token positions or error positions
+subsequently reported by the lexer/parser are relative to the enclosing
+document rather than the slice.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_position_t start = { .row = 5, .col = 20 };
+p_set_position(context, start);
+p_parse(context);
+```
+
+### `p_input_index`
+
+The `p_input_index()` function returns the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This is useful for slicing out the remaining input after a partial parse,
+or for locating tokens in the original input buffer.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_parse_inner_Statement(context, follow_tokens, n_follow_tokens);
+size_t offset = p_input_index(context);
+/* Remaining input starts at `input + offset`. */
+```
+
+### `p_set_input_index`
+
+The `p_set_input_index()` function sets the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This moves the lexer's read cursor, which can be used together with
+`p_set_position()` to rewind the input part-way through a parse in order to
+re-read an earlier section of the input.
+The byte offset is not validated; the caller is responsible for providing an
+offset within the bounds of the input text.
+A value previously returned by `p_input_index()` is a suitable argument.
+
+Example:
+
+```
+/* Save the cursor and text position at the start of a section. */
+size_t saved_index = p_input_index(context);
+p_position_t saved_position = p_position(context);
+/* ... later, rewind to re-read that section. */
+p_set_input_index(context, saved_index);
+p_set_position(context, saved_position);
+```
+
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user
@@ -1336,6 +1642,37 @@ p_tree_delete_Statement(statement_tree);
 In this case, Propane will free a `Statement` tree structure returned by the
 `p_parse_Statement(context)` function.
 
+### `p_value`
+
+The `p_value(v)` function builds an instance of the `p_value_t` with the
+default member set to the value of `v`.
+
+A `p_value_XXX(v)` function is defined for each user-defined `ptype` name with
+the user-given name in place of the `XXX`.
+
+These functions are useful for custom lexer functions which need to return a
+parser value corresponding to a lexed token.
+
+They are especially useful when tree generation mode is not active.
+In that case, the `p_value_t` union can hold one of several different possible
+value types.
+
+### `p_value_get`
+
+The `p_value_get()` accessor functions can be used to extract a user value
+from a `p_value_t` union.
+
+The `p_value_get(pvalue)` function accepts a pointer to a `p_value_t` and
+returns the value of its default member.
+
+A `p_value_get_XXX(pvalue)` function is generated for each user-defined `ptype`
+name with the user-given name in place of the `XXX`, returning the value of the
+corresponding member.
+
+These functions are the counterpart to `p_value()` constructor functions and
+are useful for reading the parser value associated with a lexed token, for
+example `p_value_get(&token_info.pvalue)`.
+
 ##> Data
 
 ### `p_token_names`
@@ -1353,7 +1690,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
         p_token_names[context->token],
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 

extra/vim/syntax/propane.vim

@@ -8,7 +8,11 @@ if exists("b:current_syntax")
 endif
 
 if !exists("b:propane_subtype")
-  let b:propane_subtype = "d"
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+    let b:propane_subtype = "d"
+  else
+    let b:propane_subtype = "cpp"
+  endif
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
@@ -16,18 +20,32 @@ exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
 syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
 
 syn match propaneComment "#.*"
-syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
+" The right-hand side of a rule (after '->' up to '<<' or ';') lists symbol
+" names that may coincide with propane keywords (e.g. 'start', 'token',
+" 'tree'). Wrap it in a region that excludes keyword matches so those names
+" are not highlighted as keywords. The '<<' is left unconsumed so the
+" propaneTarget region can still match it.
+syn region propaneRuleRhs matchgroup=propaneOperator start="->" end="\ze<<" end=";" contains=propaneFieldAlias,propaneRuleOperator,propaneComment keepend
+syn match propaneRuleOperator "?" contained
+" Keywords that introduce a user-defined name. The name is consumed by
+" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
+" is not highlighted as a keyword. These must be a match (not syn keyword)
+" because a syn keyword always wins over a contained nextgroup match.
+syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
+syn match propaneName "\<\h\w*\>" contained
+syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
+hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
+hi def link propaneRuleOperator Operator
 hi def link propaneFieldOperator Operator
 hi def link propaneDelimiter Delimiter
 hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
+        grammar = Grammar.new(File.read(input_file), input_file)
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -38,7 +38,13 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        result = erb.result(binding.clone)
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if line == "#linereset\n"
+            %[#line #{i + 2} "#{output_file}"\n]
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -179,7 +185,8 @@ class Propane
                 end
               end
               @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
-              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optcode = @grammar.tree ? nil : "$$ = $1;\n"
+              @grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number)
               optional_rules_added << component
             end
           end
@@ -290,31 +297,41 @@ class Propane
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "_pvalue->v_#{rule.ptypename}"
-          when "d"
-            "_pvalue.v_#{rule.ptypename}"
+          if @grammar.tree
+            case @language
+            when "c"
+              "((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+            when "d"
+              "(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+            end
+          else
+            case @language
+            when "c"
+              "_pvalue->v_#{rule.ptypename}"
+            when "d"
+              "_pvalue.v_#{rule.ptypename}"
+            end
           end
         end
         code = code.gsub(/\$(\d+)/) do |match|
+          parser_component_reference(rule, $1.to_i)
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
           index = $1.to_i
-          case @language
-          when "c"
-            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          when "d"
-            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
-          end
+          "get_rule_position(statevalues, #{index}, n_states, false)"
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, true)"
         end
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
-            case @language
-            when "c"
-              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
-            when "d"
-              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
-            end
+            # Field aliases are just a named reference to a positional rule
+            # component, so reuse the same expansion as `$1', `$2', etc. Note
+            # that rule.aliases stores a 0-based component index, so add 1 to
+            # convert it to the 1-based index used for positional references.
+            parser_component_reference(rule, index + 1)
           else
             raise Error.new("Field alias '#{aliasname}' not found")
           end
@@ -337,6 +354,22 @@ class Propane
             end
           end
         end
+        code = code.gsub(/\$\{position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->position"
+          when "d"
+            "out_token_info.position"
+          end
+        end
+        code = code.gsub(/\$\{end_position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->end_position"
+          when "d"
+            "out_token_info.end_position"
+          end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)
@@ -354,6 +387,53 @@ class Propane
       code
     end
 
+    # Expand a positional reference to a parser rule component.
+    #
+    # This is used to expand `$1', `$2', etc. as well as field aliases (which
+    # are just named references to a positional rule component).
+    #
+    # @param rule [Rule]
+    #   The Rule containing the user code.
+    # @param index [Integer]
+    #   1-based index of the rule component to reference.
+    #
+    # @return [String]
+    #   Expanded rule component reference.
+    def parser_component_reference(rule, index)
+      component = rule.components[index - 1]
+      if @grammar.tree
+        # In tree mode a component reference yields a pointer to that
+        # component's tree node. An optional component propagates its target
+        # node (or null), so use the optional target's node type.
+        if component.is_a?(RuleSet) && component.optional?
+          component = component.option_target
+        end
+        node_name = component.is_a?(Token) ? "Token" : component.name
+        typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
+        case @language
+        when "c"
+          "((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
+        when "d"
+          "(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
+        end
+      else
+        case @language
+        when "c"
+          "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
+        when "d"
+          "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
+        end
+      end
+    end
+
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]

lib/propane/grammar.rb

@@ -6,6 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
+    attr_reader :lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -21,7 +22,8 @@ class Propane
     attr_reader :on_token_node
     attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
       @start_rules = []
       @tokens = []
@@ -56,6 +58,10 @@ class Propane
       @tokens.size + 1
     end
 
+    def parser_user_code_used?
+      @rules.any? {|r| r.code}
+    end
+
     private
 
     def parse_grammar!
@@ -69,6 +75,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -85,6 +92,7 @@ class Propane
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
+      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -117,6 +125,12 @@ class Propane
       end
     end
 
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true
@@ -253,12 +267,8 @@ class Propane
         end
         md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        if @tree
-          consume!(/;/, "expected `;'")
-        else
-          unless code = parse_code_block!
-            consume!(/;/, "expected `;' or code block")
-          end
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @modeline = nil
@@ -296,8 +306,14 @@ class Propane
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = code.chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -315,6 +331,13 @@ class Propane
       end
     end
 
+    def parse_noline_statement!
+      if md = consume!(/noline\s*;/)
+        @noline = true
+        true
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -338,9 +361,14 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = md[1].chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         code
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.1.0"
+  VERSION = "4.8.1"
 end

spec/macros.c.propane

@@ -0,0 +1,25 @@
+<<
+#include <stdlib.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int v);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/macros.d.propane

@@ -0,0 +1,31 @@
+<<
+import test_macros;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+    int n = 0;
+    foreach (c; match)
+    {
+        n *= 10;
+        n += (c - '0');
+    }
+    $$ = n;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -0,0 +1,19 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.c.propane

@@ -0,0 +1,17 @@
+<<
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/parse_inner_nested_tree.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested_tree;
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,42 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
+    end
+
+    it "does not emit #line directives with noline statement" do
+      input = <<EOF
+noline;
+
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input, "test.propane")
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  a = b;\n  return c;]
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  writeln("Hello there");]
     end
 
     it "supports mode labels" do
@@ -144,7 +179,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -197,7 +232,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/propane_spec.rb

@@ -95,13 +95,17 @@ EOF
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
     results = Results.new(stdout, stderr, status)
-    stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
-    vgout = stdout + stderr
-    File.binwrite("spec/run/.vgout", vgout)
-    vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
-      bytes = $1.to_i
-      if bytes > 0
-        raise "Valgrind detected memory leak"
+    # Valgrind is only reliably available on Linux, so limit the leak check to
+    # Linux platforms.
+    if RUBY_PLATFORM =~ /linux/
+      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+      vgout = stdout + stderr
+      File.binwrite("spec/run/.vgout", vgout)
+      vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
+        bytes = $1.to_i
+        if bytes > 0
+          raise "Valgrind detected memory leak"
+        end
       end
     end
     results
@@ -745,6 +749,78 @@ EOF
         ])
       end
 
+      it "executes user code associated with a parser rule in tree mode" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$->n_fields;
+  ${context.start_a_value} = $$->pA->pToken1->pvalue;
+  ${context.a_value} = $1->pToken1->pvalue;
+  ${context.b_value} = $2->pToken1->pvalue;
+  ${context.b_token} = $2->pToken1->token;
+  ${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0;
+  ${context.alias_a_value} = ${ay}->pToken1->pvalue;
+  ${context.alias_b_value} = ${bee}->pToken1->pvalue;
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >>
+EOF
+        when "d"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$.n_fields;
+  ${context.start_a_value} = $$.pA.pToken1.pvalue;
+  ${context.a_value} = $1.pToken1.pvalue;
+  ${context.b_value} = $2.pToken1.pvalue;
+  ${context.b_token} = $2.pToken1.token;
+  ${context.c_field_is_null} = ($$.pC is null) ? 1 : 0;
+  ${context.alias_a_value} = ${ay}.pToken1.pvalue;
+  ${context.alias_b_value} = ${bee}.pToken1.pvalue;
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_parser_user_code_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "parses lists" do
         write_grammar <<EOF
 ptype #{language == "d" ? "uint" : "uint32_t"};
@@ -862,6 +938,44 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows setting the text position via p_set_position()" do
+        write_grammar <<EOF
+token a;
+token b;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_set_position.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "exposes the current input byte offset via p_input_index()" do
+        write_grammar <<EOF
+drop /\\s+/;
+token a;
+token b;
+start Start;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_input_index.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows rewinding the input to repeat a section during a parse" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/rewind.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_rewind.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows creating a JSON parser" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/json_parser.#{ext}.propane"))
@@ -910,6 +1024,57 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows lexer code blocks to access the matched token positions" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        when "d"
+          write_grammar <<EOF
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows the user to terminate the parser" do
         write_grammar <<EOF
 token a;
@@ -1462,6 +1627,95 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "supports parse_inner APIs that treat provided tokens as follow tokens" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+Start -> Y << $$ = $1; >>
+Y -> a << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs block success when the outer rule is unfinished" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> a Start b << $$ = $2; >>
+Start -> c << $$ = $1; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_recursive.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
+    write_grammar <<EOF
+ptype int;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b << $$ = 11; >>
+R2 -> a b << $$ = 22; >>
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_shared.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "parse_inner APIs work in tree generation mode with follow tokens" do
+    write_grammar <<EOF
+tree;
+token a;
+token b;
+start Start;
+start R1;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b;
+R2 -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_parse_inner_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "supports a reentrant nested parse driven from a custom lex function" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/parse_inner_nested.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_parse_inner_nested.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "tracks positions across a lex-function nested parse in tree mode" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/parse_inner_nested_tree.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_parse_inner_nested_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows multiple starting rules in tree mode" do
     write_grammar <<EOF
 tree;
@@ -1699,6 +1953,257 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info.token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
+
+      it "provides accessor functions to extract user values from a parser value" do
+        if language == "d"
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = string;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = "hello"; >>
+Start -> num flt str;
+EOF
+        else
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = char *;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = (char *)"hello"; >>
+Start -> num flt str;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_value_accessors.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows accessing rule and component text positions" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+Start -> ident Num <<
+    writeln("ident start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("ident end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Num start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Num end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("Start start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("Start end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    writeln("Empty start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("Empty end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("tok2 start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("tok2 end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    writeln("tok1 start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("tok1 end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Empty start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Empty end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R2 start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R2 end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    writeln("StartR start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartR end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Start -> Empty <<
+    writeln("StartEmpty start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartEmpty end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+token pct /%/;
+Start -> ident Num <<
+    printf("ident start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("ident end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Num start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Num end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("Start start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("Start end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    printf("Empty start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("Empty end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("tok2 start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("tok2 end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    printf("tok1 start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("tok1 end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Empty start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Empty end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R2 start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R2 end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    printf("StartR start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartR end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Start -> Empty <<
+    printf("StartEmpty start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartEmpty end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to eq <<EOF
+ident start: 1, 5
+ident end: 1, 9
+Num start: 3, 9
+Num end: 3, 12
+Start start: 1, 5
+Start end: 3, 12
+
+Empty start: 0, 0
+Empty end: 0, 0
+tok2 start: 2, 2
+tok2 end: 2, 5
+R start: 2, 2
+R end: 2, 5
+StartR start: 2, 2
+StartR end: 2, 5
+
+tok1 start: 1, 3
+tok1 end: 1, 6
+Empty start: 0, 0
+Empty end: 0, 0
+R2 start: 1, 3
+R2 end: 1, 6
+StartR start: 1, 3
+StartR end: 1, 6
+EOF
+      end
+
+      it "allows nested parses for macro expansions" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/macros.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_macros.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/rewind.c.propane

@@ -0,0 +1,23 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int value);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -0,0 +1,20 @@
+<<
+import test_rewind;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_input_index.c

@@ -0,0 +1,60 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   drop /\\s+/;
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_input_index() reports the parser/lexer's current byte
+     * offset into the input text. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert_eq(0u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        char const * input = "a b";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        /* The dropped space between `a` and `b` advances input_index too. */
+        assert_eq(3u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_input_index.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_input_index.c for details on the grammar and cases. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_input_index(context) == 0);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        string input = "a b";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_input_index(context) == 1);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 3);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+}

spec/test_lexer_positions.c

@@ -0,0 +1,50 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 1u);
+    assert(context->last_start.col == 1u);
+    assert(context->last_end.row == 1u);
+    assert(context->last_end.col == 3u);
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context->last_start.row == token_info.position.row);
+    assert(context->last_start.col == token_info.position.col);
+    assert(context->last_end.row == token_info.end_position.row);
+    assert(context->last_end.col == token_info.end_position.col);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 3u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 6u);
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 8u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 9u);
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context->text_position.row == 3u);
+    assert(context->text_position.col == 1u);
+
+    p_context_delete(context);
+    return 0;
+}

spec/test_lexer_positions.d

@@ -0,0 +1,42 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new(input);
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(1, 1));
+    assert(context.last_end == p_position_t(1, 3));
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context.last_start == token_info.position);
+    assert(context.last_end == token_info.end_position);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 3));
+    assert(context.last_end == p_position_t(2, 6));
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 8));
+    assert(context.last_end == p_position_t(2, 9));
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context.text_position == p_position_t(3, 1));
+}

spec/test_macros.c

@@ -0,0 +1,118 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+#include <assert.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+static p_context_t * context;
+size_t n_tokens;
+p_token_info_t token_infos[10];
+
+/* Capture the macro body tokens (everything up to the closing '}') into
+ * token_infos[]. Called from mylexfn() right after the definition's '{' has
+ * been lexed, so the input cursor is positioned at the first body token. */
+static void capture_macro_body(void)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert_eq(result, P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < sizeof(token_infos) / sizeof(token_infos[0]));
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info->token)
+        {
+        case TOKEN_macro:
+            /* Start of a macro definition: "macro macroname { ... }". */
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                /* Use of a macro: replay its captured body tokens instead of
+                 * returning the macroname to the parser. */
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            /* Definition name: pass through and keep waiting for '{'. */
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                /* Consume and store the macro body now, before the parser gets
+                 * a chance to read its lookahead token (which would otherwise
+                 * swallow the first body token). */
+                capture_macro_body();
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int nums[10];
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    char const * input =
+        "macro @m { 23 + 200 }\n"
+        "66 + 100\n"
+        "@m\n"
+        "33 + 55\n"
+        "@m\n";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(n_nums, 4);
+    assert_eq(nums[0], 166);
+    assert_eq(nums[1], 223);
+    assert_eq(nums[2], 88);
+    assert_eq(nums[3], 223);
+
+    return 0;
+}

spec/test_macros.d

@@ -0,0 +1,116 @@
+import testparser;
+import testutils;
+
+size_t n_tokens;
+p_token_info_t[10] token_infos;
+
+// Capture the macro body tokens (everything up to the closing '}') into
+// token_infos[]. Called from mylexfn() right after the definition's '{' has
+// been lexed, so the input cursor is positioned at the first body token.
+void capture_macro_body(p_context_t * context)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert(result == P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < token_infos.length);
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info.token)
+        {
+        case TOKEN_macro:
+            // Start of a macro definition: "macro macroname { ... }".
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                // Use of a macro: replay its captured body tokens instead of
+                // returning the macroname to the parser.
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            // Definition name: pass through and keep waiting for '{'.
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                // Consume and store the macro body now, before the parser gets
+                // a chance to read its lookahead token (which would otherwise
+                // swallow the first body token).
+                capture_macro_body(context);
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int[10] nums;
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "macro @m { 23 + 200 }\n" ~
+        "66 + 100\n" ~
+        "@m\n" ~
+        "33 + 55\n" ~
+        "@m\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert(n_nums == 4);
+    assert(nums[0] == 166);
+    assert(nums[1] == 223);
+    assert(nums[2] == 88);
+    assert(nums[3] == 223);
+}

spec/test_parse_inner.c

@@ -0,0 +1,73 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (chain reduce):
+     *   Start -> Y      << $$ = $1; >>
+     *   Y     -> a      << $$ = $1; >>
+     *   token a         << $$ = 1; >>
+     *
+     * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF,
+     * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with
+     * `b` as a follow token should succeed via the reduce-side retry chain
+     * (Y then Start) followed by the shift-side retry hitting $EOF at the
+     * final state. */
+
+    /* Standard parse succeeds on complete input. */
+    char const * input = "a";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(1u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with an empty (NULL) follow-token vector behaves like a
+     * standard parse. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    {
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+    }
+    p_context_delete(context);
+
+    /* parse_inner with a non-matching follow token still fails. The grammar
+     * can't consume `b` and it isn't listed as a follow token, so the retries
+     * do not fire. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner.c for details on the grammar and cases. */
+
+    /* Standard parse succeeds on complete input. */
+    string input = "a";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a null follow-token slice behaves like a standard
+     * parse. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a non-matching follow token still fails. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_eof = [TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN);
+}

spec/test_parse_inner_nested.c

@@ -0,0 +1,78 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (integer evaluator; parentheses handled by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/;
+ *   token num /\d+/          << ... atoi ... >>
+ *   Start -> Expr            << $$ = $1; >>
+ *   Expr  -> num             << $$ = $1; >>
+ *   Expr  -> Expr plus num   << $$ = $1 + $3; >>
+ *
+ * The tokens lparen and rparen appear in no grammar rule. Instead, when the
+ * lex function lexes a '(', it performs a nested parse (p_parse_inner_Start)
+ * of the parenthesized sub-expression -- reentrantly, while the outer parse is
+ * still suspended in this callback -- reads the computed value with
+ * p_result_Start, consumes the ')' that p_parse_inner deliberately left in the
+ * input, and hands a single synthesized num token carrying that value back to
+ * the outer parse. Nested groups recurse this process to arbitrary depth. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->pvalue = p_value(value);
+    }
+    return P_SUCCESS;
+}
+
+static int eval(char const * input)
+{
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5u, (size_t)eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3u, (size_t)eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14u, (size_t)eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37u, (size_t)eval("2 + (10 + (20 + 5))"));
+    assert_eq(15u, (size_t)eval("(1 + 2) + (3 + (4 + 5))"));
+
+    return 0;
+}

spec/test_parse_inner_nested.d

@@ -0,0 +1,64 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue = p_value(value);
+    }
+    return P_SUCCESS;
+}
+
+int eval(string input)
+{
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5, eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3, eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14, eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37, eval("2 + (10 + (20 + 5))"));
+    assert_eq(15, eval("(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested_tree.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (tree generation mode; parentheses handled by the lex function):
+ *   tree;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/; token num /\d+/;
+ *   Start -> Expr;
+ *   Expr  -> num;
+ *   Expr  -> Expr plus num;
+ *
+ * The same lexer-driven nested parse as test_parse_inner_nested, but in tree
+ * generation mode. Each "( ... )" group is parsed by a reentrant
+ * p_parse_inner_Start() call from the lex function; the resulting subtree is
+ * discarded and a single synthesized num token is handed to the outer parse.
+ * The synthesized token's position is set to span the whole group ('(' start
+ * through ')' end), so this verifies that positions survive the nested-parse
+ * boundary and land correctly in the outer tree. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info->position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start * inner = p_result_Start(context);
+        assert_not_null(inner);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq((size_t)(start_position.col + 1u), (size_t)inner->position.col);
+        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)inner->end_position.col);
+        p_tree_delete_Start(inner);
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->position = start_position;
+        out_token_info->end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    char const * input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start * tree = p_result(context);
+    assert_not_null(tree);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr * top = tree->pExpr;
+    assert_not_null(top);
+    assert_not_null(top->pExpr);
+    assert_not_null(top->pToken2);
+    assert_not_null(top->pToken3);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, (size_t)top->pToken2->position.row);
+    assert_eq(9u, (size_t)top->pToken2->position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, (size_t)top->pToken3->position.row);
+    assert_eq(11u, (size_t)top->pToken3->position.col);
+    assert_eq(1u, (size_t)top->pToken3->end_position.row);
+    assert_eq(17u, (size_t)top->pToken3->end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr * left = top->pExpr;
+    assert_not_null(left->pToken1);
+    assert_eq(1u, (size_t)left->pToken1->position.row);
+    assert_eq(1u, (size_t)left->pToken1->position.col);
+    assert_eq(1u, (size_t)left->pToken1->end_position.row);
+    assert_eq(7u, (size_t)left->pToken1->end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, (size_t)tree->position.col);
+    assert_eq(17u, (size_t)tree->end_position.col);
+
+    p_tree_delete(tree);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_nested_tree.d

@@ -0,0 +1,91 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested_tree.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info.position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start * inner = p_result_Start(context);
+        assert(inner !is null);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq(start_position.col + 1u, inner.position.col);
+        assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
+        p_tree_delete_Start(inner);
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    string input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start * tree = p_result(context);
+    assert(tree !is null);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr * top = tree.pExpr;
+    assert(top !is null);
+    assert(top.pExpr !is null);
+    assert(top.pToken2 !is null);
+    assert(top.pToken3 !is null);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, top.pToken2.position.row);
+    assert_eq(9u, top.pToken2.position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, top.pToken3.position.row);
+    assert_eq(11u, top.pToken3.position.col);
+    assert_eq(1u, top.pToken3.end_position.row);
+    assert_eq(17u, top.pToken3.end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr * left = top.pExpr;
+    assert(left.pToken1 !is null);
+    assert_eq(1u, left.pToken1.position.row);
+    assert_eq(1u, left.pToken1.position.col);
+    assert_eq(1u, left.pToken1.end_position.row);
+    assert_eq(7u, left.pToken1.end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, tree.position.col);
+    assert_eq(17u, tree.end_position.col);
+
+    p_tree_delete(tree);
+    p_context_delete(context);
+}

spec/test_parse_inner_recursive.c

@@ -0,0 +1,77 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (recursive):
+     *   Start -> a Start b   << $$ = $2; >>
+     *   Start -> c           << $$ = $1; >>
+     *   token a << $$ = 1; >>
+     *   token b << $$ = 2; >>
+     *   token c << $$ = 3; >>
+     *
+     * Here `Start` can appear in the middle of another `Start` rule, so the
+     * inner-parse follow-token success must be blocked whenever an unfinished
+     * outer `Start -> a Start b` remains on the parse stack (i.e. the parse
+     * stack contains more than just the initial state and the reduced start
+     * rule set). */
+
+    /* Standard parse of `c` succeeds. */
+    char const * input = "c";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `acb` succeeds (full outer rule). */
+    input = "acb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `ac` fails (`b` missing). */
+    input = "ac";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner with `ac` and follow token `b` also fails: even though the
+     * inner `Start -> c` reduces and `Start` is shifted, the outer
+     * `Start -> a Start . b` is still on the stack (stack length > 2), so the
+     * "reduced start rule is the only thing on the parse stack" invariant
+     * blocks the shift-side follow-token success. */
+    {
+        input = "ac";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    /* parse_inner with `acb` (complete outer rule) succeeds via the standard
+     * path. */
+    {
+        input = "acb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with just `c` succeeds via the standard path even when a
+     * follow-token vector is supplied. */
+    {
+        input = "c";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_recursive.d

@@ -0,0 +1,49 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_recursive.c for details on the grammar. */
+
+    /* Standard parse of `c` succeeds. */
+    string input = "c";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `acb` succeeds. */
+    input = "acb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `ac` fails. */
+    input = "ac";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `ac` fails: outer rule still on the stack. */
+    input = "ac";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `acb` succeeds via the standard path. */
+    input = "acb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* parse_inner with just `c` succeeds via the standard path. */
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+}

spec/test_parse_inner_shared.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar:
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b   << $$ = 11; >>
+     *   R2 -> a b   << $$ = 22; >>
+     *   token a; token b;
+     *
+     * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within
+     * parse_Start, the generated parser differentiates the reduce by
+     * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2
+     * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional
+     * on any lookahead. This test exercises p_parse_inner_R1() to confirm
+     * that reductions to R1 succeed even when the incoming follow token is
+     * not the natural lookahead used by parse_Start's disambiguation. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the
+     * shared "a b" state. */
+    char const * input = "aba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    /* Standard parse of R1 fails on "abb" (unexpected trailing token). */
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
+     * that parse_Start uses to select R2 over R1 in the ambiguous state, from
+     * R1's start state the reduce to R1 is unconditional, and the follow-
+     * token shift retry at the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed from
+     * the input: p_position() should point to the follow token, and a
+     * subsequent p_lex() should return it. */
+    {
+        input = "abb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `b` is at column 3 (1-based). */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token
+     * parse_Start uses to select R1, and it works here as a follow token
+     * too. */
+    {
+        input = "aba";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_a };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `a` is at column 3 (1-based) and remains in the
+         * input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_shared.d

@@ -0,0 +1,72 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_shared.c for details on the grammar. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */
+    string input = "aba";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* Standard parse of R1 fails on "abb". */
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
+     * parse_Start would use to select R2 over R1, but from R1's own start
+     * state R1 reduces unconditionally, and the follow-token shift retry at
+     * the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed: a
+     * subsequent p_lex() should return it. */
+    input = "abb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    p_position_t pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    p_token_info_t token_info;
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_b);
+    assert(token_info.position.row == 1);
+    assert(token_info.position.col == 3);
+
+    /* parse_inner_R1("aba", [a]) also succeeds. */
+    input = "aba";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_a = [TOKEN_a];
+    assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_a);
+
+    /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_R1(context, null) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+}

spec/test_parse_inner_tree.c

@@ -0,0 +1,89 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (tree generation mode, shared reduce state):
+     *   tree;
+     *   token a; token b;
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b;
+     *   R2 -> a b;
+     *
+     * Exercises p_parse_inner_R1() with a non-EOF follow token in tree
+     * generation mode. Verifies:
+     *   * The reduced tree for R1 is well-formed after a follow-token
+     *     completion.
+     *   * The follow token is not consumed and remains available for a
+     *     subsequent p_lex() call.
+     *   * p_tree_delete_R1() cleans up the returned tree without leaks
+     *     (verified in CI via valgrind). */
+
+    /* Baseline: p_parse_R1 works on "ab" and the returned tree is
+     * well-formed. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 * tree = p_result_R1(context);
+        assert_not_null(tree);
+        assert_not_null(tree->pToken1);
+        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
+        assert_not_null(tree->pToken2);
+        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
+        p_tree_delete_R1(tree);
+        p_context_delete(context);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 * tree = p_result_R1(context);
+        assert_not_null(tree);
+        assert_not_null(tree->pToken1);
+        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
+        assert_eq(1u, (size_t)tree->pToken1->position.row);
+        assert_eq(1u, (size_t)tree->pToken1->position.col);
+        assert_not_null(tree->pToken2);
+        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
+        assert_eq(1u, (size_t)tree->pToken2->position.row);
+        assert_eq(2u, (size_t)tree->pToken2->position.col);
+
+        /* The R1 tree covers positions 1..2 — the third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert_eq(1u, (size_t)tree->position.row);
+        assert_eq(1u, (size_t)tree->position.col);
+        assert_eq(1u, (size_t)tree->end_position.row);
+        assert_eq(2u, (size_t)tree->end_position.col);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+
+        /* p_tree_delete_R1 must free every node reachable from the tree
+         * without leaking anything. valgrind (invoked by the spec runner on
+         * Linux) will detect any missed frees. */
+        p_tree_delete_R1(tree);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_tree.d

@@ -0,0 +1,68 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_tree.c for details on the grammar and cases. */
+
+    /* Baseline: p_parse_R1 works on "ab". */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 * tree = p_result_R1(context);
+        assert(tree !is null);
+        assert(tree.pToken1 !is null);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.token == TOKEN_b);
+        p_tree_delete_R1(tree);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 * tree = p_result_R1(context);
+        assert(tree !is null);
+        assert(tree.pToken1 !is null);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken1.position.row == 1);
+        assert(tree.pToken1.position.col == 1);
+        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.token == TOKEN_b);
+        assert(tree.pToken2.position.row == 1);
+        assert(tree.pToken2.position.col == 2);
+
+        /* The R1 tree covers positions 1..2. The third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert(tree.position.row == 1);
+        assert(tree.position.col == 1);
+        assert(tree.end_position.row == 1);
+        assert(tree.end_position.col == 2);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 3);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 3);
+
+        p_tree_delete_R1(tree);
+    }
+}

spec/test_parser_user_code_tree.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context->start_n_fields);
+    assert_eq(11, context->start_a_value);
+    assert_eq(11, context->a_value);
+    assert_eq(22, context->b_value);
+    assert_eq(TOKEN_b, context->b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context->c_is_null);
+    assert_eq(1, context->c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context->alias_a_value);
+    assert_eq(22, context->alias_b_value);
+
+    Start * start = p_result(context);
+    assert(start->pA != NULL);
+    assert(start->pB != NULL);
+    assert(start->pC == NULL);
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parser_user_code_tree.d

@@ -0,0 +1,39 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context.start_n_fields);
+    assert_eq(11, context.start_a_value);
+    assert_eq(11, context.a_value);
+    assert_eq(22, context.b_value);
+    assert_eq(TOKEN_b, context.b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context.c_is_null);
+    assert_eq(1, context.c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context.alias_a_value);
+    assert_eq(22, context.alias_b_value);
+
+    Start * start = p_result(context);
+    assert(start.pA !is null);
+    assert(start.pB !is null);
+    assert(start.pC is null);
+    p_tree_delete(start);
+}

spec/test_positions.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "\n tok2";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "  tok1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_positions.d

@@ -0,0 +1,27 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "\n tok2";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "  tok1";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_rewind.c

@@ -0,0 +1,124 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (statement list of additions; a "repeat" directive handled entirely
+ * by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token repeat /repeat/; token lbrace /\{/; token rbrace /\}/;
+ *   token plus /\+/; token num /\d+/ << ... atoi ... >>
+ *   Start      -> Statements;
+ *   Statements -> ;
+ *   Statements -> Statement Statements;
+ *   Statement  -> Add;
+ *   Add        -> num plus num  << record($1 + $3); >>
+ *
+ * Scenario: a "repeat <count> { <body> }" directive that expands its body
+ * <count> times, similar to loop unrolling in a configuration DSL. The tokens
+ * repeat, lbrace, and rbrace appear in no grammar rule; the lex function
+ * interprets the directive and feeds the body's tokens to the parser <count>
+ * times. Rather than buffering the body tokens, the lex function records the
+ * input byte offset and text position at the start of the body (with
+ * p_input_index() and p_position()) and, each time it reaches the closing '}',
+ * rewinds the lexer back to that point (with p_set_input_index() and
+ * p_set_position()) to re-read the body from the original input. Rewinding the
+ * text position as well as the byte offset means each expansion reports the
+ * same token positions as the first. */
+
+static int nums[16];
+static size_t n_nums;
+static uint32_t num_cols[16];
+static size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info->token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info->token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info->token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info->position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    char const * input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* The additions were recorded once per body expansion, then once more for
+     * the trailing statement. */
+    assert_eq(4u, n_nums);
+    assert_eq(30u, (size_t)nums[0]);
+    assert_eq(30u, (size_t)nums[1]);
+    assert_eq(30u, (size_t)nums[2]);
+    assert_eq(10u, (size_t)nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, (size_t)num_cols[0]);
+    assert_eq(17u, (size_t)num_cols[1]);
+    assert_eq(12u, (size_t)num_cols[2]);
+    assert_eq(17u, (size_t)num_cols[3]);
+    assert_eq(12u, (size_t)num_cols[4]);
+    assert_eq(17u, (size_t)num_cols[5]);
+    assert_eq(22u, (size_t)num_cols[6]);
+    assert_eq(26u, (size_t)num_cols[7]);
+
+    return 0;
+}

spec/test_rewind.d

@@ -0,0 +1,101 @@
+import testparser;
+import testutils;
+
+/* Grammar and scenario: see test_rewind.c. */
+
+int[16] nums;
+size_t n_nums;
+uint[16] num_cols;
+size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info.token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info.token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info.token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info.position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    string input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(4u, n_nums);
+    assert_eq(30, nums[0]);
+    assert_eq(30, nums[1]);
+    assert_eq(30, nums[2]);
+    assert_eq(10, nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, num_cols[0]);
+    assert_eq(17u, num_cols[1]);
+    assert_eq(12u, num_cols[2]);
+    assert_eq(17u, num_cols[3]);
+    assert_eq(12u, num_cols[4]);
+    assert_eq(17u, num_cols[5]);
+    assert_eq(22u, num_cols[6]);
+    assert_eq(26u, num_cols[7]);
+}

spec/test_set_position.c

@@ -0,0 +1,109 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_set_position() overrides the default (1, 1) starting
+     * position so that lexed tokens and error positions are reported
+     * relative to the caller-supplied position. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(1u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() overrides the initial position; subsequent lex calls
+     * report token positions relative to the set position. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {5u, 20u};
+        p_set_position(context, initial);
+        p_position_t pos = p_position(context);
+        assert_eq(5u, (size_t)pos.row);
+        assert_eq(20u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(20u, (size_t)token_info.position.col);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(21u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a full parse: successful parse still works and
+     * text_position tracking is relative to the set starting point. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {3u, 7u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a parse that fails: the reported error
+     * position is relative to the set starting point. */
+    {
+        char const * input = "aa";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {10u, 2u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        /* Error is at the second `a`, which is one column past the initial
+         * column. */
+        assert_eq(10u, (size_t)err_pos.row);
+        assert_eq(3u, (size_t)err_pos.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input: both
+     * tokens are lexed, then the cursor and text position are rewound to the
+     * start so that the same tokens are produced again with the same reported
+     * positions. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert_eq(0u, start_index);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(2u, p_input_index(context));
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert_eq(0u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_set_position.d

@@ -0,0 +1,90 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_set_position.c for details. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 1);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+
+    /* p_set_position() overrides the initial position. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(5u, 20u));
+        p_position_t pos = p_position(context);
+        assert(pos.row == 5);
+        assert(pos.col == 20);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 20);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 21);
+    }
+
+    /* p_set_position() before a full parse still parses successfully. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(3u, 7u));
+        assert(p_parse_Start(context) == P_SUCCESS);
+    }
+
+    /* p_set_position() before a parse that fails: error position is
+     * relative to the set starting point. */
+    {
+        string input = "aa";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(10u, 2u));
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        assert(err_pos.row == 10);
+        assert(err_pos.col == 3);
+    }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert(start_index == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 2);
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert(p_input_index(context) == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+}

spec/test_value_accessors.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    p_token_info_t token_info;
+    char const * input = "42 f s";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(strcmp(p_value_get_string(&token_info.pvalue), "hello") == 0);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_value_accessors.d

@@ -0,0 +1,29 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    p_token_info_t token_info;
+    string input = "42 f s";
+    p_context_t * context;
+    context = p_context_new(input);
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(p_value_get_string(&token_info.pvalue) == "hello");
+}