Add Rust target

Store tree nodes in congruent, compact arena array.
Define handle types to refer to tree nodes rather than pointers to
structure instances.
Free tree with context.

.github/workflows/run-tests.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - name: Install dependencies (Linux)
       if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc valgrind
 
     - name: Install dependencies (macOS)
       if: runner.os == 'macOS'

CHANGELOG.md

@@ -1,3 +1,115 @@
+## v5.0.0
+
+### New Features
+
+- Add Rust target language output.
+
+### API Changes
+
+- Tree generation mode now stores all tree nodes in a compact arena owned by
+  the parser context (a flat node array plus a shared child-link array).
+  This replaces the previous design of one heap allocation per node with
+  layout-punned typed structs.
+- Tree nodes are now referenced by lightweight handles rather than pointers.
+  `p_result()` and the field accessors return handle values.
+- The whole tree is freed together with the context by `p_context_delete()`.
+  The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
+  tree node handles are only valid while the context is alive.
+- Tree node field access changed per target language:
+  - C: per-field accessor functions (e.g. `p_Start_pItems(node)`) plus tree
+    walk macros (e.g. `p_tree_walk_Start(node, pItems, pItem, pToken1, token)`),
+    and generic accessors `p_node_valid()`, `p_node_position()`,
+    `p_node_end_position()`, `p_node_n_fields()`, `p_node_data()`, `p_node_id()`.
+  - C++: handle methods called with `()` (e.g. `node.pItems().pToken1().token()`),
+    plus the same C-style functions/macros for convenience.
+  - D: `@property` accessors preserving the previous field-access syntax
+    (e.g. `node.pItems.pToken1.token`); null checks use `.valid` instead of
+    `is null`.
+- Tree-mode parser rule user code: `$$` and `$1` etc. now yield node handles.
+  Reference child fields through the target-language accessors described above
+  rather than through struct pointer members.
+
+## v4.8.1
+
+### Fixes
+
+- Fix tree node struct type forward-declarations for C/C++
+
+## v4.8.0
+
+### New Features
+
+- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow
+  tokens. These behave the same as `p_parse_XXX()` by parsing starting at the
+  given start rule, but instead of expecting the rest of the input to match
+  the start rule they allow specifying a set of tokens that may follow the
+  start rule.
+- Add `p_set_position()` API to set the current text position stored in the
+  context. Useful for setting the initial text position to something other
+  than `(1, 1)` for a nested parse operation.
+- Add `p_input_index()` API to get the current input text byte offset.
+- Add `p_set_input_index()` API to set the current input text byte offset.
+  Useful together with `p_set_position()` to rewind the input part-way through
+  a parse in order to re-read an earlier section of the input.
+
+## v4.7.0
+
+### New Features
+
+- Support parser rule user code blocks in tree generation mode.
+
+### Fixes
+
+- propane.vim: do not highlight rule components as propane keywords
+
+## v4.6.0
+
+### New Features
+
+- Add lexer user code API to access matched input text positions
+- Track rule component text positions and add parser user code API to access
+
+### Fixes
+
+- Fixed a few user guide and source comments related to text input positions
+
+## v4.5.0
+
+### New Features
+
+- Add `noline` grammar statement to skip emitting `#line` directives
+- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
+
+### Fixes
+
+- Fix #line reset directives
+- Update keyword list in extra/vim/syntax/propane.vim
+- Fix propane.vim keyword detection
+
+## v4.4.0
+
+### New Features
+
+- Add p_value_get() / p_value_get_XXX() accessors
+
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
+## v4.1.0
+
+### New Features
+
+- Add `p_context_delete()` and `p_tree_delete()` for D targets.
+
 ## v4.0.0
 
 ### New Features

Gemfile.lock

@@ -5,11 +5,11 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.1)
-    psych (5.3.1)
+    erb (6.0.4)
+    psych (5.4.0)
       date
       stringio
-    rake (13.3.1)
+    rake (13.4.2)
     rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.14

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C, C++, or D language outputs
+  * targets C, C++, D, or Rust language outputs
   * optionally supports automatic full parse tree generation
   * supports starting parsing from multiple start rules
   * tracks input text start and end positions for all matched tokens/rules

UPGRADING.md

@@ -1,3 +1,43 @@
+## v5.0.0
+
+The generated API for tree generation mode (`tree;`) has been changed
+significantly for this version.
+The lexer/parser value APIs for non-tree grammars are unchanged.
+
+### Tree memory management
+
+- Remove all calls to `p_tree_delete()` / `p_tree_delete_XXX()`. Tree nodes now
+  live in the parser context and are freed by `p_context_delete()`.
+- Tree node handles (returned by `p_result()` and the field accessors) are only
+  valid while the context is alive. Do not use them after `p_context_delete()`.
+
+### Tree node field access
+
+Tree nodes are now referenced by handle values instead of pointers, and field
+access differs per target language:
+
+- C: replace `node->field` with the accessor function `p_TYPE_field(node)`, or
+  use the tree walk macro `p_tree_walk_TYPE(node, field1, field2, ...)`. Replace
+  `x != NULL` / `x == NULL` node checks with `p_node_valid(x)` /
+  `!p_node_valid(x)`. Read positions with `p_node_position(node)` /
+  `p_node_end_position(node)`, token payload with `p_TYPE_token(node)` /
+  `p_TYPE_pvalue(node)` or `p_node_data(node)->field`, and compare node identity
+  with `p_node_id(a) == p_node_id(b)`.
+- C++: replace `node->field` with the handle method `node.field()`. Use
+  `node.valid()`, `node.position()`, `node.token()`, `node.pvalue()`, and
+  `node.data()->field` for user token fields. (The C-style functions and macros
+  above are also available.)
+- D: replace pointer declarations (`Start * s`) with value handles (`Start s`)
+  and replace `x !is null` / `x is null` with `x.valid` / `!x.valid`. Field
+  access syntax (`node.field.field`) is otherwise unchanged.
+
+### Tree-mode parser rule user code
+
+In tree generation mode `$$` and `$1`, `$2`, ... now expand to node handles.
+Reference child fields through the target-language accessors above (for example
+`$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
+in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
+
 ## v4.0.0
 
 ### API Changes

assets/parser.c.erb

@@ -68,6 +68,18 @@ const char * <%= @grammar.prefix %>token_names[] = {
     context->text_position.row = 1u;
     context->text_position.col = 1u;
     context->mode = <%= @lexer.mode_id("default") %>;
+<% if @grammar.tree %>
+
+    /* Reserve node ID 0 as the null tree node. */
+<%   if @cpp %>
+    context-><%= @grammar.prefix %>tree_nodes.resize(1);
+<%   else %>
+    context-><%= @grammar.prefix %>tree_nodes_capacity = 16u;
+    context-><%= @grammar.prefix %>tree_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(16u * sizeof(<%= @grammar.prefix %>node_data_t));
+    memset(&context-><%= @grammar.prefix %>tree_nodes[0], 0, sizeof(<%= @grammar.prefix %>node_data_t));
+    context-><%= @grammar.prefix %>tree_nodes_length = 1u;
+<%   end %>
+<% end %>
 
     return context;
 }
@@ -84,9 +96,27 @@ const char * <%= @grammar.prefix %>token_names[] = {
  */
 void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
 {
+<% if @grammar.tree && @grammar.free_token_node != "" %>
+<%   if @cpp %>
+    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes.size(); i++)
+<%   else %>
+    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes_length; i++)
+<%   end %>
+    {
+        if (context-><%= @grammar.prefix %>tree_nodes[i].is_token)
+        {
+            <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[i];
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+        }
+    }
+<% end %>
 <% if @cpp %>
     delete context;
 <% else %>
+<%   if @grammar.tree %>
+    free(context-><%= @grammar.prefix %>tree_nodes);
+    free(context-><%= @grammar.prefix %>tree_children);
+<%   end %>
     free(context);
 <% end %>
 }
@@ -486,6 +516,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     case P_SUCCESS:
         {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
             uint8_t const * match = &context->input[context->input_index];
@@ -524,17 +570,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         }
         return P_SUCCESS;
@@ -707,26 +742,16 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
-    /** tree node. */
-    void * tree_node;
-<% end %>
-} state_value_t;
-
-<% if @grammar.tree %>
-/** Common tree node structure. */
-typedef struct TreeNode_s
-{
+    /** Tree node ID. */
+    <%= @grammar.prefix %>node_id_t node_id;
+<% else %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-    uint16_t n_fields;
-    uint8_t is_token;
-    struct TreeNode_s * fields[];
-} TreeNode;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
+} state_value_t;
 
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
@@ -865,7 +890,131 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
+<% if @grammar.tree %>
+/* Tree arena helpers. */
+
+/**
+ * Allocate a new (zeroed) tree node in the context arena.
+ *
+ * @return The new node ID.
+ */
+static <%= @grammar.prefix %>node_id_t tree_new_node(<%= @grammar.prefix %>context_t * context)
+{
+<% if @cpp %>
+    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes.size();
+    context-><%= @grammar.prefix %>tree_nodes.emplace_back();
+    return id;
+<% else %>
+    if (context-><%= @grammar.prefix %>tree_nodes_length >= context-><%= @grammar.prefix %>tree_nodes_capacity)
+    {
+        size_t new_capacity = context-><%= @grammar.prefix %>tree_nodes_capacity * 2u;
+        <%= @grammar.prefix %>node_data_t * new_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_data_t));
+        memcpy(new_nodes, context-><%= @grammar.prefix %>tree_nodes, context-><%= @grammar.prefix %>tree_nodes_length * sizeof(<%= @grammar.prefix %>node_data_t));
+        free(context-><%= @grammar.prefix %>tree_nodes);
+        context-><%= @grammar.prefix %>tree_nodes = new_nodes;
+        context-><%= @grammar.prefix %>tree_nodes_capacity = new_capacity;
+    }
+    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes_length;
+    memset(&context-><%= @grammar.prefix %>tree_nodes[id], 0, sizeof(<%= @grammar.prefix %>node_data_t));
+    context-><%= @grammar.prefix %>tree_nodes_length += 1u;
+    return id;
+<% end %>
+}
+
+/**
+ * Reserve n contiguous (zeroed) child slots in the shared children array.
+ *
+ * @return The offset of the first reserved slot.
+ */
+static <%= @grammar.prefix %>node_id_t tree_reserve_children(<%= @grammar.prefix %>context_t * context, size_t n)
+{
+<% if @cpp %>
+    <%= @grammar.prefix %>node_id_t offset = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_children.size();
+    context-><%= @grammar.prefix %>tree_children.resize(context-><%= @grammar.prefix %>tree_children.size() + n);
+    return offset;
+<% else %>
+    size_t offset = context-><%= @grammar.prefix %>tree_children_length;
+    size_t needed = offset + n;
+    if (needed > context-><%= @grammar.prefix %>tree_children_capacity)
+    {
+        size_t new_capacity = context-><%= @grammar.prefix %>tree_children_capacity ? context-><%= @grammar.prefix %>tree_children_capacity : 1u;
+        while (new_capacity < needed)
+        {
+            new_capacity *= 2u;
+        }
+        <%= @grammar.prefix %>node_id_t * new_children = (<%= @grammar.prefix %>node_id_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_id_t));
+        if (context-><%= @grammar.prefix %>tree_children != NULL)
+        {
+            memcpy(new_children, context-><%= @grammar.prefix %>tree_children, context-><%= @grammar.prefix %>tree_children_length * sizeof(<%= @grammar.prefix %>node_id_t));
+            free(context-><%= @grammar.prefix %>tree_children);
+        }
+        context-><%= @grammar.prefix %>tree_children = new_children;
+        context-><%= @grammar.prefix %>tree_children_capacity = new_capacity;
+    }
+    memset(&context-><%= @grammar.prefix %>tree_children[offset], 0, n * sizeof(<%= @grammar.prefix %>node_id_t));
+    context-><%= @grammar.prefix %>tree_children_length = needed;
+    return (<%= @grammar.prefix %>node_id_t)offset;
+<% end %>
+}
+
+/* Tree node field accessor functions. */
+<%= c_tree_accessor_defs %>
+<% end %>
+
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                int stack_index = -1;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
+                    {
+                        return sv->end_position;
+                    }
+                }
+            }
+            else
+            {
+                int stack_index = -(int)n_states;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->position))
+                    {
+                        return sv->position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
+            }
+            else
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
+            }
+        }
+    }
+    <%= @grammar.prefix %>position_t empty_pos;
+    memset(&empty_pos, 0, sizeof(empty_pos));
+    return empty_pos;
+}
+<% end %>
+
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -876,7 +1025,7 @@ static void state_values_stack_free(state_values_stack_t * stack)
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -948,8 +1097,17 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-NULL, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional array of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be NULL for a standard parse.
+ * @param n_follow_tokens
+ *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -962,15 +1120,20 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
-    void * reduced_parser_node;
+    <%= @grammar.prefix %>node_id_t reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     state_values_stack_init(&statevalues);
@@ -981,7 +1144,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -989,6 +1152,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        for (size_t i = 0u; i < n_follow_tokens; i++)
+        {
+            if (token == follow_tokens[i])
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1001,28 +1176,58 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* Successful parse. */
 <% if @grammar.tree %>
-                context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
                 result = P_SUCCESS;
                 break;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context->input_index -= token_info.length;
+                    context->text_position = token_info.position;
+<% if @grammar.tree %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
+<% else %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
+                    result = P_SUCCESS;
+                    break;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-<%   if @cpp %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
-<%   else %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
-<%   end %>
+                <%= @grammar.prefix %>node_id_t token_node_id = tree_new_node(context);
+                <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[token_node_id];
                 token_tree_node->position = token_info.position;
                 token_tree_node->end_position = token_info.end_position;
                 token_tree_node->n_fields = 0u;
@@ -1030,9 +1235,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
+                new_state_info->node_id = token_node_id;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+                new_state_info->position = token_info.position;
+                new_state_info->end_position = token_info.end_position;
+                new_state_info->pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
             }
@@ -1040,9 +1247,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
+                new_state_info->node_id = reduced_parser_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                new_state_info->pvalue = reduced_parser_value;
+                new_state_info->position = reduced_position;
+                new_state_info->end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
@@ -1053,57 +1262,77 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
 <% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->node_id;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
-                TreeNode * node = (TreeNode *)malloc(bytes);
-                memset(node, 0, bytes);
-                node->position = INVALID_POSITION;
-                node->end_position = INVALID_POSITION;
-                node->n_fields = n_fields;
+                uint16_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                /* Reserve child slots. New slots are zero-initialized
+                 * (null node ID) so absent optional children remain null. */
+                <%= @grammar.prefix %>node_id_t child_offset = tree_reserve_children(context, n_fields);
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
+                        context-><%= @grammar.prefix %>tree_children[child_offset + i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
                     }
                 }
                 else
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
+                        context-><%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
                     }
                 }
+                <%= @grammar.prefix %>node_id_t node_id = tree_new_node(context);
+                <%= @grammar.prefix %>node_data_t * node = &context-><%= @grammar.prefix %>tree_nodes[node_id];
+                node->position = INVALID_POSITION;
+                node->end_position = INVALID_POSITION;
+                node->child_offset = child_offset;
+                node->n_fields = n_fields;
+                node->is_token = 0u;
                 bool position_found = false;
-                for (size_t i = 0; i < n_fields; i++)
+                for (uint16_t i = 0; i < n_fields; i++)
                 {
-                    TreeNode * child = node->fields[i];
-                    if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
+                    <%= @grammar.prefix %>node_id_t child_id = context-><%= @grammar.prefix %>tree_children[child_offset + i];
+                    if ((child_id != 0u) && <%= @grammar.prefix %>position_valid(context-><%= @grammar.prefix %>tree_nodes[child_id].position))
                     {
                         if (!position_found)
                         {
-                            node->position = child->position;
+                            node->position = context-><%= @grammar.prefix %>tree_nodes[child_id].position;
                             position_found = true;
                         }
-                        node->end_position = child->end_position;
+                        node->end_position = context-><%= @grammar.prefix %>tree_nodes[child_id].end_position;
                     }
                 }
-                reduced_parser_node = node;
+                reduced_parser_node = node_id;
             }
             else
             {
-                reduced_parser_node = NULL;
+                reduced_parser_node = 0u;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                state_values_stack_free(&statevalues);
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
@@ -1113,6 +1342,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                memset(&reduced_position, 0, sizeof(reduced_position));
+                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
@@ -1135,14 +1374,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
+}
+
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
 }
 <% end %>
 
@@ -1155,14 +1400,14 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
  * @return Parse result value.
  */
 <% if @grammar.tree %>
-<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
+    return <%= tree_handle(h_type(@grammar.start_rules[0]), "context->parse_result") %>;
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
+    return <%= tree_handle(h_type(start_rule), "context->parse_result") %>;
 }
 <%   end %>
 <% else %>
@@ -1191,6 +1436,58 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
     return context->text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context->text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context->input_index;
+}
+
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context->input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *
@@ -1213,48 +1510,3 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->token;
 }
-<% if @grammar.tree %>
-
-static void tree_delete(TreeNode * node)
-{
-    if (node->is_token)
-    {
-        <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
-<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
-<%   if @cpp %>
-        delete token_tree_node;
-<%   else %>
-        free(token_tree_node);
-<%   end %>
-    }
-    else if (node->n_fields > 0u)
-    {
-        for (size_t i = 0u; i < node->n_fields; i++)
-        {
-            if (node->fields[i] != NULL)
-            {
-                tree_delete(node->fields[i]);
-            }
-        }
-        free(node);
-    }
-}
-
-/**
- * Free all tree node memory.
- */
-void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
-{
-    tree_delete((TreeNode *)tree);
-}
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-
-/**
- * Free all tree node memory.
- */
-void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
-{
-    tree_delete((TreeNode *)tree);
-}
-<%   end %>
-<% end %>

assets/parser.d.erb

@@ -9,7 +9,7 @@ module <%= @grammar.modulename %>;
 <% end %>
 
 import core.memory;
-import core.stdc.stdlib : malloc;
+import core.stdc.stdlib : malloc, free;
 
 /**************************************************************************
  * User code blocks
@@ -59,10 +59,10 @@ public alias <%= @grammar.prefix %>code_point_t = uint;
  */
 public struct <%= @grammar.prefix %>position_t
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint col;
 
     /** Invalid position value. */
@@ -86,42 +86,127 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
+{
+    return pvalue.v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
-/** Common tree node structure. */
-private struct TreeNode
-{
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    void *[0] fields;
-}
+/** Tree node ID type (index into the context node arena). ID 0 is null. */
+public alias <%= @grammar.prefix %>node_id_t = uint;
 
-/** Tree node types. @{ */
-public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+/**
+ * Tree node record.
+ *
+ * All tree nodes are stored contiguously in the context node arena. Child
+ * links are stored in a shared children array: a node's children
+ * occupy children[child_offset .. child_offset + n_fields]. Token payload
+ * fields (token, pvalue, and any user fields) are only meaningful when
+ * is_token is true.
+ */
+private struct <%= @grammar.prefix %>node_data_t
 {
-    /* TreeNode fields must be present in the same order here. */
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    <%= @grammar.prefix %>node_id_t child_offset;
+    ushort n_fields;
+    bool is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 <%= @grammar.token_user_fields %>
 }
 
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
+/** Tree node handle types. @{ */
+
+/** Token tree node handle. */
+public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 {
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-<%     rule_set.tree_fields.each do |fields| %>
-    union
+    private <%= @grammar.prefix %>context_t * __context;
+    private <%= @grammar.prefix %>node_id_t __id;
+
+    this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
     {
-<%       fields.each do |field_name, type| %>
-        <%= type %> * <%= field_name %>;
-<%       end %>
+        this.__context = context;
+        this.__id = id;
     }
+
+    /** Return whether this handle refers to a valid (non-null) node. */
+    @property bool valid()
+    {
+        return __id != 0u;
+    }
+
+    /** Access the underlying node record (token, pvalue, and user fields). */
+    @property ref <%= @grammar.prefix %>node_data_t __node()
+    {
+        return __context.<%= @grammar.prefix %>tree_nodes[__id];
+    }
+
+    alias __node this;
+}
+
+<%   tree_node_rule_sets.each do |rule_set| %>
+/** <%= rule_set.name %> tree node handle. */
+public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_suffix %>
+{
+    private <%= @grammar.prefix %>context_t * __context;
+    private <%= @grammar.prefix %>node_id_t __id;
+
+    this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
+    {
+        this.__context = context;
+        this.__id = id;
+    }
+
+    /** Return whether this handle refers to a valid (non-null) node. */
+    @property bool valid()
+    {
+        return __id != 0u;
+    }
+
+    /** Text position of the first code point spanned by this node. */
+    @property <%= @grammar.prefix %>position_t position()
+    {
+        return __context.<%= @grammar.prefix %>tree_nodes[__id].position;
+    }
+
+    /** Text position of the last code point spanned by this node. */
+    @property <%= @grammar.prefix %>position_t end_position()
+    {
+        return __context.<%= @grammar.prefix %>tree_nodes[__id].end_position;
+    }
+
+    /** Number of child fields in this node. */
+    @property ushort n_fields()
+    {
+        return __id ? __context.<%= @grammar.prefix %>tree_nodes[__id].n_fields : cast(ushort)0u;
+    }
+<%     rule_set.tree_fields.each_with_index do |fields, i| %>
+<%       fields.each do |field_name, type| %>
+
+    /** Access the <%= field_name %> child node. */
+    @property <%= type %> <%= field_name %>()
+    {
+        if (__id == 0u)
+        {
+            return <%= type %>(__context, 0u);
+        }
+        return <%= type %>(__context, __context.<%= @grammar.prefix %>tree_children[__context.<%= @grammar.prefix %>tree_nodes[__id].child_offset + <%= i %>u]);
+    }
+<%       end %>
 <%     end %>
 }
 
@@ -174,7 +259,13 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Parse result value. */
 <% if @grammar.tree %>
-    void * parse_result;
+    <%= @grammar.prefix %>node_id_t parse_result;
+
+    /** Tree node arena. Node ID 0 is reserved as the null node. */
+    <%= @grammar.prefix %>node_data_t[] <%= @grammar.prefix %>tree_nodes;
+
+    /** Shared tree child links. */
+    <%= @grammar.prefix %>node_id_t[] <%= @grammar.prefix %>tree_children;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -225,6 +316,7 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *************************************************************************/
 
 /**
+ * Allocate and initialize lexer/parser context structure.
  *
  * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
@@ -245,10 +337,35 @@ private enum size_t INVALID_ID = cast(size_t)-1;
     context.text_position.row = 1u;
     context.text_position.col = 1u;
     context.mode = <%= @lexer.mode_id("default") %>;
+<% if @grammar.tree %>
+
+    /* Reserve node ID 0 as the null tree node. */
+    context.<%= @grammar.prefix %>tree_nodes = new <%= @grammar.prefix %>node_data_t[](1);
+<% end %>
 
     return context;
 }
 
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
+<% if @grammar.tree && @grammar.free_token_node != "" %>
+    foreach (ref node; context.<%= @grammar.prefix %>tree_nodes)
+    {
+        if (node.is_token)
+        {
+            <%= @grammar.prefix %>node_data_t * token_tree_node = &node;
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+        }
+    }
+<% end %>
+}
+
 /**************************************************************************
  * Decoder
  *************************************************************************/
@@ -632,6 +749,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     {
     case P_SUCCESS:
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
@@ -670,17 +803,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -849,12 +971,14 @@ private struct state_value_t
     /** Parser state ID. */
     size_t state_id;
 
+<% if @grammar.tree %>
+    /** Tree node ID. */
+    <%= @grammar.prefix %>node_id_t node_id;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
-
-<% if @grammar.tree %>
-    /** Tree node. */
-    void * tree_node;
 <% end %>
 
     this(size_t state_id)
@@ -908,6 +1032,55 @@ private immutable parser_state_t[] parser_state_table = [
 ];
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-1-j];
+                    if (sv.end_position.valid)
+                    {
+                        return sv.end_position;
+                    }
+                }
+            }
+            else
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-n_states+j];
+                    if (sv.position.valid)
+                    {
+                        return sv.position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return statevalues[$-1-n_states+i].end_position;
+            }
+            else
+            {
+                return statevalues[$-1-n_states+i].position;
+            }
+        }
+    }
+    return <%= @grammar.prefix %>position_t.INVALID;
+}
+<% end %>
+
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -918,7 +1091,7 @@ private immutable parser_state_t[] parser_state_table = [
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -990,8 +1163,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1004,29 +1185,46 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
-    void * reduced_parser_node;
+    <%= @grammar.prefix %>node_id_t reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        foreach (eof_token; follow_tokens)
+        {
+            if (token == eof_token)
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1039,25 +1237,66 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* Successful parse. */
 <% if @grammar.tree %>
-                context.parse_result = statevalues[$-1].tree_node;
+                context.parse_result = statevalues[$-1].node_id;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
                 return P_SUCCESS;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
+                    context.input_index -= token_info.length;
+                    context.text_position = token_info.position;
+<% if @grammar.tree %>
+                    context.parse_result = statevalues[$-1].node_id;
+<% else %>
+                    context.parse_result = statevalues[$-1].pvalue;
+<% end %>
+                    return P_SUCCESS;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             statevalues ~= state_value_t(shift_state);
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
+                <%= @grammar.prefix %>node_id_t token_node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
+                context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
+                <%= @grammar.prefix %>node_data_t * token_tree_node = &context.<%= @grammar.prefix %>tree_nodes[token_node_id];
+                token_tree_node.position = token_info.position;
+                token_tree_node.end_position = token_info.end_position;
+                token_tree_node.n_fields = 0u;
+                token_tree_node.is_token = true;
+                token_tree_node.token = token;
+                token_tree_node.pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                statevalues[$-1].tree_node = token_tree_node;
+                statevalues[$-1].node_id = token_node_id;
 <% else %>
+                statevalues[$-1].position = token_info.position;
+                statevalues[$-1].end_position = token_info.end_position;
                 statevalues[$-1].pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
@@ -1066,9 +1305,11 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                statevalues[$-1].tree_node = reduced_parser_node;
+                statevalues[$-1].node_id = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
+                statevalues[$-1].position = reduced_position;
+                statevalues[$-1].end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
 <% end %>
@@ -1078,60 +1319,78 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
         }
 
         size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
 <% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = statevalues[$ - 1].tree_node;
+                reduced_parser_node = statevalues[$ - 1].node_id;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
-                TreeNode * node = cast(TreeNode *)malloc(node_size);
-                GC.addRange(node, node_size);
-                node.position = <%= @grammar.prefix %>position_t.INVALID;
-                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
-                foreach (i; 0..n_fields)
-                {
-                    node.fields[i] = null;
-                }
+                ushort n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                /* Reserve child slots. New slots are zero-initialized
+                 * (null node ID) so absent optional children remain null. */
+                <%= @grammar.prefix %>node_id_t child_offset = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_children.length;
+                context.<%= @grammar.prefix %>tree_children.length += n_fields;
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
+                        context.<%= @grammar.prefix %>tree_children[child_offset + i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
                     }
                 }
                 else
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
+                        context.<%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
                     }
                 }
+                <%= @grammar.prefix %>node_id_t node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
+                context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
+                <%= @grammar.prefix %>node_data_t * node = &context.<%= @grammar.prefix %>tree_nodes[node_id];
+                node.position = <%= @grammar.prefix %>position_t.INVALID;
+                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.child_offset = child_offset;
+                node.n_fields = n_fields;
+                node.is_token = false;
                 bool position_found = false;
                 foreach (i; 0..n_fields)
                 {
-                    TreeNode * child = cast(TreeNode *)node.fields[i];
-                    if (child && child.position.valid)
+                    <%= @grammar.prefix %>node_id_t child_id = context.<%= @grammar.prefix %>tree_children[child_offset + i];
+                    if (child_id != 0u && context.<%= @grammar.prefix %>tree_nodes[child_id].position.valid)
                     {
                         if (!position_found)
                         {
-                            node.position = child.position;
+                            node.position = context.<%= @grammar.prefix %>tree_nodes[child_id].position;
                             position_found = true;
                         }
-                        node.end_position = child.end_position;
+                        node.end_position = context.<%= @grammar.prefix %>tree_nodes[child_id].end_position;
                     }
                 }
-                reduced_parser_node = node;
+                reduced_parser_node = node_id;
             }
             else
             {
-                reduced_parser_node = null;
+                reduced_parser_node = 0u;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1139,6 +1398,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                reduced_position = <%= @grammar.prefix %>position_t.INVALID;
+                reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
@@ -1158,14 +1427,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 
 public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
+}
+
+public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
 }
 <% end %>
 
@@ -1178,14 +1453,14 @@ public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix
  * @return Parse result value.
  */
 <% if @grammar.tree %>
-public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
+    return <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %>(context, context.parse_result);
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
+    return <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %>(context, context.parse_result);
 }
 <%   end %>
 <% else %>
@@ -1201,6 +1476,7 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
+
 /**
  * Get the current text input position.
  *
@@ -1214,6 +1490,58 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
     return context.text_position;
 }
 
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
+public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
+{
+    context.text_position = position;
+}
+
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context.input_index;
+}
+
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context.input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -8,6 +8,9 @@
 
 #include <stdint.h>
 #include <stddef.h>
+<% if @cpp %>
+#include <vector>
+<% end %>
 
 /**************************************************************************
  * Public types
@@ -45,10 +48,10 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
  */
 typedef struct
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint32_t row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint32_t col;
 } <%= @grammar.prefix %>position_t;
 
@@ -69,50 +72,48 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
+{
+    return pvalue->v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
-/** Tree node types. @{ */
-typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+/** Tree node ID type (index into the context node arena). ID 0 is null. */
+typedef uint32_t <%= @grammar.prefix %>node_id_t;
+
+/**
+ * Tree node record.
+ *
+ * All tree nodes are stored contiguously in the context node arena. Child
+ * links are stored in a shared children array: a node's children
+ * occupy children[child_offset .. child_offset + n_fields]. Token payload
+ * fields (token, pvalue, and any user fields) are only meaningful when
+ * is_token is nonzero.
+ */
+typedef struct
 {
-    <% # TreeNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    <%= @grammar.prefix %>node_id_t child_offset;
     uint16_t n_fields;
     uint8_t is_token;
-<%= @grammar.token_user_fields %>
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
-} <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
-
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-struct <%= name %>;
-<%   end %>
-
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
-{
-    <% # TreeNode fields must be present in the same order here. # %>
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    uint16_t n_fields;
-    uint8_t is_token;
-<%     rule_set.tree_fields.each do |fields| %>
-    union
-    {
-<%       fields.each do |field_name, type| %>
-        struct <%= type %> * <%= field_name %>;
-<%       end %>
-    };
-<%     end %>
-} <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
-
-<%   end %>
-/** @} */
+<%= @grammar.token_user_fields %>
+} <%= @grammar.prefix %>node_data_t;
 <% end %>
 
 /** Lexed token information. */
@@ -163,7 +164,25 @@ typedef struct
 
     /** Parse result value. */
 <% if @grammar.tree %>
-    void * parse_result;
+    <%= @grammar.prefix %>node_id_t parse_result;
+
+<%   if @cpp %>
+    /** Tree node arena. Node ID 0 is reserved as the null node. */
+    std::vector<<%= @grammar.prefix %>node_data_t> <%= @grammar.prefix %>tree_nodes;
+
+    /** Shared tree child links. */
+    std::vector<<%= @grammar.prefix %>node_id_t> <%= @grammar.prefix %>tree_children;
+<%   else %>
+    /** Tree node arena. Node ID 0 is reserved as the null node. */
+    <%= @grammar.prefix %>node_data_t * <%= @grammar.prefix %>tree_nodes;
+    size_t <%= @grammar.prefix %>tree_nodes_length;
+    size_t <%= @grammar.prefix %>tree_nodes_capacity;
+
+    /** Shared tree child links. */
+    <%= @grammar.prefix %>node_id_t * <%= @grammar.prefix %>tree_children;
+    size_t <%= @grammar.prefix %>tree_children_length;
+    size_t <%= @grammar.prefix %>tree_children_capacity;
+<%   end %>
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -177,6 +196,10 @@ typedef struct
 <%= @grammar.context_user_fields %>
 } <%= @grammar.prefix %>context_t;
 
+<% if @grammar.tree %>
+<%= c_tree_types_header %>
+<% end %>
+
 /**************************************************************************
  * Public data
  *************************************************************************/
@@ -196,12 +219,14 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
 <%   end %>
 
 <% if @grammar.tree %>
-<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
 <%   end %>
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
@@ -210,15 +235,14 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
 <%   end %>
 <% end %>
 
-<% if @grammar.tree %>
-void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
-<%   end %>
-<% end %>
-
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
+void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
+
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
+
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
+
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -13,7 +13,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C, C++, or D language outputs
+  * targets C, C++, D, or Rust language outputs
   * optionally supports automatic full parse tree generation
   * supports starting parsing from multiple start rules
   * tracks input text start and end positions for all matched tokens/rules
@@ -151,12 +151,37 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
-#### C/C++
+Lexer code blocks appear between `<<` and `>>` markers following a `drop`,
+`token`, or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+The input text positions of the matched token can also be accessed from within
+a lexer code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+The start position of the matched token is accessed with `${position}`, and
+the end position of the matched token is accessed with `${end_position}`.
+
+Example:
+
+```
+token integer /\d+/ <<
+  printf("integer token on row %d, col %d\n",
+      ${position}.row, ${position}.col);
+  $$ = parse_integer(match, match_length);
+>>
+```
+
+#### C/C++ lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a pointer points to the text matched by the lexer pattern.
-  * `match_length` - length of the matched text.
+  * `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern.
+  * `match_length` (`size_t`) - length of the matched text.
 
 Example:
 
@@ -174,11 +199,11 @@ token integer /\d+/ <<
 >>
 ```
 
-#### D
+#### D lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a slice containing the text matched by the lexer pattern.
+  * `match` (`string`) - a slice containing the text matched by the lexer pattern.
 
 ```
 ptype ulong;
@@ -194,13 +219,6 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `drop`, `token`, or pattern expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
 ### Parser rule code blocks
 
 Example:
@@ -217,9 +235,94 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...
 
-Parser rule code blocks are not available in tree generation mode.
+The input text positions for the reduced rule and for the individual rule
+components can also be accessed from within a parser rule code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+
+The start position of the overall reduced rule is accessed with
+`${$.position}`, and the end position of the overall reduced rule is accessed
+with `${$.end_position}`.
+
+The start and end positions of an individual rule component are accessed
+positionally with `${N.position}` and `${N.end_position}`, where `N` is the
+1-based index of the component (`${1.position}` for the first component,
+`${2.position}` for the second, and so on).
+
+Example:
+
+```
+Assignment -> ident equals Expr <<
+    printf("assignment on row %d, col %d\n",
+        ${$.position}.row, ${$.position}.col);
+    printf("target identifier ends on row %d, col %d\n",
+        ${1.end_position}.row, ${1.end_position}.col);
+    printf("expression starts on row %d, col %d\n",
+        ${3.position}.row, ${3.position}.col);
+>>
+```
+
+A rule or rule component that allows for an empty match may not have valid
+positions.
+In this case the position should be checked for validity before its `row` and
+`col` fields are used (see `${#p_position_valid}`).
+For C targets this can be accomplished with
+`if (p_position_valid(${$.position}))` and for D targets this can be
+accomplished with `if (${$.position}.valid)`.
+
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
+Parser rule code blocks are still supported in tree generation mode, but they
+behave differently than when tree generation mode is not active.
+The code block for a rule is executed after the rule has been matched and its
+tree node has been fully formed.
+Within the code block, `$$` refers to the tree node handle for the reduced
+rule, and the rule components are accessed positionally with `$1`, `$2`, `$3`,
+etc..., each a tree node handle for that component (a rule node or a `Token`
+node).
+Field aliases (see the "Specifying parser rules" section) may also be used to
+reference a component tree node by name; a field alias behaves identically to
+the positional reference for that component.
+
+Tree nodes are stored in a compact arena owned by the parser context and are
+referenced by lightweight handles rather than pointers. The whole tree is freed
+together with the context by `p_context_delete()`; there is no separate tree
+delete function, and tree node handles are only valid while the context is
+alive.
+
+Child fields, positions, and token payloads are accessed through per-language
+accessors on a node handle:
+
+  * C: field accessor functions `p_TYPE_field(node)` and tree walk macros
+    `p_tree_walk_TYPE(node, field1, field2, ...)`; generic accessors
+    `p_node_valid(node)`, `p_node_position(node)`, `p_node_end_position(node)`,
+    `p_node_n_fields(node)`, `p_node_data(node)` (a pointer to the node record,
+    for token payload and user fields), and `p_node_id(node)` (for identity
+    comparison).
+  * C++: handle methods called with `()`, e.g. `node.field()`, `node.valid()`,
+    `node.position()`, `node.token()`, `node.pvalue()`, and `node.data()`. The
+    C-style functions and macros above are also available.
+  * D: `@property` accessors, e.g. `node.field`, `node.valid`, `node.position`,
+    `node.token`, `node.pvalue`.
+
+The positional position expansions (`${$.position}`, `${N.position}`, etc...)
+are not available in tree generation mode; use the position accessors above
+instead.
+
+C example:
+
+```
+tree;
+
+Assignment -> ident equals Expr <<
+    /* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is
+     * the Expr rule node. */
+    printf("assignment on row %d, col %d\n",
+        p_node_position($$).row, p_node_position($$).col);
+    printf("target identifier ends on row %d, col %d\n",
+        p_node_end_position($1).row, p_node_end_position($1).col);
+>>
+```
 
 ##> `context_user_fields` statement - adding custom fields to the context
 
@@ -316,6 +419,96 @@ free_token_node <<
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
+##> `lex_fn` statement - specifying a custom lexer function
+
+Propane generates both a lexer and a parser.
+By default, the parser uses the generated `p_lex()` function directly to
+return information for the next lexed token from the input stream.
+
+However, the user can specify a custom lex function.
+This function may or may not use the Propane generated `p_lex()` function under
+the hood.
+For example, a token sequence could be injected or repeated from a previously
+saved macro definition.
+
+Example (C/C++):
+
+```
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+lex_fn mylexfn;
+```
+
+The `lex_fn` statement takes one argument specifying the name of the custom
+lexer function.
+The user must supply a value for the `token` field of the `p_token_info_t`
+output structure so that the parser knows what token was lexed.
+
+Additionally, if the parser user code makes use of the token's pvalue, then
+the lexer function must supply a value for the `pvalue` field of the
+`p_token_info_t` structure.
+The `p_value()` generated API function could be useful for specifying
+parser values to associate with the lexed token when tree generation is not
+enabled.
+When tree generation is enabled, the `pvalue` field can be set to an instance
+of whatever type the user has defined as the `ptype` type.
+
+The custom lex function returns a result code to the parser.
+Returning `P_SUCCESS` indicates that the function has produced a token in the
+`token` output field and the parse should proceed.
+Returning any other result code stops the parse immediately; the parser
+propagates that code out of `p_parse()` (and the `p_parse_XXX()` and
+`p_parse_inner_XXX()` functions) unchanged.
+This allows a custom lex function to surface an error condition, for example by
+propagating a `P_UNEXPECTED_TOKEN`, `P_UNEXPECTED_INPUT`, or `P_DECODE_ERROR`
+result from a nested `p_parse_inner_XXX()` or `p_lex()` call.
+
+Observe the following contract when returning a result code other than
+`P_SUCCESS`:
+
+* Do not return `P_DROP`.
+  This code is an internal lexer signal and is never returned to the parser by
+  the generated `p_lex()` function.
+  Returning it from a custom lex function would be reported as a spurious parse
+  failure. If drop functionality is required, the custom lex function should
+  loop and return the next non-drop token.
+* Do not return `P_EOF` to signal the end of the input.
+  The end of input is communicated to the parser by returning `P_SUCCESS` with
+  the `token` field set to the end-of-input token (`TOKEN___EOF`), which is what
+  the generated `p_lex()` function does.
+  Returning `P_EOF` aborts the parse rather than allowing it to complete.
+* The `p_token()` and `p_position()` accessors are populated by the parser only
+  when the parser itself detects an unexpected token.
+  When a custom lex function returns an error code, `p_token()` is not updated
+  and may not reflect a meaningful token, and `p_position()` reflects the
+  lexer's current text position rather than a parser-identified error location.
+  A custom lex function that wants a specific reported position can set it with
+  `p_set_position()` before returning.
+* Returning `P_USER_TERMINATED` does not populate the user terminate code
+  returned by `p_user_terminate_code()`. This user terminate code is normally
+  populated by the `$terminate()` function in the user code block.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -328,6 +521,27 @@ module proj.parser;
 If a module statement is not present, then the generated D module will not
 contain a module statement and the default module name will be used.
 
+##> `noline` statement - disabling `#line` directives
+
+By default, Propane emits `#line` directives into the generated output around
+user code blocks.
+These directives instruct the compiler to report any warnings or errors in the
+user code using the file name and line number of the original grammar file,
+rather than the generated output file.
+This makes it easier to locate the source of a compiler diagnostic in the
+grammar file.
+
+The `noline` statement disables the emission of these `#line` directives.
+
+```
+noline;
+```
+
+When the `noline` statement is present, user code blocks are copied to the
+generated output without any surrounding `#line` directives.
+This can be useful when debugging the generated parser itself, or when the
+`#line` directives interfere with other tooling.
+
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
 The `on_token_node` statement can be used to provide code that initializes
@@ -581,7 +795,9 @@ It is recommended to place this statement early in the grammar.
 In tree generation mode various aspects of propane's behavior are changed:
 
   * Only one `ptype` is allowed.
-  * Parser user code blocks are not supported.
+  * Parser user code blocks execute after the rule's tree node has been formed
+  and access the tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser rule
+  code blocks" section).
   * Structure types are generated to represent the parsed tokens and rules as
   defined in the grammar.
   * The parse result from `p_result()` points to a `Start` struct containing
@@ -930,8 +1146,12 @@ The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.
 
-Parser rule code blocks are not allowed and not used when tree generation mode
-is active.
+In tree generation mode, parser rule code blocks access the reduced rule tree
+node and its component tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser
+rule code blocks" section).
+Field aliases may still be used in tree generation mode to reference a component
+tree node by name, behaving identically to the corresponding positional
+reference.
 
 ##> User termination of the lexer or parser
 
@@ -1007,6 +1227,17 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_value_t`
+
+If tree generation mode is enabled, the `p_value_t` type is defined to be the
+type given to the `ptype` statement in the grammar file.
+
+If tree generation mode is not enabled, there could be more than one `ptype`
+given, so the `p_value_t` type is a union of all possible `ptype` types.
+In this case, the API functions `p_value()` and `p_value_XXX()` for each given
+`ptype` name `XXX` are generated to return `p_value_t` instances holding the
+corresponding `ptype`.
+
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -1016,6 +1247,9 @@ The `p_token_info_t` structure contains the following fields:
 * `length` (`size_t`) holds the number of input bytes used by the token.
 * `token` (`p_token_t`) holds the token ID of the lexed token
 * `pvalue` (`p_value_t`) holds the parser value associated with the token.
+  The actual user value can be extracted with `p_value_get(&token_info.pvalue)`
+  for the default value or `p_value_get_XXX(&token_info.pvalue)` for named
+  `ptype` values.
 
 ### Tree Node Types
 
@@ -1130,8 +1364,6 @@ p_context_t * context = p_context_new(input);
 The `p_context_delete()` function must be called to deinitialize and deallocate
 a context structure allocated by `p_context_init()`.
 
-This function is not available for D language since D has a garbage collector.
-
 ### `p_lex`
 
 The `p_lex()` function is the main entry point to the lexer.
@@ -1191,6 +1423,42 @@ size_t result = p_parse_Statement(context);
 
 In this case, the parser will start parsing with the `Statement` rule.
 
+### `p_parse_inner_XXX`
+
+For each start rule, a `p_parse_inner_XXX()` function is also generated.
+This variant of the parser entry point accepts a caller-provided array of
+"follow tokens" -- tokens the caller allows to appear immediately after the
+start rule in some outer grammar context.
+It is useful when embedding a Propane-generated sub-parser within an outer
+parser and the outer parser knows which tokens naturally terminate the
+sub-parse.
+
+For C targets, the signature is (example for a rule named `Statement`):
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    p_token_t const * follow_tokens, size_t n_follow_tokens);
+```
+
+Passing a `NULL` pointer (or a count of zero) makes the function behave
+identically to `p_parse_Statement()`.
+
+For D targets, the signature accepts a slice:
+
+```
+size_t p_parse_inner_Statement(p_context_t * context,
+    const(p_token_t)[] follow_tokens);
+```
+
+Passing `null` for the slice makes the function behave identically to
+`p_parse_Statement()`.
+
+When the parse is completed via a non-EOF follow token, that follow token is
+**not** consumed from the input stream.
+The parser rewinds the input index and text position to the start of the follow
+token so that a subsequent call to `p_lex()` or another parse function sees the
+same token.
+
 ### `p_position_valid`
 
 The `p_position_valid()` function is only generated for C targets.
@@ -1258,10 +1526,67 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token at row %u column %u\n",
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 
+### `p_set_position`
+
+The `p_set_position()` function sets the current text position stored in the
+context.
+This is useful when performing a nested parse operation over a slice of a
+larger enclosing document: by setting the initial text position to something
+other than the default `(1, 1)`, any token positions or error positions
+subsequently reported by the lexer/parser are relative to the enclosing
+document rather than the slice.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_position_t start = { .row = 5, .col = 20 };
+p_set_position(context, start);
+p_parse(context);
+```
+
+### `p_input_index`
+
+The `p_input_index()` function returns the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This is useful for slicing out the remaining input after a partial parse,
+or for locating tokens in the original input buffer.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_parse_inner_Statement(context, follow_tokens, n_follow_tokens);
+size_t offset = p_input_index(context);
+/* Remaining input starts at `input + offset`. */
+```
+
+### `p_set_input_index`
+
+The `p_set_input_index()` function sets the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This moves the lexer's read cursor, which can be used together with
+`p_set_position()` to rewind the input part-way through a parse in order to
+re-read an earlier section of the input.
+The byte offset is not validated; the caller is responsible for providing an
+offset within the bounds of the input text.
+A value previously returned by `p_input_index()` is a suitable argument.
+
+Example:
+
+```
+/* Save the cursor and text position at the start of a section. */
+size_t saved_index = p_input_index(context);
+p_position_t saved_position = p_position(context);
+/* ... later, rewind to re-read that section. */
+p_set_input_index(context, saved_index);
+p_set_position(context, saved_position);
+```
+
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user
@@ -1320,8 +1645,6 @@ assert(code_point_length == 4u);
 The `p_tree_delete()` function can be used to free the memory used by the tree.
 It should be passed the same value that is returned by `p_result()`.
 
-The `p_tree_delete()` function is only available for C/C++ output targets.
-
 Note that if any lexer user code block allocates memory to store in a token's
 `pvalue`, in order to properly free this memory the `free_token_node` statement
 should be used to provide a code block that frees this memory.
@@ -1340,6 +1663,37 @@ p_tree_delete_Statement(statement_tree);
 In this case, Propane will free a `Statement` tree structure returned by the
 `p_parse_Statement(context)` function.
 
+### `p_value`
+
+The `p_value(v)` function builds an instance of the `p_value_t` with the
+default member set to the value of `v`.
+
+A `p_value_XXX(v)` function is defined for each user-defined `ptype` name with
+the user-given name in place of the `XXX`.
+
+These functions are useful for custom lexer functions which need to return a
+parser value corresponding to a lexed token.
+
+They are especially useful when tree generation mode is not active.
+In that case, the `p_value_t` union can hold one of several different possible
+value types.
+
+### `p_value_get`
+
+The `p_value_get()` accessor functions can be used to extract a user value
+from a `p_value_t` union.
+
+The `p_value_get(pvalue)` function accepts a pointer to a `p_value_t` and
+returns the value of its default member.
+
+A `p_value_get_XXX(pvalue)` function is generated for each user-defined `ptype`
+name with the user-given name in place of the `XXX`, returning the value of the
+corresponding member.
+
+These functions are the counterpart to `p_value()` constructor functions and
+are useful for reading the parser value associated with a lexed token, for
+example `p_value_get(&token_info.pvalue)`.
+
 ##> Data
 
 ### `p_token_names`
@@ -1357,7 +1711,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
         p_token_names[context->token],
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 

extra/vim/syntax/propane.vim

@@ -8,7 +8,11 @@ if exists("b:current_syntax")
 endif
 
 if !exists("b:propane_subtype")
-  let b:propane_subtype = "d"
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+    let b:propane_subtype = "d"
+  else
+    let b:propane_subtype = "cpp"
+  endif
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
@@ -16,18 +20,32 @@ exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
 syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
 
 syn match propaneComment "#.*"
-syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
+" The right-hand side of a rule (after '->' up to '<<' or ';') lists symbol
+" names that may coincide with propane keywords (e.g. 'start', 'token',
+" 'tree'). Wrap it in a region that excludes keyword matches so those names
+" are not highlighted as keywords. The '<<' is left unconsumed so the
+" propaneTarget region can still match it.
+syn region propaneRuleRhs matchgroup=propaneOperator start="->" end="\ze<<" end=";" contains=propaneFieldAlias,propaneRuleOperator,propaneComment keepend
+syn match propaneRuleOperator "?" contained
+" Keywords that introduce a user-defined name. The name is consumed by
+" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
+" is not highlighted as a keyword. These must be a match (not syn keyword)
+" because a syn keyword always wins over a contained nextgroup match.
+syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
+syn match propaneName "\<\h\w*\>" contained
+syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
+hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
+hi def link propaneRuleOperator Operator
 hi def link propaneFieldOperator Operator
 hi def link propaneDelimiter Delimiter
 hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
+        grammar = Grammar.new(File.read(input_file), input_file)
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -18,6 +18,8 @@ class Propane
         elsif output_file =~ %r{\.(cc|cpp|cxx)$}
           @cpp = true
           "c"
+        elsif output_file.end_with?(".rs")
+          "rust"
         else
           raise Error.new("Could not determine target language from output file name (#{output_file})")
         end
@@ -31,14 +33,26 @@ class Propane
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension || @language}.erb")
+        template_language = @language == "rust" ? "rs" : @language
+        template = Assets.get("parser.#{extension || template_language}.erb")
         if extension
           output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         else
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        result = erb.result(binding.clone)
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if @language == "rust"
+            # Rust has no #line directive support, so strip the directives that
+            # the grammar embeds in user code blocks.
+            line = line.sub(/^#line \d+ "[^"]*"/, "")
+            line == "#linereset\n" ? "" : line
+          elsif line == "#linereset\n"
+            %[#line #{i + 2} "#{output_file}"\n]
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -179,7 +193,8 @@ class Propane
                 end
               end
               @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
-              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optcode = @grammar.tree ? nil : "$$ = $1;\n"
+              @grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number)
               optional_rules_added << component
             end
           end
@@ -268,6 +283,8 @@ class Propane
           "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
         when "d"
           "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "rust"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
         end
       end
       code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
@@ -277,6 +294,8 @@ class Propane
           "context->#{fieldname}"
         when "d"
           "context.#{fieldname}"
+        when "rust"
+          "context.#{fieldname}"
         end
       end
       code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
@@ -286,35 +305,52 @@ class Propane
           "token_tree_node->#{fieldname}"
         when "d"
           "token_tree_node.#{fieldname}"
+        when "rust"
+          "token_tree_node.#{fieldname}"
         end
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "_pvalue->v_#{rule.ptypename}"
-          when "d"
-            "_pvalue.v_#{rule.ptypename}"
+          if @grammar.tree
+            typename = "#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix}"
+            case @language
+            when "c"
+              tree_handle(typename, "_node_id")
+            when "d"
+              tree_handle(typename, "_node_id")
+            when "rust"
+              tree_handle(typename, "_node_id")
+            end
+          else
+            case @language
+            when "c"
+              "_pvalue->v_#{rule.ptypename}"
+            when "d"
+              "_pvalue.v_#{rule.ptypename}"
+            when "rust"
+              "(*_pvalue.v_#{rule.ptypename}_mut())"
+            end
           end
         end
         code = code.gsub(/\$(\d+)/) do |match|
+          parser_component_reference(rule, $1.to_i)
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
           index = $1.to_i
-          case @language
-          when "c"
-            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          when "d"
-            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
-          end
+          "get_rule_position(statevalues, #{index}, n_states, false)"
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, true)"
         end
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
-            case @language
-            when "c"
-              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
-            when "d"
-              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
-            end
+            # Field aliases are just a named reference to a positional rule
+            # component, so reuse the same expansion as `$1', `$2', etc. Note
+            # that rule.aliases stores a 0-based component index, so add 1 to
+            # convert it to the 1-based index used for positional references.
+            parser_component_reference(rule, index + 1)
           else
             raise Error.new("Field alias '#{aliasname}' not found")
           end
@@ -327,6 +363,8 @@ class Propane
               "out_token_info->pvalue"
             when "d"
               "out_token_info.pvalue"
+            when "rust"
+              "out_token_info.pvalue"
             end
           else
             case @language
@@ -334,9 +372,31 @@ class Propane
               "out_token_info->pvalue.v_#{pattern.ptypename}"
             when "d"
               "out_token_info.pvalue.v_#{pattern.ptypename}"
+            when "rust"
+              "(*out_token_info.pvalue.v_#{pattern.ptypename}_mut())"
             end
           end
         end
+        code = code.gsub(/\$\{position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->position"
+          when "d"
+            "out_token_info.position"
+          when "rust"
+            "out_token_info.position"
+          end
+        end
+        code = code.gsub(/\$\{end_position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->end_position"
+          when "d"
+            "out_token_info.end_position"
+          when "rust"
+            "out_token_info.end_position"
+          end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)
@@ -348,12 +408,434 @@ class Propane
             "context->mode = #{mode_id}u"
           when "d"
             "context.mode = #{mode_id}u"
+          when "rust"
+            "context.mode = #{mode_id}"
           end
         end
       end
       code
     end
 
+    # Expand a positional reference to a parser rule component.
+    #
+    # This is used to expand `$1', `$2', etc. as well as field aliases (which
+    # are just named references to a positional rule component).
+    #
+    # @param rule [Rule]
+    #   The Rule containing the user code.
+    # @param index [Integer]
+    #   1-based index of the rule component to reference.
+    #
+    # @return [String]
+    #   Expanded rule component reference.
+    def parser_component_reference(rule, index)
+      component = rule.components[index - 1]
+      if @grammar.tree
+        # In tree mode a component reference yields a handle to that
+        # component's tree node. An optional component propagates its target
+        # node (or null), so use the optional target's node type.
+        if component.is_a?(RuleSet) && component.optional?
+          component = component.option_target
+        end
+        node_name = component.is_a?(Token) ? "Token" : component.name
+        typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
+        case @language
+        when "c"
+          tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
+        when "d"
+          tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
+        when "rust"
+          tree_handle(typename, "statevalues[statevalues.len() - 1 - n_states + #{index}].node_id")
+        end
+      else
+        case @language
+        when "c"
+          "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
+        when "d"
+          "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
+        when "rust"
+          "statevalues[statevalues.len() - 1 - n_states + #{index}].pvalue.get_v_#{component.ptypename}()"
+        end
+      end
+    end
+
+    # Construct a tree node handle expression for the target language.
+    #
+    # A handle is a small value pairing the parser context with a node ID
+    # (an index into the context's node arena). All handle types share this
+    # layout; the distinct types exist for documentation and, in C, to drive
+    # the tree walk macro's type threading.
+    #
+    # @param typename [String]
+    #   Handle type name.
+    # @param id_expr [String]
+    #   Expression yielding the node ID.
+    #
+    # @return [String]
+    #   Handle constructor expression.
+    def tree_handle(typename, id_expr)
+      if @cpp
+        "(#{typename}{context, #{id_expr}})"
+      elsif @language == "c"
+        "((#{typename}){context, #{id_expr}})"
+      elsif @language == "rust"
+        "(#{typename} { context, id: #{id_expr} })"
+      else
+        "#{typename}(context, #{id_expr})"
+      end
+    end
+
+    # Get the list of non-optional, non-internal rule sets that get a tree node
+    # handle type generated for them.
+    #
+    # @return [Array<Propane::RuleSet>]
+    #   Rule sets with generated tree node handle types.
+    def tree_node_rule_sets
+      @parser.rule_sets.reject do |name, rule_set|
+        name.start_with?("$") || rule_set.optional?
+      end.map {|name, rule_set| rule_set}
+    end
+
+    # Maximum number of chained fields supported by a single C tree walk macro
+    # invocation. Deeper navigation can be expressed by nesting walk calls.
+    C_TREE_WALK_MAX = 16
+
+    # Get the tree node handle type name for a node name.
+    #
+    # @param name [String]
+    #   Rule set name, or "Token".
+    #
+    # @return [String]
+    #   Handle type name.
+    def h_type(name)
+      "#{@grammar.tree_prefix}#{name}#{@grammar.tree_suffix}"
+    end
+
+    # Get the list of all tree node handle type names (Token plus rule sets).
+    #
+    # @return [Array<String>]
+    #   Handle type names.
+    def tree_handle_types
+      [h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}
+    end
+
+    # Enumerate the navigation fields of a rule set's tree node.
+    #
+    # @yield [rtype, field_name, child_type, slot]
+    #   Handle type name, field accessor name, child handle type, and child
+    #   slot index.
+    def each_tree_field(rule_set)
+      rtype = h_type(rule_set.name)
+      rule_set.tree_fields.each_with_index do |fields, slot|
+        fields.each do |field_name, child_type|
+          yield rtype, field_name, child_type, slot
+        end
+      end
+    end
+
+    # Generate the C/C++ tree node handle type section for the header.
+    #
+    # @return [String]
+    #   Header handle section.
+    def c_tree_types_header
+      @cpp ? cpp_tree_types_header : c_only_tree_types_header
+    end
+
+    # Generate the C (non-C++) tree node handle type section for the header.
+    def c_only_tree_types_header
+      p = @grammar.prefix
+      out = []
+      out << "/** Tree node handle types. @{ */"
+      tree_handle_types.each do |t|
+        out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
+      end
+      out << ""
+      out << c_common_accessors_header
+      out << "/** @} */"
+      out.join("\n")
+    end
+
+    # Generate the C-style (function + macro) tree node accessors shared by the
+    # C and C++ headers. In C++ these are provided in addition to the handle
+    # methods so that C-style code (and the tree walk macros) also works.
+    def c_common_accessors_header
+      p = @grammar.prefix
+      out = []
+      out << "/** Generic tree node accessors (usable on any handle type). */"
+      out << "#define #{p}node_valid(h) ((h).__id != 0u)"
+      out << "#define #{p}node_id(h) ((h).__id)"
+      out << "#define #{p}node_data(h) (&(h).__context->#{p}tree_nodes[(h).__id])"
+      out << "#define #{p}node_position(h) ((h).__context->#{p}tree_nodes[(h).__id].position)"
+      out << "#define #{p}node_end_position(h) ((h).__context->#{p}tree_nodes[(h).__id].end_position)"
+      out << "#define #{p}node_n_fields(h) ((h).__id ? (h).__context->#{p}tree_nodes[(h).__id].n_fields : (uint16_t)0u)"
+      out << ""
+      out << "/** Tree node field accessor functions. */"
+      out << "#{p}token_t #{p}#{h_type("Token")}_token(#{h_type("Token")} node);"
+      out << "#{p}value_t #{p}#{h_type("Token")}_pvalue(#{h_type("Token")} node);"
+      tree_node_rule_sets.each do |rule_set|
+        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
+          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node);"
+        end
+      end
+      out << ""
+      out << c_tree_walk_macros
+      out.join("\n")
+    end
+
+    # Generate the C tree walk macro machinery.
+    def c_tree_walk_macros
+      p = @grammar.prefix
+      max = C_TREE_WALK_MAX
+      out = []
+      out << "/* Tree walk macros: p_tree_walk_<Type>(handle, field, ...). */"
+      out << "#define #{p}CAT_(a, b) a##b"
+      out << "#define #{p}CAT(a, b) #{p}CAT_(a, b)"
+      out << "#define #{p}TA(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}TYPEAFTER_, t), _), f)"
+      out << "#define #{p}ACC(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}, t), _), f)"
+      argn = (1..max).map {|i| "_#{i}"}.join(", ")
+      rseq = (0..max).to_a.reverse.join(", ")
+      out << "#define #{p}ARG_N(#{argn}, N, ...) N"
+      out << "#define #{p}NARG(...) #{p}ARG_N(__VA_ARGS__, #{rseq})"
+      (1..max).each do |n|
+        fparams = (1..n).map {|k| "f#{k}"}.join(", ")
+        call = "h"
+        (1..n).each do |k|
+          texpr = "R"
+          (1...k).each {|j| texpr = "#{p}TA(#{texpr}, f#{j})"}
+          call = "#{p}ACC(#{texpr}, f#{k})(#{call})"
+        end
+        out << "#define #{p}tree_walk_#{n}(R, h, #{fparams}) #{call}"
+      end
+      out << "#define #{p}tree_walk_dispatch(R, h, ...) #{p}CAT(#{p}tree_walk_, #{p}NARG(__VA_ARGS__))(R, h, __VA_ARGS__)"
+      # Type transition map (navigation fields only).
+      tree_node_rule_sets.each do |rule_set|
+        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
+          out << "#define #{p}TYPEAFTER_#{rtype}_#{field_name} #{child_type}"
+        end
+      end
+      # Per-handle-type walk entry points.
+      tree_handle_types.each do |t|
+        out << "#define #{p}tree_walk_#{t}(...) #{p}tree_walk_dispatch(#{t}, __VA_ARGS__)"
+      end
+      out.join("\n")
+    end
+
+    # Generate the C tree node accessor function definitions for the source.
+    #
+    # @return [String]
+    #   Accessor function definitions.
+    def c_tree_accessor_defs
+      p = @grammar.prefix
+      tt = h_type("Token")
+      out = []
+      out << "#{p}token_t #{p}#{tt}_token(#{tt} node)"
+      out << "{"
+      out << "    return node.__context->#{p}tree_nodes[node.__id].token;"
+      out << "}"
+      out << ""
+      out << "#{p}value_t #{p}#{tt}_pvalue(#{tt} node)"
+      out << "{"
+      out << "    return node.__context->#{p}tree_nodes[node.__id].pvalue;"
+      out << "}"
+      tree_node_rule_sets.each do |rule_set|
+        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
+          out << ""
+          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node)"
+          out << "{"
+          out << "    #{child_type} result;"
+          out << "    result.__context = node.__context;"
+          out << "    if (node.__id == 0u)"
+          out << "    {"
+          out << "        result.__id = 0u;"
+          out << "        return result;"
+          out << "    }"
+          out << "    result.__id = node.__context->#{p}tree_children[node.__context->#{p}tree_nodes[node.__id].child_offset + #{slot}u];"
+          out << "    return result;"
+          out << "}"
+        end
+      end
+      out.join("\n")
+    end
+
+    # Generate the C++ tree node handle type section for the header.
+    def cpp_tree_types_header
+      p = @grammar.prefix
+      out = []
+      out << "/** Tree node handle types. @{ */"
+      tree_handle_types.each {|t| out << "struct #{t};"}
+      out << ""
+      # Token handle (all methods inline; no handle-typed returns).
+      tt = h_type("Token")
+      out << "struct #{tt}"
+      out << "{"
+      out << "    #{p}context_t * __context;"
+      out << "    #{p}node_id_t __id;"
+      out << "    bool valid() const { return __id != 0u; }"
+      out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
+      out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
+      out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+      out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+      out << "    #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
+      out << "    #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << "};"
+      out << ""
+      # Rule set handles: navigation methods declared, defined out-of-line below.
+      tree_node_rule_sets.each do |rule_set|
+        rtype = h_type(rule_set.name)
+        out << "struct #{rtype}"
+        out << "{"
+        out << "    #{p}context_t * __context;"
+        out << "    #{p}node_id_t __id;"
+        out << "    bool valid() const { return __id != 0u; }"
+        out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
+        out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
+        out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+        out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
+          out << "    #{child_type} #{field_name}() const;"
+        end
+        out << "};"
+        out << ""
+      end
+      # Out-of-line navigation method bodies (all handle types now complete).
+      tree_node_rule_sets.each do |rule_set|
+        rtype = h_type(rule_set.name)
+        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
+          out << "inline #{child_type} #{rtype}::#{field_name}() const"
+          out << "{"
+          out << "    if (__id == 0u)"
+          out << "    {"
+          out << "        return #{child_type}{__context, 0u};"
+          out << "    }"
+          out << "    return #{child_type}{__context, __context->#{p}tree_children[__context->#{p}tree_nodes[__id].child_offset + #{slot}u]};"
+          out << "}"
+        end
+      end
+      out << ""
+      out << "/*"
+      out << " * C-style function and macro accessors, provided in addition to the handle"
+      out << " * methods above so that C-style code and the tree walk macros also work."
+      out << " */"
+      out << c_common_accessors_header
+      out << "/** @} */"
+      out.join("\n")
+    end
+
+    # Rust keywords that must be escaped as raw identifiers when used as a
+    # generated identifier (e.g. a field alias named `type`).
+    RUST_KEYWORDS = %w[
+      as break const continue dyn else enum extern false fn for if impl in let
+      loop match mod move mut pub ref return static struct trait true type
+      unsafe use where while async await abstract become box do final macro
+      override priv typeof unsized virtual yield try gen
+    ]
+
+    # Escape a name as a Rust raw identifier if it is a reserved keyword.
+    #
+    # @param name [String]
+    #   Identifier name.
+    #
+    # @return [String]
+    #   Name, escaped as a raw identifier if necessary.
+    def rust_ident(name)
+      RUST_KEYWORDS.include?(name) ? "r##{name}" : name
+    end
+
+    # Map a ptype type string to a valid Rust type.
+    #
+    # The default ptype is a C "void *"; for Rust with no declared ptype we use
+    # the unit type instead.
+    #
+    # @param typestring [String]
+    #   ptype type string.
+    #
+    # @return [String]
+    #   Rust type string.
+    def rust_ptype(typestring)
+      typestring == "void *" ? "()" : typestring
+    end
+
+    # Generate the Rust tree node record and handle types.
+    #
+    # Mirrors the C tree node record plus the C++ handle structs: each rule set
+    # and the Token node get a handle type ({context, id}) with accessor methods.
+    #
+    # @return [String]
+    #   Rust tree node type definitions.
+    def rust_tree_types
+      p = @grammar.prefix
+      out = []
+      out << "/** Tree node record. */"
+      out << "#[derive(Clone, Default)]"
+      out << "pub struct #{p}node_data_t {"
+      out << "    pub position: #{p}position_t,"
+      out << "    pub end_position: #{p}position_t,"
+      out << "    pub child_offset: #{p}node_id_t,"
+      out << "    pub n_fields: u16,"
+      out << "    pub is_token: bool,"
+      out << "    pub token: #{p}token_t,"
+      out << "    pub pvalue: #{p}value_t,"
+      unless @grammar.token_user_fields.to_s.strip.empty?
+        out << @grammar.token_user_fields
+      end
+      out << "}"
+      out << ""
+      out << "/** Tree node handle types. */"
+      tree_handle_types.each do |t|
+        out << "#[derive(Clone, Copy)]"
+        out << "pub struct #{t}<'a> { context: &'a #{p}context_t, id: #{p}node_id_t }"
+      end
+      out << ""
+      # Common accessors for every handle type.
+      tree_handle_types.each do |t|
+        out << "impl<'a> #{t}<'a> {"
+        out << "    /** Return whether this handle refers to a valid (non-null) node. */"
+        out << "    pub fn valid(&self) -> bool { self.id != 0 }"
+        out << "    /** Return the node ID (for identity comparison). */"
+        out << "    pub fn node_id(&self) -> #{p}node_id_t { self.id }"
+        out << "    /** Access the underlying node record. */"
+        out << "    pub fn data(&self) -> &'a #{p}node_data_t { &self.context.#{p}tree_nodes[self.id as usize] }"
+        out << "    /** Text position of the first code point spanned by this node. */"
+        out << "    pub fn position(&self) -> #{p}position_t { self.context.#{p}tree_nodes[self.id as usize].position }"
+        out << "    /** Text position of the last code point spanned by this node. */"
+        out << "    pub fn end_position(&self) -> #{p}position_t { self.context.#{p}tree_nodes[self.id as usize].end_position }"
+        out << "    /** Number of child fields in this node. */"
+        out << "    pub fn n_fields(&self) -> u16 { if self.id != 0 { self.context.#{p}tree_nodes[self.id as usize].n_fields } else { 0 } }"
+        if t == h_type("Token")
+          out << "    /** Token ID for this token node. */"
+          out << "    pub fn token(&self) -> #{p}token_t { self.context.#{p}tree_nodes[self.id as usize].token }"
+          out << "    /** Parser value associated with this token node. */"
+          out << "    pub fn pvalue(&self) -> #{p}value_t { self.context.#{p}tree_nodes[self.id as usize].pvalue.clone() }"
+        end
+        out << "}"
+      end
+      out << ""
+      # Navigation accessors for rule set handles.
+      tree_node_rule_sets.each do |rule_set|
+        rtype = h_type(rule_set.name)
+        out << "impl<'a> #{rtype}<'a> {"
+        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
+          out << "    /** Access the #{field_name} child node. */"
+          out << "    pub fn #{rust_ident(field_name)}(&self) -> #{child_type}<'a> {"
+          out << "        if self.id == 0 {"
+          out << "            return #{child_type} { context: self.context, id: 0 };"
+          out << "        }"
+          out << "        #{child_type} { context: self.context, id: self.context.#{p}tree_children[self.context.#{p}tree_nodes[self.id as usize].child_offset as usize + #{slot}] }"
+          out << "    }"
+        end
+        out << "}"
+      end
+      out.join("\n")
+    end
+
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]
@@ -379,6 +861,8 @@ class Propane
           "uint8_t"
         when "d"
           "ubyte"
+        when "rust"
+          "u8"
         end
       elsif max <= 0xFFFF
         case @language
@@ -386,11 +870,15 @@ class Propane
           "uint16_t"
         when "d"
           "ushort"
+        when "rust"
+          "u16"
         end
       else
         case @language
         when "c"
           "uint32_t"
+        when "rust"
+          "u32"
         else
           "uint"
         end

lib/propane/grammar.rb

@@ -6,6 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
+    attr_reader :lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -21,7 +22,8 @@ class Propane
     attr_reader :on_token_node
     attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
       @start_rules = []
       @tokens = []
@@ -56,6 +58,10 @@ class Propane
       @tokens.size + 1
     end
 
+    def parser_user_code_used?
+      @rules.any? {|r| r.code}
+    end
+
     private
 
     def parse_grammar!
@@ -69,6 +75,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -85,6 +92,7 @@ class Propane
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
+      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -117,6 +125,12 @@ class Propane
       end
     end
 
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true
@@ -253,12 +267,8 @@ class Propane
         end
         md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        if @tree
-          consume!(/;/, "expected `;'")
-        else
-          unless code = parse_code_block!
-            consume!(/;/, "expected `;' or code block")
-          end
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @modeline = nil
@@ -296,8 +306,14 @@ class Propane
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = code.chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -315,6 +331,13 @@ class Propane
       end
     end
 
+    def parse_noline_statement!
+      if md = consume!(/noline\s*;/)
+        @noline = true
+        true
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -338,9 +361,14 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = md[1].chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         code
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.0.0"
+  VERSION = "4.8.1"
 end

spec/json_parser.rust.propane

@@ -0,0 +1,176 @@
+<<header
+pub const JSON_OBJECT: usize = 0;
+pub const JSON_ARRAY: usize = 1;
+pub const JSON_NUMBER: usize = 2;
+pub const JSON_STRING: usize = 3;
+pub const JSON_TRUE: usize = 4;
+pub const JSON_FALSE: usize = 5;
+pub const JSON_NULL: usize = 6;
+
+#[derive(Clone, Default)]
+pub enum JSONValue {
+    #[default]
+    Null,
+    Object(Vec<(String, JSONValue)>),
+    Array(Vec<JSONValue>),
+    Number(f64),
+    StringVal(String),
+    True,
+    False,
+}
+
+impl JSONValue {
+    pub fn id(&self) -> usize {
+        match self {
+            JSONValue::Object(_) => JSON_OBJECT,
+            JSONValue::Array(_) => JSON_ARRAY,
+            JSONValue::Number(_) => JSON_NUMBER,
+            JSONValue::StringVal(_) => JSON_STRING,
+            JSONValue::True => JSON_TRUE,
+            JSONValue::False => JSON_FALSE,
+            JSONValue::Null => JSON_NULL,
+        }
+    }
+    pub fn number(&self) -> f64 {
+        if let JSONValue::Number(n) = self { *n } else { 0.0 }
+    }
+    pub fn string(&self) -> &str {
+        if let JSONValue::StringVal(s) = self { s.as_str() } else { "" }
+    }
+    pub fn object_len(&self) -> usize {
+        if let JSONValue::Object(e) = self { e.len() } else { 0 }
+    }
+    pub fn array_len(&self) -> usize {
+        if let JSONValue::Array(e) = self { e.len() } else { 0 }
+    }
+}
+>>
+
+context_user_fields <<
+    pub string_value: String,
+>>
+
+ptype JSONValue;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
+    $$ = JSONValue::Number(n);
+>>
+token true <<
+  $$ = JSONValue::True;
+>>
+token false <<
+  $$ = JSONValue::False;
+>>
+token null <<
+  $$ = JSONValue::Null;
+>>
+/"/ <<
+  $mode(string);
+  ${context.string_value} = String::new();
+>>
+string: token string /"/ <<
+  $$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value}));
+  $mode(default);
+>>
+string: /\\"/ <<
+  ${context.string_value}.push('"');
+>>
+string: /\\\\/ <<
+  ${context.string_value}.push('\\');
+>>
+string: /\\\// <<
+  ${context.string_value}.push('/');
+>>
+string: /\\b/ <<
+  ${context.string_value}.push('\u{0008}');
+>>
+string: /\\f/ <<
+  ${context.string_value}.push('\u{000C}');
+>>
+string: /\\n/ <<
+  ${context.string_value}.push('\n');
+>>
+string: /\\r/ <<
+  ${context.string_value}.push('\r');
+>>
+string: /\\t/ <<
+  ${context.string_value}.push('\t');
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
+  ${context.string_value}.push_str(&s);
+>>
+string: /[^\\]/ <<
+  ${context.string_value}.push(match_[0] as char);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONValue::Object(Vec::new());
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  let mut obj = $1;
+  if let JSONValue::Object(kve) = $3 {
+      if let JSONValue::Object(entries) = &mut obj {
+          entries.extend(kve);
+      }
+  }
+  $$ = obj;
+>>
+KeyValue -> string colon Value <<
+  let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() };
+  $$ = JSONValue::Object(vec![(name, $3)]);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONValue::Array(Vec::new());
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  let mut arr = $1;
+  if let JSONValue::Array(elems) = &mut arr {
+      elems.push($3);
+  }
+  $$ = arr;
+>>

spec/macros.c.propane

@@ -0,0 +1,25 @@
+<<
+#include <stdlib.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int v);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/macros.d.propane

@@ -0,0 +1,31 @@
+<<
+import test_macros;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+    int n = 0;
+    foreach (c; match)
+    {
+        n *= 10;
+        n += (c - '0');
+    }
+    $$ = n;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; record($$); >>
+MacroStart -> macro macroname lbrace;

spec/macros.rust.propane

@@ -0,0 +1,80 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    loop {
+        if context.expanding {
+            let ei = context.expand_i;
+            context.expand_i += 1;
+            if context.expand_i >= context.token_infos.len() {
+                context.expanding = false;
+            }
+            *out_token_info = context.token_infos[ei].clone();
+            return P_SUCCESS;
+        }
+
+        let lex_result = p_lex(context, out_token_info);
+        if lex_result != P_SUCCESS {
+            return lex_result;
+        }
+
+        if out_token_info.token == TOKEN_macro {
+            context.defining = true;
+        } else if out_token_info.token == TOKEN_macroname {
+            if !context.defining {
+                context.expanding = true;
+                context.expand_i = 0;
+                continue;
+            }
+        } else if out_token_info.token == TOKEN_lbrace {
+            if context.defining {
+                /* Capture the macro body tokens (up to the closing '}'). */
+                let mut infos: Vec<p_token_info_t> = Vec::new();
+                loop {
+                    let mut ti = p_token_info_t::default();
+                    assert_eq!(P_SUCCESS, p_lex(context, &mut ti));
+                    if ti.token == TOKEN_rbrace {
+                        break;
+                    }
+                    infos.push(ti);
+                }
+                context.token_infos = infos;
+                context.defining = false;
+            }
+        } else {
+            context.defining = false;
+        }
+        return lex_result;
+    }
+}
+>>
+
+context_user_fields <<
+    pub defining: bool,
+    pub expanding: bool,
+    pub expand_i: usize,
+    pub token_infos: Vec<p_token_info_t>,
+    pub nums: Vec<i64>,
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >>
+MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -0,0 +1,19 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.rust.propane

@@ -0,0 +1,41 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let result = p_lex(context, out_token_info);
+    if result != P_SUCCESS {
+        return result;
+    }
+    if out_token_info.token == TOKEN_lparen {
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
+        if inner_result != P_SUCCESS {
+            return inner_result;
+        }
+        let value = p_result_Start(context);
+        /* p_parse_inner rewound the input so ')' was not consumed; consume it. */
+        let mut rparen_info = p_token_info_t::default();
+        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
+        assert_eq!(TOKEN_rparen, rparen_info.token);
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue = p_value(value);
+    }
+    P_SUCCESS
+}
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.c.propane

@@ -0,0 +1,17 @@
+<<
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/parse_inner_nested_tree.d.propane

@@ -0,0 +1,17 @@
+<<
+import test_parse_inner_nested_tree;
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/parse_inner_nested_tree.rust.propane

@@ -0,0 +1,44 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let result = p_lex(context, out_token_info);
+    if result != P_SUCCESS {
+        return result;
+    }
+    if out_token_info.token == TOKEN_lparen {
+        let start_position = out_token_info.position;
+        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
+        if inner_result != P_SUCCESS {
+            return inner_result;
+        }
+        /* Read the inner subtree's span before re-borrowing context to lex. */
+        let inner = p_result_Start(context);
+        assert!(inner.valid());
+        let inner_start_col = inner.position().col;
+        let inner_end_col = inner.end_position().col;
+        let mut rparen_info = p_token_info_t::default();
+        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
+        assert_eq!(TOKEN_rparen, rparen_info.token);
+        assert_eq!(start_position.col + 1, inner_start_col);
+        assert_eq!(rparen_info.position.col - 1, inner_end_col);
+        /* Synthesize a num token spanning the whole "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    P_SUCCESS
+}
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,42 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
+    end
+
+    it "does not emit #line directives with noline statement" do
+      input = <<EOF
+noline;
+
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input, "test.propane")
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  a = b;\n  return c;]
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  writeln("Hello there");]
     end
 
     it "supports mode labels" do
@@ -144,7 +179,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -197,7 +232,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/rewind.c.propane

@@ -0,0 +1,23 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int value);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -0,0 +1,20 @@
+<<
+import test_rewind;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/rewind.rust.propane

@@ -0,0 +1,67 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    loop {
+        let result = p_lex(context, out_token_info);
+        if result != P_SUCCESS {
+            return result;
+        }
+        if out_token_info.token == TOKEN_repeat {
+            let mut count_info = p_token_info_t::default();
+            assert_eq!(P_SUCCESS, p_lex(context, &mut count_info));
+            assert_eq!(TOKEN_num, count_info.token);
+            let mut brace_info = p_token_info_t::default();
+            assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info));
+            assert_eq!(TOKEN_lbrace, brace_info.token);
+            context.remaining = p_value_get(&count_info.pvalue);
+            context.body_index = p_input_index(context);
+            context.body_position = p_position(context);
+            continue;
+        }
+        if out_token_info.token == TOKEN_rbrace {
+            if context.remaining > 1 {
+                context.remaining -= 1;
+                let bi = context.body_index;
+                let bp = context.body_position;
+                p_set_input_index(context, bi);
+                p_set_position(context, bp);
+                continue;
+            }
+            context.remaining = 0;
+            continue;
+        }
+        if out_token_info.token == TOKEN_num {
+            context.num_cols.push(out_token_info.position.col);
+        }
+        return result;
+    }
+}
+>>
+
+context_user_fields <<
+    pub nums: Vec<i64>,
+    pub num_cols: Vec<u32>,
+    pub remaining: i64,
+    pub body_index: usize,
+    pub body_position: p_position_t,
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << ${context.nums}.push($1 + $3); >>

spec/test_basic_math_grammar.rs

@@ -0,0 +1,16 @@
+use testparser::*;
+
+fn main() {
+    let cases: [(&[u8], u64); 4] = [
+        (b"1 + 2 * 3 + 4", 11),
+        (b"1 * 2 ** 4 * 3", 48),
+        (b"(1 + 2) * 3 + 4", 13),
+        (b"(2 * 2) ** 3 + 4 + 5", 73),
+    ];
+    for (input, expected) in cases {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(expected, p_result(&context));
+        p_context_delete(context);
+    }
+}

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_custom_lex_fn.rs

@@ -0,0 +1,8 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"cbacba");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(0x932187932187, p_result(&c));
+    p_context_delete(c);
+}

spec/test_drop_code_block.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"    # comment 1\n#    comment 2\na\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_error_positions.rs

@@ -0,0 +1,35 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a 42");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a\n123\na  a");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(4, p_position(&c).col);
+    assert_eq!(TOKEN_a, p_token(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"12");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
+    assert_eq!(1, p_position(&c).row);
+    assert_eq!(1, p_position(&c).col);
+    assert_eq!(TOKEN_num, p_token(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a 12\n\nab");
+    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(2, p_position(&c).col);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a 12\n\na\n\n77\na   \xAA");
+    assert_eq!(P_DECODE_ERROR, p_parse(&mut c));
+    assert_eq!(6, p_position(&c).row);
+    assert_eq!(5, p_position(&c).col);
+    assert_eq!("a", p_token_names[TOKEN_a as usize]);
+    assert_eq!("num", p_token_names[TOKEN_num as usize]);
+    p_context_delete(c);
+}

spec/test_field_aliases.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"foo1\nbar2");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_input_index.c

@@ -0,0 +1,60 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   drop /\\s+/;
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_input_index() reports the parser/lexer's current byte
+     * offset into the input text. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert_eq(0u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        char const * input = "a b";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        /* The dropped space between `a` and `b` advances input_index too. */
+        assert_eq(3u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_input_index.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_input_index.c for details on the grammar and cases. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_input_index(context) == 0);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        string input = "a b";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_input_index(context) == 1);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 3);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+}

spec/test_input_index.rs

@@ -0,0 +1,28 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"ab");
+    assert_eq!(0, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a b");
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, p_input_index(&c));
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(2, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    let follow = [TOKEN_b];
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow));
+    assert_eq!(2, p_input_index(&c));
+    p_context_delete(c);
+}

spec/test_lexer.rs

@@ -0,0 +1,49 @@
+use testparser::*;
+
+fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) {
+    assert_eq!(row, ti.position.row);
+    assert_eq!(col, ti.position.col);
+    assert_eq!(erow, ti.end_position.row);
+    assert_eq!(ecol, ti.end_position.col);
+    assert_eq!(len, ti.length);
+    assert_eq!(token, ti.token);
+}
+
+fn main() {
+    let mut cp: p_code_point_t = 0;
+    let mut cpl: u8 = 0;
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl));
+    assert_eq!('5' as u32, cp);
+    assert_eq!(1, cpl);
+
+    assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl));
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl));
+    assert_eq!(0xA9, cp);
+    assert_eq!(2, cpl);
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl));
+    assert_eq!(0x1F9E1, cp);
+    assert_eq!(4, cpl);
+
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl));
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl));
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl));
+
+    let mut context = p_context_new(b"5 + 4 * \n677 + 567");
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF);
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF);
+    p_context_delete(context);
+}

spec/test_lexer_match_text.rs

@@ -0,0 +1,8 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"identifier_123");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+}

spec/test_lexer_modes.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abc \"a string\" def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abc \"abc def\" def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_lexer_multiple_modes.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abc.def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abc .  abc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_lexer_positions.c

@@ -0,0 +1,50 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 1u);
+    assert(context->last_start.col == 1u);
+    assert(context->last_end.row == 1u);
+    assert(context->last_end.col == 3u);
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context->last_start.row == token_info.position.row);
+    assert(context->last_start.col == token_info.position.col);
+    assert(context->last_end.row == token_info.end_position.row);
+    assert(context->last_end.col == token_info.end_position.col);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 3u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 6u);
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 8u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 9u);
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context->text_position.row == 3u);
+    assert(context->text_position.col == 1u);
+
+    p_context_delete(context);
+    return 0;
+}

spec/test_lexer_positions.d

@@ -0,0 +1,42 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new(input);
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(1, 1));
+    assert(context.last_end == p_position_t(1, 3));
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context.last_start == token_info.position);
+    assert(context.last_end == token_info.end_position);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 3));
+    assert(context.last_end == p_position_t(2, 6));
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 8));
+    assert(context.last_end == p_position_t(2, 9));
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context.text_position == p_position_t(3, 1));
+}

spec/test_lexer_positions.rs

@@ -0,0 +1,38 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"abc\n  defg hi\n!");
+    let mut ti = p_token_info_t::default();
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(1, c.last_start.row);
+    assert_eq!(1, c.last_start.col);
+    assert_eq!(1, c.last_end.row);
+    assert_eq!(3, c.last_end.col);
+    assert_eq!(c.last_start.row, ti.position.row);
+    assert_eq!(c.last_start.col, ti.position.col);
+    assert_eq!(c.last_end.row, ti.end_position.row);
+    assert_eq!(c.last_end.col, ti.end_position.col);
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(2, c.last_start.row);
+    assert_eq!(3, c.last_start.col);
+    assert_eq!(2, c.last_end.row);
+    assert_eq!(6, c.last_end.col);
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(2, c.last_start.row);
+    assert_eq!(8, c.last_start.col);
+    assert_eq!(2, c.last_end.row);
+    assert_eq!(9, c.last_end.col);
+
+    assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti));
+    assert_eq!(42, p_user_terminate_code(&c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(1, p_position(&c).col);
+
+    p_context_delete(c);
+}

spec/test_lexer_result_value.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"x");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(1, p_result(&context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"fabulous");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(8, p_result(&context));
+    p_context_delete(context);
+}

spec/test_lexer_unknown_character.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"x");
+    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"123");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(123, p_result(&context));
+    p_context_delete(context);
+}

spec/test_macros.c

@@ -0,0 +1,118 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+#include <assert.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+static p_context_t * context;
+size_t n_tokens;
+p_token_info_t token_infos[10];
+
+/* Capture the macro body tokens (everything up to the closing '}') into
+ * token_infos[]. Called from mylexfn() right after the definition's '{' has
+ * been lexed, so the input cursor is positioned at the first body token. */
+static void capture_macro_body(void)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert_eq(result, P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < sizeof(token_infos) / sizeof(token_infos[0]));
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info->token)
+        {
+        case TOKEN_macro:
+            /* Start of a macro definition: "macro macroname { ... }". */
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                /* Use of a macro: replay its captured body tokens instead of
+                 * returning the macroname to the parser. */
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            /* Definition name: pass through and keep waiting for '{'. */
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                /* Consume and store the macro body now, before the parser gets
+                 * a chance to read its lookahead token (which would otherwise
+                 * swallow the first body token). */
+                capture_macro_body();
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int nums[10];
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    char const * input =
+        "macro @m { 23 + 200 }\n"
+        "66 + 100\n"
+        "@m\n"
+        "33 + 55\n"
+        "@m\n";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(n_nums, 4);
+    assert_eq(nums[0], 166);
+    assert_eq(nums[1], 223);
+    assert_eq(nums[2], 88);
+    assert_eq(nums[3], 223);
+
+    return 0;
+}

spec/test_macros.d

@@ -0,0 +1,116 @@
+import testparser;
+import testutils;
+
+size_t n_tokens;
+p_token_info_t[10] token_infos;
+
+// Capture the macro body tokens (everything up to the closing '}') into
+// token_infos[]. Called from mylexfn() right after the definition's '{' has
+// been lexed, so the input cursor is positioned at the first body token.
+void capture_macro_body(p_context_t * context)
+{
+    n_tokens = 0u;
+    for (;;)
+    {
+        size_t result = p_lex(context, &token_infos[n_tokens]);
+        assert(result == P_SUCCESS);
+        if (token_infos[n_tokens].token == TOKEN_rbrace)
+        {
+            break;
+        }
+        n_tokens++;
+        assert(n_tokens < token_infos.length);
+    }
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static bool defining;
+    static bool expanding;
+    static size_t expand_i;
+
+    for (;;)
+    {
+        if (expanding)
+        {
+            size_t ei = expand_i++;
+            if (expand_i >= n_tokens)
+            {
+                expanding = false;
+            }
+            *out_token_info = token_infos[ei];
+            return P_SUCCESS;
+        }
+
+        size_t lex_result = p_lex(context, out_token_info);
+        if (lex_result != P_SUCCESS)
+        {
+            return lex_result;
+        }
+
+        switch (out_token_info.token)
+        {
+        case TOKEN_macro:
+            // Start of a macro definition: "macro macroname { ... }".
+            defining = true;
+            break;
+        case TOKEN_macroname:
+            if (!defining)
+            {
+                // Use of a macro: replay its captured body tokens instead of
+                // returning the macroname to the parser.
+                expanding = true;
+                expand_i = 0u;
+                continue;
+            }
+            // Definition name: pass through and keep waiting for '{'.
+            break;
+        case TOKEN_lbrace:
+            if (defining)
+            {
+                // Consume and store the macro body now, before the parser gets
+                // a chance to read its lookahead token (which would otherwise
+                // swallow the first body token).
+                capture_macro_body(context);
+                defining = false;
+            }
+            break;
+        default:
+            defining = false;
+            break;
+        }
+        return lex_result;
+    }
+}
+
+size_t n_nums;
+int[10] nums;
+
+void record(int v)
+{
+    nums[n_nums++] = v;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "macro @m { 23 + 200 }\n" ~
+        "66 + 100\n" ~
+        "@m\n" ~
+        "33 + 55\n" ~
+        "@m\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert(n_nums == 4);
+    assert(nums[0] == 166);
+    assert(nums[1] == 223);
+    assert(nums[2] == 88);
+    assert(nums[3] == 223);
+}

spec/test_macros.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    let input = b"macro @m { 23 + 200 }\n66 + 100\n@m\n33 + 55\n@m\n";
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(vec![166, 223, 88, 223], c.nums);
+    p_context_delete(c);
+}

spec/test_match_backslashes.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"\x07\x08\t\n\x0b\x0c\rt");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_multiple_parsers.rs

@@ -0,0 +1,12 @@
+use testparsermyp1 as m1;
+use testparsermyp2 as m2;
+
+fn main() {
+    let mut context1 = m1::myp1_context_new(b"a\n1");
+    assert_eq!(m1::P_SUCCESS, m1::myp1_parse(&mut context1));
+    m1::myp1_context_delete(context1);
+
+    let mut context2 = m2::myp2_context_new(b"bcb");
+    assert_eq!(m2::P_SUCCESS, m2::myp2_parse(&mut context2));
+    m2::myp2_context_delete(context2);
+}

spec/test_named_optional_rule_component_tree.c

@@ -9,46 +9,42 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start->a == NULL);
-    assert(start->pToken2 != NULL);
-    assert_eq(TOKEN_b, start->pToken2->token);
-    assert(start->pR3 == NULL);
-    assert(start->pR == NULL);
-    assert(start->r == NULL);
+    Start start = p_result(context);
+    assert(!p_node_valid(p_Start_a(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
+    assert(!p_node_valid(p_Start_pR3(start)));
+    assert(!p_node_valid(p_Start_pR(start)));
+    assert(!p_node_valid(p_Start_r(start)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "abcd";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->a != NULL);
-    assert_eq(TOKEN_a, start->pToken1->token);
-    assert(start->pToken2 != NULL);
-    assert(start->pR3 != NULL);
-    assert(start->pR != NULL);
-    assert(start->r != NULL);
-    assert(start->pR == start->pR3);
-    assert(start->pR == start->r);
-    assert_eq(TOKEN_c, start->pR->pToken1->token);
+    assert(p_node_valid(p_Start_a(start)));
+    assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_pR3(start)));
+    assert(p_node_valid(p_Start_pR(start)));
+    assert(p_node_valid(p_Start_r(start)));
+    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
+    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_r(start)));
+    assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "bdc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->a == NULL);
-    assert(start->pToken2 != NULL);
-    assert(start->r != NULL);
-    assert_eq(TOKEN_d, start->pR->pToken1->token);
+    assert(!p_node_valid(p_Start_a(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_r(start)));
+    assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;
 }
-

spec/test_named_optional_rule_component_tree.d

@@ -10,37 +10,42 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
+    Start start = p_result(context);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
     assert_eq(TOKEN_b, start.pToken2.token);
-    assert(start.pR3 is null);
-    assert(start.pR is null);
-    assert(start.r is null);
+    assert(!start.pR3.valid);
+    assert(!start.pR.valid);
+    assert(!start.r.valid);
+
+    p_context_delete(context);
 
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 != null);
+    assert(start.pToken1.valid);
     assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2 != null);
-    assert(start.pR3 != null);
-    assert(start.pR != null);
-    assert(start.r != null);
+    assert(start.pToken2.valid);
+    assert(start.pR3.valid);
+    assert(start.pR.valid);
+    assert(start.r.valid);
     assert(start.pR == start.pR3);
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_context_delete(context);
+
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert(start.pR !is null);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
+    assert(start.pR.valid);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_context_delete(context);
 }

spec/test_named_optional_rule_component_tree.rs

@@ -0,0 +1,43 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.a().valid());
+        assert!(start.pToken2().valid());
+        assert_eq!(TOKEN_b, start.pToken2().token());
+        assert!(!start.pR3().valid());
+        assert!(!start.pR().valid());
+        assert!(!start.r().valid());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcd");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.a().valid());
+        assert_eq!(TOKEN_a, start.pToken1().token());
+        assert!(start.pToken2().valid());
+        assert!(start.pR3().valid());
+        assert!(start.pR().valid());
+        assert!(start.r().valid());
+        assert_eq!(start.pR().node_id(), start.pR3().node_id());
+        assert_eq!(start.pR().node_id(), start.r().node_id());
+        assert_eq!(TOKEN_c, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bdc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.a().valid());
+        assert!(start.pToken2().valid());
+        assert!(start.r().valid());
+        assert_eq!(TOKEN_d, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+}

spec/test_optional_rule_component.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    for input in [&b"b"[..], &b"abcd"[..], &b"abdc"[..]] {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        p_context_delete(context);
+    }
+}

spec/test_optional_rule_component_tree.c

@@ -9,43 +9,39 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start->pToken1 == NULL);
-    assert(start->pToken2 != NULL);
-    assert_eq(TOKEN_b, start->pToken2->token);
-    assert(start->pR3 == NULL);
-    assert(start->pR == NULL);
+    Start start = p_result(context);
+    assert(!p_node_valid(p_Start_pToken1(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
+    assert(!p_node_valid(p_Start_pR3(start)));
+    assert(!p_node_valid(p_Start_pR(start)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "abcd";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->pToken1 != NULL);
-    assert_eq(TOKEN_a, start->pToken1->token);
-    assert(start->pToken2 != NULL);
-    assert(start->pR3 != NULL);
-    assert(start->pR != NULL);
-    assert(start->pR == start->pR3);
-    assert_eq(TOKEN_c, start->pR->pToken1->token);
+    assert(p_node_valid(p_Start_pToken1(start)));
+    assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_pR3(start)));
+    assert(p_node_valid(p_Start_pR(start)));
+    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
+    assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "bdc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->pToken1 == NULL);
-    assert(start->pToken2 != NULL);
-    assert(start->pR != NULL);
-    assert_eq(TOKEN_d, start->pR->pToken1->token);
+    assert(!p_node_valid(p_Start_pToken1(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_pR(start)));
+    assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;
 }
-

spec/test_optional_rule_component_tree.d

@@ -10,34 +10,39 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
+    Start start = p_result(context);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
     assert_eq(TOKEN_b, start.pToken2.token);
-    assert(start.pR3 is null);
-    assert(start.pR is null);
+    assert(!start.pR3.valid);
+    assert(!start.pR.valid);
+
+    p_context_delete(context);
 
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 != null);
+    assert(start.pToken1.valid);
     assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2 != null);
-    assert(start.pR3 != null);
-    assert(start.pR != null);
+    assert(start.pToken2.valid);
+    assert(start.pR3.valid);
+    assert(start.pR.valid);
     assert(start.pR == start.pR3);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_context_delete(context);
+
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert(start.pR !is null);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
+    assert(start.pR.valid);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_context_delete(context);
 }

spec/test_optional_rule_component_tree.rs

@@ -0,0 +1,40 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.pToken1().valid());
+        assert!(start.pToken2().valid());
+        assert_eq!(TOKEN_b, start.pToken2().token());
+        assert!(!start.pR3().valid());
+        assert!(!start.pR().valid());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcd");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pToken1().valid());
+        assert_eq!(TOKEN_a, start.pToken1().token());
+        assert!(start.pToken2().valid());
+        assert!(start.pR3().valid());
+        assert!(start.pR().valid());
+        assert_eq!(start.pR().node_id(), start.pR3().node_id());
+        assert_eq!(TOKEN_c, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bdc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.pToken1().valid());
+        assert!(start.pToken2().valid());
+        assert!(start.pR().valid());
+        assert_eq!(TOKEN_d, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+}

spec/test_parse_inner.c

@@ -0,0 +1,73 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (chain reduce):
+     *   Start -> Y      << $$ = $1; >>
+     *   Y     -> a      << $$ = $1; >>
+     *   token a         << $$ = 1; >>
+     *
+     * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF,
+     * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with
+     * `b` as a follow token should succeed via the reduce-side retry chain
+     * (Y then Start) followed by the shift-side retry hitting $EOF at the
+     * final state. */
+
+    /* Standard parse succeeds on complete input. */
+    char const * input = "a";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(1u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with an empty (NULL) follow-token vector behaves like a
+     * standard parse. */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    {
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(1u, (size_t)p_result_Start(context));
+    }
+    p_context_delete(context);
+
+    /* parse_inner with a non-matching follow token still fails. The grammar
+     * can't consume `b` and it isn't listed as a follow token, so the retries
+     * do not fire. */
+    {
+        input = "ab";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner.c for details on the grammar and cases. */
+
+    /* Standard parse succeeds on complete input. */
+    string input = "a";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* Standard parse fails when there's an unexpected trailing token. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
+     * followed by the shift-side retry hitting $EOF at the final state. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a null follow-token slice behaves like a standard
+     * parse. */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner behaves like a standard parse when the input matches the
+     * grammar fully. */
+    input = "a";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 1);
+
+    /* parse_inner with a non-matching follow token still fails. */
+    input = "ab";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_eof = [TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN);
+}

spec/test_parse_inner.rs

@@ -0,0 +1,30 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[]));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN___EOF]));
+    p_context_delete(c);
+}

spec/test_parse_inner_nested.c

@@ -0,0 +1,78 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (integer evaluator; parentheses handled by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token lparen /\(/; token rparen /\)/; token plus /\+/;
+ *   token num /\d+/          << ... atoi ... >>
+ *   Start -> Expr            << $$ = $1; >>
+ *   Expr  -> num             << $$ = $1; >>
+ *   Expr  -> Expr plus num   << $$ = $1 + $3; >>
+ *
+ * The tokens lparen and rparen appear in no grammar rule. Instead, when the
+ * lex function lexes a '(', it performs a nested parse (p_parse_inner_Start)
+ * of the parenthesized sub-expression -- reentrantly, while the outer parse is
+ * still suspended in this callback -- reads the computed value with
+ * p_result_Start, consumes the ')' that p_parse_inner deliberately left in the
+ * input, and hands a single synthesized num token carrying that value back to
+ * the outer parse. Nested groups recurse this process to arbitrary depth. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->pvalue = p_value(value);
+    }
+    return P_SUCCESS;
+}
+
+static int eval(char const * input)
+{
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5u, (size_t)eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3u, (size_t)eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14u, (size_t)eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37u, (size_t)eval("2 + (10 + (20 + 5))"));
+    assert_eq(15u, (size_t)eval("(1 + 2) + (3 + (4 + 5))"));
+
+    return 0;
+}

spec/test_parse_inner_nested.d

@@ -0,0 +1,64 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        /* Nested parse of the parenthesized sub-expression, stopping at the
+         * closing ')' follow token. This re-enters the parser while the outer
+         * parse is suspended in this lex callback. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        int value = p_result_Start(context);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* Replace the '(' token with a synthesized num carrying the nested
+         * parse result. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue = p_value(value);
+    }
+    return P_SUCCESS;
+}
+
+int eval(string input)
+{
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int value = p_result(context);
+    p_context_delete(context);
+    return value;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* No parentheses: plain outer parse. */
+    assert_eq(5, eval("2 + 3"));
+    /* A single group evaluated by the nested parse. */
+    assert_eq(3, eval("(1 + 2)"));
+    /* A group in the middle of an outer expression. */
+    assert_eq(14, eval("2 + (3 + 4) + 5"));
+    /* Nested groups: the nested parse re-enters itself. */
+    assert_eq(37, eval("2 + (10 + (20 + 5))"));
+    assert_eq(15, eval("(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested.rs

@@ -0,0 +1,17 @@
+use testparser::*;
+
+fn eval(input: &[u8]) -> i64 {
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    let v = p_result(&c);
+    p_context_delete(c);
+    v
+}
+
+fn main() {
+    assert_eq!(5, eval(b"2 + 3"));
+    assert_eq!(3, eval(b"(1 + 2)"));
+    assert_eq!(14, eval(b"2 + (3 + 4) + 5"));
+    assert_eq!(37, eval(b"2 + (10 + (20 + 5))"));
+    assert_eq!(15, eval(b"(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested_tree.c

@@ -0,0 +1,91 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar: see the D variant / spec. Tree generation mode; parentheses handled
+ * by the lex function. Tree nodes live in the context arena. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info->token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info->position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t follow_tokens[] = { TOKEN_rparen };
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start inner = p_result_Start(context);
+        assert(p_node_valid(inner));
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq((size_t)(start_position.col + 1u), (size_t)p_node_position(inner).col);
+        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)p_node_end_position(inner).col);
+        /* The inner subtree is discarded (the lexer synthesizes a num token in
+         * its place), but its nodes remain in the shared context arena and are
+         * freed with the context. */
+        out_token_info->token = TOKEN_num;
+        out_token_info->position = start_position;
+        out_token_info->end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    char const * input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start tree = p_result(context);
+    assert(p_node_valid(tree));
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr top = p_Start_pExpr(tree);
+    assert(p_node_valid(top));
+    assert(p_node_valid(p_Expr_pExpr(top)));
+    assert(p_node_valid(p_Expr_pToken2(top)));
+    assert(p_node_valid(p_Expr_pToken3(top)));
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken2(top)).row);
+    assert_eq(9u, (size_t)p_node_position(p_Expr_pToken2(top)).col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken3(top)).row);
+    assert_eq(11u, (size_t)p_node_position(p_Expr_pToken3(top)).col);
+    assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken3(top)).row);
+    assert_eq(17u, (size_t)p_node_end_position(p_Expr_pToken3(top)).col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr left = p_Expr_pExpr(top);
+    assert(p_node_valid(p_Expr_pToken1(left)));
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).row);
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).col);
+    assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken1(left)).row);
+    assert_eq(7u, (size_t)p_node_end_position(p_Expr_pToken1(left)).col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, (size_t)p_node_position(tree).col);
+    assert_eq(17u, (size_t)p_node_end_position(tree).col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_nested_tree.d

@@ -0,0 +1,92 @@
+import testparser;
+import testutils;
+
+/* Grammar: see test_parse_inner_nested_tree.c. */
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    size_t result = p_lex(context, out_token_info);
+    if (result != P_SUCCESS)
+    {
+        return result;
+    }
+    if (out_token_info.token == TOKEN_lparen)
+    {
+        p_position_t start_position = out_token_info.position;
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        p_token_t[] follow_tokens = [TOKEN_rparen];
+        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
+        if (inner_result != P_SUCCESS)
+        {
+            return inner_result;
+        }
+        Start inner = p_result_Start(context);
+        assert(inner.valid);
+        /* p_parse_inner rewound the input so that ')' was not consumed; consume
+         * it now. */
+        p_token_info_t rparen_info;
+        size_t rparen_result = p_lex(context, &rparen_info);
+        assert(rparen_result == P_SUCCESS);
+        assert(rparen_info.token == TOKEN_rparen);
+        /* The subtree covers the region strictly between the parentheses. */
+        assert_eq(start_position.col + 1u, inner.position.col);
+        assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
+        /* The inner subtree is discarded (the lexer synthesizes a num token in
+         * its place), but its nodes remain in the shared context arena and are
+         * freed with the context. */
+        /* Synthesize a num token spanning the entire "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    return P_SUCCESS;
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
+     * lexer into a single num token spanning its group. */
+    string input = "(3 + 4) + (5 + 6)";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    Start tree = p_result(context);
+    assert(tree.valid);
+
+    /* Start -> Expr, where the top Expr is "Expr plus num". */
+    Expr top = tree.pExpr;
+    assert(top.valid);
+    assert(top.pExpr.valid);
+    assert(top.pToken2.valid);
+    assert(top.pToken3.valid);
+
+    /* The '+' joining the two groups is at column 9. */
+    assert_eq(1u, top.pToken2.position.row);
+    assert_eq(9u, top.pToken2.position.col);
+
+    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
+    assert_eq(1u, top.pToken3.position.row);
+    assert_eq(11u, top.pToken3.position.col);
+    assert_eq(1u, top.pToken3.end_position.row);
+    assert_eq(17u, top.pToken3.end_position.col);
+
+    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
+     * columns 1..7. */
+    Expr left = top.pExpr;
+    assert(left.pToken1.valid);
+    assert_eq(1u, left.pToken1.position.row);
+    assert_eq(1u, left.pToken1.position.col);
+    assert_eq(1u, left.pToken1.end_position.row);
+    assert_eq(7u, left.pToken1.end_position.col);
+
+    /* The whole tree spans columns 1..17. */
+    assert_eq(1u, tree.position.col);
+    assert_eq(17u, tree.end_position.col);
+
+    p_context_delete(context);
+}

spec/test_parse_inner_nested_tree.rs

@@ -0,0 +1,36 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"(3 + 4) + (5 + 6)");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let tree = p_result(&c);
+        assert!(tree.valid());
+
+        /* Start -> Expr, where the top Expr is "Expr plus num". */
+        let top = tree.pExpr();
+        assert!(top.valid());
+        assert!(top.pExpr().valid());
+        assert!(top.pToken2().valid());
+        assert!(top.pToken3().valid());
+
+        /* The '+' joining the two groups is at column 9. */
+        assert_eq!(1, top.pToken2().position().row);
+        assert_eq!(9, top.pToken2().position().col);
+
+        /* Right operand: synthesized num for "(5 + 6)", columns 11..17. */
+        assert_eq!(11, top.pToken3().position().col);
+        assert_eq!(17, top.pToken3().end_position().col);
+
+        /* Left operand: synthesized num for "(3 + 4)", columns 1..7. */
+        let left = top.pExpr();
+        assert!(left.pToken1().valid());
+        assert_eq!(1, left.pToken1().position().col);
+        assert_eq!(7, left.pToken1().end_position().col);
+
+        /* The whole tree spans columns 1..17. */
+        assert_eq!(1, tree.position().col);
+        assert_eq!(17, tree.end_position().col);
+    }
+    p_context_delete(c);
+}

spec/test_parse_inner_recursive.c

@@ -0,0 +1,77 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (recursive):
+     *   Start -> a Start b   << $$ = $2; >>
+     *   Start -> c           << $$ = $1; >>
+     *   token a << $$ = 1; >>
+     *   token b << $$ = 2; >>
+     *   token c << $$ = 3; >>
+     *
+     * Here `Start` can appear in the middle of another `Start` rule, so the
+     * inner-parse follow-token success must be blocked whenever an unfinished
+     * outer `Start -> a Start b` remains on the parse stack (i.e. the parse
+     * stack contains more than just the initial state and the reduced start
+     * rule set). */
+
+    /* Standard parse of `c` succeeds. */
+    char const * input = "c";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `acb` succeeds (full outer rule). */
+    input = "acb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert_eq(3u, (size_t)p_result_Start(context));
+    p_context_delete(context);
+
+    /* Standard parse of `ac` fails (`b` missing). */
+    input = "ac";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner with `ac` and follow token `b` also fails: even though the
+     * inner `Start -> c` reduces and `Start` is shifted, the outer
+     * `Start -> a Start . b` is still on the stack (stack length > 2), so the
+     * "reduced start rule is the only thing on the parse stack" invariant
+     * blocks the shift-side follow-token success. */
+    {
+        input = "ac";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF };
+        assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN);
+        p_context_delete(context);
+    }
+
+    /* parse_inner with `acb` (complete outer rule) succeeds via the standard
+     * path. */
+    {
+        input = "acb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    /* parse_inner with just `c` succeeds via the standard path even when a
+     * follow-token vector is supplied. */
+    {
+        input = "c";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(3u, (size_t)p_result_Start(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_recursive.d

@@ -0,0 +1,49 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_recursive.c for details on the grammar. */
+
+    /* Standard parse of `c` succeeds. */
+    string input = "c";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `acb` succeeds. */
+    input = "acb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* Standard parse of `ac` fails. */
+    input = "ac";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `ac` fails: outer rule still on the stack. */
+    input = "ac";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF];
+    assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner with `acb` succeeds via the standard path. */
+    input = "acb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+
+    /* parse_inner with just `c` succeeds via the standard path. */
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_Start(context) == 3);
+}

spec/test_parse_inner_recursive.rs

@@ -0,0 +1,31 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"acb");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ac");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ac");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN_b, TOKEN___EOF]));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"acb");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+}

spec/test_parse_inner_shared.c

@@ -0,0 +1,104 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar:
+     *   start Start;
+     *   start R1;
+     *   Start -> R1 a;
+     *   Start -> R2 b;
+     *   R1 -> a b   << $$ = 11; >>
+     *   R2 -> a b   << $$ = 22; >>
+     *   token a; token b;
+     *
+     * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within
+     * parse_Start, the generated parser differentiates the reduce by
+     * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2
+     * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional
+     * on any lookahead. This test exercises p_parse_inner_R1() to confirm
+     * that reductions to R1 succeed even when the incoming follow token is
+     * not the natural lookahead used by parse_Start's disambiguation. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the
+     * shared "a b" state. */
+    char const * input = "aba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Start(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    /* Standard parse of R1 fails on "abb" (unexpected trailing token). */
+    input = "abb";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+    p_context_delete(context);
+
+    /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
+     * that parse_Start uses to select R2 over R1 in the ambiguous state, from
+     * R1's start state the reduce to R1 is unconditional, and the follow-
+     * token shift retry at the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed from
+     * the input: p_position() should point to the follow token, and a
+     * subsequent p_lex() should return it. */
+    {
+        input = "abb";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `b` is at column 3 (1-based). */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token
+     * parse_Start uses to select R1, and it works here as a follow token
+     * too. */
+    {
+        input = "aba";
+        context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_a };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(11u, (size_t)p_result_R1(context));
+        /* Follow token `a` is at column 3 (1-based) and remains in the
+         * input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        p_context_delete(context);
+    }
+
+    /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS);
+    assert_eq(11u, (size_t)p_result_R1(context));
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parse_inner_shared.d

@@ -0,0 +1,72 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_shared.c for details on the grammar. */
+
+    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */
+    string input = "aba";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_Start(context) == P_SUCCESS);
+
+    /* Standard parse of R1 succeeds on "ab". */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+
+    /* Standard parse of R1 fails on "abb". */
+    input = "abb";
+    context = p_context_new(input);
+    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
+
+    /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
+     * parse_Start would use to select R2 over R1, but from R1's own start
+     * state R1 reduces unconditionally, and the follow-token shift retry at
+     * the R1-accepting state completes the parse.
+     *
+     * The follow token that completed the parse must not be consumed: a
+     * subsequent p_lex() should return it. */
+    input = "abb";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_b = [TOKEN_b];
+    assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    p_position_t pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    p_token_info_t token_info;
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_b);
+    assert(token_info.position.row == 1);
+    assert(token_info.position.col == 3);
+
+    /* parse_inner_R1("aba", [a]) also succeeds. */
+    input = "aba";
+    context = p_context_new(input);
+    p_token_t[] follow_tokens_a = [TOKEN_a];
+    assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+    pos = p_position(context);
+    assert(pos.row == 1);
+    assert(pos.col == 3);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_a);
+
+    /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
+    input = "ab";
+    context = p_context_new(input);
+    assert(p_parse_inner_R1(context, null) == P_SUCCESS);
+    assert(p_result_R1(context) == 11);
+}

spec/test_parse_inner_shared.rs

@@ -0,0 +1,47 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"aba");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
+    assert_eq!(11, p_result_R1(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_R1(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
+    assert_eq!(11, p_result_R1(&c));
+    let pos = p_position(&c);
+    assert_eq!(1, pos.row);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, ti.position.col);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"aba");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_a]));
+    assert_eq!(11, p_result_R1(&c));
+    let pos = p_position(&c);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[]));
+    assert_eq!(11, p_result_R1(&c));
+    p_context_delete(c);
+}

spec/test_parse_inner_tree.c

@@ -0,0 +1,68 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* See the D variant / grammar comments for details. In tree mode the tree
+     * nodes live in the context arena and are freed with p_context_delete(). */
+
+    /* Baseline: p_parse_R1 works on "ab" and the returned tree is
+     * well-formed. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 tree = p_result_R1(context);
+        assert(p_node_valid(tree));
+        assert(p_node_valid(p_R1_pToken1(tree)));
+        assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
+        assert(p_node_valid(p_R1_pToken2(tree)));
+        assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
+        p_context_delete(context);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 tree = p_result_R1(context);
+        assert(p_node_valid(tree));
+        assert(p_node_valid(p_R1_pToken1(tree)));
+        assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
+        assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).row);
+        assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).col);
+        assert(p_node_valid(p_R1_pToken2(tree)));
+        assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
+        assert_eq(1u, (size_t)p_node_position(p_R1_pToken2(tree)).row);
+        assert_eq(2u, (size_t)p_node_position(p_R1_pToken2(tree)).col);
+
+        /* The R1 tree covers positions 1..2 - the third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert_eq(1u, (size_t)p_node_position(tree).row);
+        assert_eq(1u, (size_t)p_node_position(tree).col);
+        assert_eq(1u, (size_t)p_node_end_position(tree).row);
+        assert_eq(2u, (size_t)p_node_end_position(tree).col);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(3u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(3u, (size_t)token_info.position.col);
+
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_parse_inner_tree.d

@@ -0,0 +1,68 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_parse_inner_tree.c for details on the grammar and cases. */
+
+    /* Baseline: p_parse_R1 works on "ab". */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_R1(context) == P_SUCCESS);
+        R1 tree = p_result_R1(context);
+        assert(tree.valid);
+        assert(tree.pToken1.valid);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken2.valid);
+        assert(tree.pToken2.token == TOKEN_b);
+        p_context_delete(context);
+    }
+
+    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
+     * the parse, returns a well-formed tree, and leaves the follow token
+     * unconsumed. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
+
+        /* Tree is well-formed. */
+        R1 tree = p_result_R1(context);
+        assert(tree.valid);
+        assert(tree.pToken1.valid);
+        assert(tree.pToken1.token == TOKEN_a);
+        assert(tree.pToken1.position.row == 1);
+        assert(tree.pToken1.position.col == 1);
+        assert(tree.pToken2.valid);
+        assert(tree.pToken2.token == TOKEN_b);
+        assert(tree.pToken2.position.row == 1);
+        assert(tree.pToken2.position.col == 2);
+
+        /* The R1 tree covers positions 1..2. The third `b` at column 3 is
+         * the follow token and is not part of the tree. */
+        assert(tree.position.row == 1);
+        assert(tree.position.col == 1);
+        assert(tree.end_position.row == 1);
+        assert(tree.end_position.col == 2);
+
+        /* Follow token remains in the input. */
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 3);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 3);
+
+        p_context_delete(context);
+    }
+}

spec/test_parse_inner_tree.rs

@@ -0,0 +1,36 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
+    {
+        let tree = p_result_R1(&c);
+        assert!(tree.valid());
+        assert!(tree.pToken1().valid());
+        assert_eq!(TOKEN_a, tree.pToken1().token());
+        assert!(tree.pToken2().valid());
+        assert_eq!(TOKEN_b, tree.pToken2().token());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
+    {
+        let tree = p_result_R1(&c);
+        assert!(tree.valid());
+        assert_eq!(TOKEN_a, tree.pToken1().token());
+        assert_eq!(1, tree.pToken1().position().row);
+        assert_eq!(1, tree.pToken1().position().col);
+        assert_eq!(TOKEN_b, tree.pToken2().token());
+        assert_eq!(2, tree.pToken2().position().col);
+        assert_eq!(1, tree.position().col);
+        assert_eq!(2, tree.end_position().col);
+    }
+    let pos = p_position(&c);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, ti.position.col);
+    p_context_delete(c);
+}

spec/test_parser_identical_rules_lookahead.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    for input in [&b"aba"[..], &b"abb"[..]] {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        p_context_delete(context);
+    }
+}

spec/test_parser_rule_from_multiple_states.rs

@@ -0,0 +1,18 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"a");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut context));
+    assert_eq!(1, p_position(&context).row);
+    assert_eq!(2, p_position(&context).col);
+    assert_eq!(TOKEN___EOF, p_token(&context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bb");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_parser_rule_user_code.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_parser_user_code_tree.c

@@ -0,0 +1,38 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context->start_n_fields);
+    assert_eq(11, context->start_a_value);
+    assert_eq(11, context->a_value);
+    assert_eq(22, context->b_value);
+    assert_eq(TOKEN_b, context->b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context->c_is_null);
+    assert_eq(1, context->c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context->alias_a_value);
+    assert_eq(22, context->alias_b_value);
+
+    Start start = p_result(context);
+    assert(p_node_valid(p_Start_pA(start)));
+    assert(p_node_valid(p_Start_pB(start)));
+    assert(!p_node_valid(p_Start_pC(start)));
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parser_user_code_tree.d

@@ -0,0 +1,39 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context.start_n_fields);
+    assert_eq(11, context.start_a_value);
+    assert_eq(11, context.a_value);
+    assert_eq(22, context.b_value);
+    assert_eq(TOKEN_b, context.b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context.c_is_null);
+    assert_eq(1, context.c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context.alias_a_value);
+    assert_eq(22, context.alias_b_value);
+
+    Start start = p_result(context);
+    assert(start.pA.valid);
+    assert(start.pB.valid);
+    assert(!start.pC.valid);
+    p_context_delete(context);
+}

spec/test_parser_user_code_tree.rs

@@ -0,0 +1,24 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+
+    assert_eq!(3, context.start_n_fields);
+    assert_eq!(11, context.start_a_value);
+    assert_eq!(11, context.a_value);
+    assert_eq!(22, context.b_value);
+    assert_eq!(TOKEN_b, context.b_token);
+    assert_eq!(1, context.c_is_null);
+    assert_eq!(1, context.c_field_is_null);
+    assert_eq!(11, context.alias_a_value);
+    assert_eq!(22, context.alias_b_value);
+
+    {
+        let start = p_result(&context);
+        assert!(start.pA().valid());
+        assert!(start.pB().valid());
+        assert!(!start.pC().valid());
+    }
+    p_context_delete(context);
+}

spec/test_parsing_json.rs

@@ -0,0 +1,31 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"{}");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_OBJECT, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"[]");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_ARRAY, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"-45.6");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_NUMBER, p_result(&c).id());
+    assert_eq!(-45.6, p_result(&c).number());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"{\"hi\":true}");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_OBJECT, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"[1, 2, \"three\", [4, 5], {\"six\": 6}]");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    let v = p_result(&c);
+    assert_eq!(JSON_ARRAY, v.id());
+    assert_eq!(5, v.array_len());
+    p_context_delete(c);
+}

spec/test_parsing_lists.rs

@@ -0,0 +1,11 @@
+use testparser::*;
+
+fn main() {
+    let cases: [(&[u8], u32); 3] = [(b"a", 1), (b"", 0), (b"aaaaaaaaaaaaaaaa", 16)];
+    for (input, expected) in cases {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(expected, p_result(&context));
+        p_context_delete(context);
+    }
+}

spec/test_pattern.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"defabcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_positions.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "\n tok2";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "  tok1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_positions.d

@@ -0,0 +1,27 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "\n tok2";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "  tok1";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_positions.rs

@@ -0,0 +1,19 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"    Hello\n\n        4200\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    println!();
+
+    let mut c = p_context_new(b"\n tok2");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    println!();
+
+    let mut c = p_context_new(b"  tok1");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_return_token_from_pattern.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"defghidef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_rewind.c

@@ -0,0 +1,124 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (statement list of additions; a "repeat" directive handled entirely
+ * by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token repeat /repeat/; token lbrace /\{/; token rbrace /\}/;
+ *   token plus /\+/; token num /\d+/ << ... atoi ... >>
+ *   Start      -> Statements;
+ *   Statements -> ;
+ *   Statements -> Statement Statements;
+ *   Statement  -> Add;
+ *   Add        -> num plus num  << record($1 + $3); >>
+ *
+ * Scenario: a "repeat <count> { <body> }" directive that expands its body
+ * <count> times, similar to loop unrolling in a configuration DSL. The tokens
+ * repeat, lbrace, and rbrace appear in no grammar rule; the lex function
+ * interprets the directive and feeds the body's tokens to the parser <count>
+ * times. Rather than buffering the body tokens, the lex function records the
+ * input byte offset and text position at the start of the body (with
+ * p_input_index() and p_position()) and, each time it reaches the closing '}',
+ * rewinds the lexer back to that point (with p_set_input_index() and
+ * p_set_position()) to re-read the body from the original input. Rewinding the
+ * text position as well as the byte offset means each expansion reports the
+ * same token positions as the first. */
+
+static int nums[16];
+static size_t n_nums;
+static uint32_t num_cols[16];
+static size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info->token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info->token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info->token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info->position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    char const * input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* The additions were recorded once per body expansion, then once more for
+     * the trailing statement. */
+    assert_eq(4u, n_nums);
+    assert_eq(30u, (size_t)nums[0]);
+    assert_eq(30u, (size_t)nums[1]);
+    assert_eq(30u, (size_t)nums[2]);
+    assert_eq(10u, (size_t)nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, (size_t)num_cols[0]);
+    assert_eq(17u, (size_t)num_cols[1]);
+    assert_eq(12u, (size_t)num_cols[2]);
+    assert_eq(17u, (size_t)num_cols[3]);
+    assert_eq(12u, (size_t)num_cols[4]);
+    assert_eq(17u, (size_t)num_cols[5]);
+    assert_eq(22u, (size_t)num_cols[6]);
+    assert_eq(26u, (size_t)num_cols[7]);
+
+    return 0;
+}

spec/test_rewind.d

@@ -0,0 +1,101 @@
+import testparser;
+import testutils;
+
+/* Grammar and scenario: see test_rewind.c. */
+
+int[16] nums;
+size_t n_nums;
+uint[16] num_cols;
+size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info.token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info.token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info.token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info.position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    string input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(4u, n_nums);
+    assert_eq(30, nums[0]);
+    assert_eq(30, nums[1]);
+    assert_eq(30, nums[2]);
+    assert_eq(10, nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, num_cols[0]);
+    assert_eq(17u, num_cols[1]);
+    assert_eq(12u, num_cols[2]);
+    assert_eq(17u, num_cols[3]);
+    assert_eq(12u, num_cols[4]);
+    assert_eq(17u, num_cols[5]);
+    assert_eq(22u, num_cols[6]);
+    assert_eq(26u, num_cols[7]);
+}

spec/test_rewind.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    let mut c = p_context_new(b"repeat 3 { 10 + 20 } 5 + 5");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+
+    assert_eq!(vec![30, 30, 30, 10], c.nums);
+    assert_eq!(vec![12, 17, 12, 17, 12, 17, 22, 26], c.num_cols);
+
+    p_context_delete(c);
+}

spec/test_set_position.c

@@ -0,0 +1,109 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_set_position() overrides the default (1, 1) starting
+     * position so that lexed tokens and error positions are reported
+     * relative to the caller-supplied position. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t pos = p_position(context);
+        assert_eq(1u, (size_t)pos.row);
+        assert_eq(1u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() overrides the initial position; subsequent lex calls
+     * report token positions relative to the set position. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {5u, 20u};
+        p_set_position(context, initial);
+        p_position_t pos = p_position(context);
+        assert_eq(5u, (size_t)pos.row);
+        assert_eq(20u, (size_t)pos.col);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(20u, (size_t)token_info.position.col);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(5u, (size_t)token_info.position.row);
+        assert_eq(21u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a full parse: successful parse still works and
+     * text_position tracking is relative to the set starting point. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {3u, 7u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        p_context_delete(context);
+    }
+
+    /* p_set_position() before a parse that fails: the reported error
+     * position is relative to the set starting point. */
+    {
+        char const * input = "aa";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_position_t initial = {10u, 2u};
+        p_set_position(context, initial);
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        /* Error is at the second `a`, which is one column past the initial
+         * column. */
+        assert_eq(10u, (size_t)err_pos.row);
+        assert_eq(3u, (size_t)err_pos.col);
+        p_context_delete(context);
+    }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input: both
+     * tokens are lexed, then the cursor and text position are rewound to the
+     * start so that the same tokens are produced again with the same reported
+     * positions. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert_eq(0u, start_index);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(2u, p_input_index(context));
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert_eq(0u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_set_position.d

@@ -0,0 +1,90 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_set_position.c for details. */
+
+    /* Baseline: without p_set_position(), positions start at (1, 1). */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_position_t pos = p_position(context);
+        assert(pos.row == 1);
+        assert(pos.col == 1);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+
+    /* p_set_position() overrides the initial position. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(5u, 20u));
+        p_position_t pos = p_position(context);
+        assert(pos.row == 5);
+        assert(pos.col == 20);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 20);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(token_info.position.row == 5);
+        assert(token_info.position.col == 21);
+    }
+
+    /* p_set_position() before a full parse still parses successfully. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(3u, 7u));
+        assert(p_parse_Start(context) == P_SUCCESS);
+    }
+
+    /* p_set_position() before a parse that fails: error position is
+     * relative to the set starting point. */
+    {
+        string input = "aa";
+        p_context_t * context = p_context_new(input);
+        p_set_position(context, p_position_t(10u, 2u));
+        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
+        p_position_t err_pos = p_position(context);
+        assert(err_pos.row == 10);
+        assert(err_pos.col == 3);
+    }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert(start_index == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 2);
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert(p_input_index(context) == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
+}

spec/test_set_position.rs

@@ -0,0 +1,66 @@
+use testparser::*;
+
+fn main() {
+    let mut ti = p_token_info_t::default();
+
+    /* Baseline: default (1, 1). */
+    let mut c = p_context_new(b"ab");
+    let pos = p_position(&c);
+    assert_eq!(1, pos.row);
+    assert_eq!(1, pos.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, ti.position.row);
+    assert_eq!(1, ti.position.col);
+    p_context_delete(c);
+
+    /* p_set_position overrides the initial position. */
+    let mut c = p_context_new(b"ab");
+    p_set_position(&mut c, p_position_t { row: 5, col: 20 });
+    let pos = p_position(&c);
+    assert_eq!(5, pos.row);
+    assert_eq!(20, pos.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(5, ti.position.row);
+    assert_eq!(20, ti.position.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(5, ti.position.row);
+    assert_eq!(21, ti.position.col);
+    p_context_delete(c);
+
+    /* Set position before a full parse. */
+    let mut c = p_context_new(b"ab");
+    p_set_position(&mut c, p_position_t { row: 3, col: 7 });
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    /* Set position before a failing parse: error position is relative. */
+    let mut c = p_context_new(b"aa");
+    p_set_position(&mut c, p_position_t { row: 10, col: 2 });
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    let ep = p_position(&c);
+    assert_eq!(10, ep.row);
+    assert_eq!(3, ep.col);
+    p_context_delete(c);
+
+    /* p_set_input_index rewinds the byte cursor to re-read a section. */
+    let mut c = p_context_new(b"ab");
+    let start_index = p_input_index(&c);
+    let start_position = p_position(&c);
+    assert_eq!(0, start_index);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(2, p_input_index(&c));
+    p_set_input_index(&mut c, start_index);
+    p_set_position(&mut c, start_position);
+    assert_eq!(0, p_input_index(&c));
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, ti.position.row);
+    assert_eq!(1, ti.position.col);
+    p_context_delete(c);
+}